clean

### Describe Your Changes

Some messages were written to `stdout` using `fmt.Printf` and
`fmt.Println`, while the other messages like import statistics were
written to `stderr` through the `log` package.

This led to ordering problems where the `Import finished!` +
`VictoriaMetrics importer stats` messages, which expected to be the last
messages, appeared before `Continue import process with filter`
messages, creating confusing output for users.

```
2025/08/20 13:07:26 Import finished!
2025/08/20 13:07:26 VictoriaMetrics importer stats:
  time spent while importing: 20h49m10.8497184s;
  total bytes: 277.1 GB;
  bytes/s: 3.7 MB;
  requests: 7978614;
  requests retries: 0;
2025/08/20 13:07:26 Total time: 20h49m10.851006088s
Continue import process with filter
        filter: match[]={__name__!=""}
        start: 2025-08-08T00:00:00Z
        end: 2025-08-15T00:00:00Z:
Continue import process with filter
        filter: match[]={__name__!=""}
        start: 2025-08-15T00:00:00Z
        end: 2025-08-19T16:18:15Z:
```


### Checklist

The following checks are **mandatory**:

- [x] My change adheres to [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
- [x] My change adheres to [VictoriaMetrics development
goals](https://docs.victoriametrics.com/victoriametrics/goals/).

.github/copilot-instructions.md

@@ -0,0 +1,23 @@
+# Project Overview
+
+VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
+
+## Folder Structure
+
+- `/app`: Contains the compilable binaries.
+- `/lib`: Contains the golang reusable libraries
+- `/docs/victoriametrics`: Contains documentation for the project.
+- `/apptest/tests`: Contains integration tests.
+
+## Libraries and Frameworks
+
+- Backend: Golang, no framework. Use third-party libraries sparingly.
+- Frontend: React.
+
+## Code review guidelines
+
+Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
+Verify the entry is under the ## tip section and matches the structure and style of existing entries.
+Chore-only changes may be omitted from the changelog.
+
+

.github/pull_request_template.md

@@ -7,3 +7,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 The following checks are **mandatory**:
 
 - [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
+- [ ] My change adheres to [VictoriaMetrics development goals](https://docs.victoriametrics.com/victoriametrics/goals/).

.github/workflows/build.yml

@@ -7,16 +7,20 @@ on:
       - master
     paths:
       - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Dockerfile'
       - '**/Makefile'
+      - '!app/vmui/**'
+      - '.github/workflows/build.yml'
   pull_request:
     branches:
       - cluster
       - master
     paths:
       - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
+      - '**/Dockerfile'
       - '**/Makefile'
+      - '!app/vmui/**'
+      - '.github/workflows/build.yml'
 
 permissions:
   contents: read
@@ -27,28 +31,39 @@ concurrency:
 
 jobs:
   build:
-    name: Build
+    name: ${{ matrix.os }}-${{ matrix.arch }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+          - os: linux
+            arch: arm
+          - os: linux
+            arch: ppc64le
+          - os: linux
+            arch: 386
+          - os: freebsd
+            arch: amd64
+          - os: openbsd
+            arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         id: go
         uses: actions/setup-go@v5
         with:
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
           go-version: stable
-          cache: false
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
-      - name: Run crossbuild
-        run: make crossbuild
+      - name: Build vmcluster for ${{ matrix.os }}-${{ matrix.arch }}
+        run: make vmcluster-${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/codeql-analysis-go.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Go
         id: go

.github/workflows/codeql-analysis-js-ts.yml

@@ -1,46 +0,0 @@
-name: 'CodeQL JS/TS'
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: javascript-typescript
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: 'language:js/ts'

.github/workflows/docs.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: main
+name: test
 
 on:
   push:
@@ -7,12 +7,16 @@ on:
       - master
     paths:
       - '**.go'
+      - 'go.*'
+      - '.github/workflows/main.yml'
   pull_request:
     branches:
       - cluster
       - master
     paths:
       - '**.go'
+      - 'go.*'
+      - '.github/workflows/main.yml'
 
 permissions:
   contents: read
@@ -21,39 +25,41 @@ concurrency:
   cancel-in-progress: true
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
 
+
 jobs:
   lint:
     name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         id: go
         uses: actions/setup-go@v5
         with:
-          cache: false
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
           go-version: stable
 
-      - name: Cache Go artifacts
+
+      - name: Cache golangci-lint
         uses: actions/cache@v4
         with:
           path: |
-            ~/.cache/go-build
+            ~/.cache/golangci-lint
             ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-all-
+          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
 
       - name: Run check-all
         run: |
           make check-all
           git diff --exit-code
 
-  test:
-    name: test
-    needs: lint
+  unit:
+    name: unit
     runs-on: ubuntu-latest
 
     strategy:
@@ -65,25 +71,18 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         id: go
         uses: actions/setup-go@v5
         with:
-          cache: false
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
           go-version: stable
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
       - name: Run tests
         run: GOGC=10 make ${{ matrix.scenario}}
 
@@ -92,31 +91,23 @@ jobs:
         with:
           files: ./coverage.txt
 
-  integration-test:
-    name: integration-test
-    needs: [lint, test]
+  integration:
+    name: integration
     runs-on: ubuntu-latest
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Go
         id: go
         uses: actions/setup-go@v5
         with:
-          cache: false
+          cache-dependency-path: |
+            go.sum
+            Makefile
+            app/**/Makefile
           go-version: stable
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
       - name: Run integration tests
         run: make integration-test

.github/workflows/vmui.yml

@@ -0,0 +1,82 @@
+name: vmui
+
+on:
+  push:
+    branches:
+      - cluster
+      - master
+    paths:
+      - 'app/vmui/packages/vmui/**'
+      - '.github/workflows/vmui.yml'
+  pull_request:
+    branches:
+      - cluster
+      - master
+    paths:
+      - 'app/vmui/packages/vmui/**'
+      - '.github/workflows/vmui.yml'
+
+permissions:
+  contents: read
+  packages: read
+  pull-requests: read
+  checks: write
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  vmui-checks:
+    name: VMUI Checks (lint, test, typecheck)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24.x'
+
+      - name: Cache node-modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            app/vmui/packages/vmui/node_modules
+          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
+          restore-keys: vmui-artifacts-${{ runner.os }}-
+
+      - name: Run lint
+        id: lint
+        run: make vmui-lint
+        continue-on-error: true
+
+      - name: Run tests
+        id: test
+        run: make vmui-test
+        continue-on-error: true
+
+      - name: Run typecheck
+        id: typecheck
+        run: make vmui-typecheck
+        continue-on-error: true
+
+      - name: Annotate Code Linting Results
+        uses: ataylorme/eslint-annotate-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          report-json: app/vmui/packages/vmui/vmui-lint-report.json
+
+      - name: Check overall status
+        run: |
+          echo "Lint status: ${{ steps.lint.outcome }}"
+          echo "Test status: ${{ steps.test.outcome }}"
+          echo "Typecheck status: ${{ steps.typecheck.outcome }}"
+
+          if [[ "${{ steps.lint.outcome }}" == "failure" || "${{ steps.test.outcome }}" == "failure" || "${{ steps.typecheck.outcome }}" == "failure" ]]; then
+            echo "One or more checks failed"
+            exit 1
+          else
+            echo "All checks passed"
+          fi

.golangci.yml

@@ -1,22 +1,29 @@
-run:
-  timeout: 2m
-
+version: "2"
 linters:
-  enable:
-  - revive
-
-issues:
-  exclude-rules:
-  - linters:
-    - staticcheck
-    text: "SA(4003|1019|5011):"
-  include:
-  - EXC0012
-  - EXC0014
-
-linters-settings:
-  errcheck:
-    exclude-functions:
-      - "fmt.Fprintf"
-      - "fmt.Fprint"
-      - "(net/http.ResponseWriter).Write"
+  settings:
+    errcheck:
+      exclude-functions:
+        - fmt.Fprintf
+        - fmt.Fprint
+        - (net/http.ResponseWriter).Write
+  exclusions:
+    generated: lax
+    presets:
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - linters:
+          - staticcheck
+        text: 'SA(4003|1019|5011):'
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

Makefile

@@ -11,11 +11,14 @@ ifeq ($(PKG_TAG),)
 PKG_TAG := $(BUILDINFO_TAG)
 endif
 
-EXTRA_DOCKER_TAG_SUFFIX ?= EXTRA_DOCKER_TAG_SUFFIX
+EXTRA_DOCKER_TAG_SUFFIX ?=
+EXTRA_GO_BUILD_TAGS ?=
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
+GOLANGCI_LINT_VERSION := 2.4.0
+
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
@@ -24,6 +27,7 @@ include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
 include package/release/Makefile
+include benchmarks/Makefile
 
 all: \
 	vminsert \
@@ -88,8 +92,10 @@ vmcluster-darwin-arm64: \
 	vmselect-darwin-arm64 \
 	vmstorage-darwin-arm64
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 crossbuild: vmcluster-crossbuild
 
+# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 vmcluster-crossbuild:
 	$(MAKE_PARALLEL) vmcluster-linux-amd64 \
 		vmcluster-linux-arm64 \
@@ -149,10 +155,6 @@ publish-latest:
 	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
 	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
 
-publish-victoria-logs-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-logs $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG) APP_NAME=vlogscli $(MAKE) publish-via-docker-latest
-
 publish-release:
 	rm -rf bin/*
 	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
@@ -263,7 +265,11 @@ test-full:
 test-full-386:
 	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
-integration-test: all vmctl vmbackup vmrestore victoria-logs
+integration-test:
+	$(MAKE) apptest
+
+apptest:
+	$(MAKE) all vmctl vmbackup vmrestore
 	go test ./apptest/... -skip="^TestSingle.*"
 
 benchmark:
@@ -281,16 +287,16 @@ vendor-update:
 	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-goos-goarch:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-windows-goarch:
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
@@ -303,7 +309,7 @@ golangci-lint: install-golangci-lint
 	GOEXPERIMENT=synctest golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7
+	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

app/victoria-logs/Makefile

@@ -1,113 +0,0 @@
-# All these commands must run from repository root.
-
-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) app-local
-
-victoria-logs-race:
-	APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
-
-victoria-logs-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker
-
-victoria-logs-pure-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
-
-victoria-logs-linux-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
-
-victoria-logs-linux-arm-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
-
-victoria-logs-linux-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
-
-victoria-logs-linux-ppc64le-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
-
-victoria-logs-linux-386-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
-
-victoria-logs-darwin-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
-
-victoria-logs-darwin-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
-
-victoria-logs-freebsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
-
-victoria-logs-openbsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
-
-victoria-logs-windows-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
-
-package-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker
-
-package-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
-
-package-victoria-logs-amd64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
-
-package-victoria-logs-arm:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
-
-package-victoria-logs-arm64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
-
-package-victoria-logs-ppc64le:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
-
-package-victoria-logs-386:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-386
-
-publish-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) publish-via-docker
-
-victoria-logs-linux-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-ppc64le:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-s390x:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-loong64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-386:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-freebsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-openbsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-windows-amd64:
-	GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
-
-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) app-local-pure
-
-run-victoria-logs:
-	mkdir -p victoria-logs-data
-	DOCKER_OPTS='-v $(shell pwd)/victoria-logs-data:/victoria-logs-data' \
-	APP_NAME=victoria-logs \
-	ARGS='' \
-	$(MAKE) run-via-docker

app/victoria-logs/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-EXPOSE 9428
-
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -1,113 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-)
-
-var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9428"}
-	}
-	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
-	startTime := time.Now()
-
-	vlstorage.Init()
-	vlselect.Init()
-
-	insertutil.SetLogRowsStorage(&vlstorage.Storage{})
-	vlinsert.Init()
-
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
-	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
-
-	pushmetrics.Init()
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
-
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
-
-	vlinsert.Stop()
-	vlselect.Stop()
-	vlstorage.Stop()
-
-	fs.MustStopDirRemover()
-
-	logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
-}
-
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
-			return false
-		}
-		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/'>https://docs.victoriametrics.com/victorialogs/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints:</br>")
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"select/vmui", "Web UI for VictoriaLogs"},
-			{"metrics", "available service metrics"},
-			{"flags", "command-line flags"},
-		})
-		return true
-	}
-	if vlinsert.RequestHandler(w, r) {
-		return true
-	}
-	if vlselect.RequestHandler(w, r) {
-		return true
-	}
-	if vlstorage.RequestHandler(w, r) {
-		return true
-	}
-	return false
-}
-
-func usage() {
-	const s = `
-victoria-logs is a log management and analytics service.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/
-`
-	flagutil.Usage(s)
-}

app/victoria-logs/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 9428
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG TARGETARCH
-COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/vlagent/Makefile

@@ -1,106 +0,0 @@
-# All these commands must run from repository root.
-
-vlagent:
-	APP_NAME=vlagent $(MAKE) app-local
-
-vlagent-race:
-	APP_NAME=vlagent RACE=-race $(MAKE) app-local
-
-vlagent-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker
-
-vlagent-pure-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-pure
-
-vlagent-linux-amd64-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-linux-amd64
-
-vlagent-linux-arm-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-linux-arm
-
-vlagent-linux-arm64-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-linux-arm64
-
-vlagent-linux-ppc64le-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-linux-ppc64le
-
-vlagent-linux-386-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-linux-386
-
-vlagent-darwin-amd64-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-darwin-amd64
-
-vlagent-darwin-arm64-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-darwin-arm64
-
-vlagent-freebsd-amd64-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-freebsd-amd64
-
-vlagent-openbsd-amd64-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-openbsd-amd64
-
-vlagent-windows-amd64-prod:
-	APP_NAME=vlagent $(MAKE) app-via-docker-windows-amd64
-
-package-vlagent:
-	APP_NAME=vlagent $(MAKE) package-via-docker
-
-package-vlagent-pure:
-	APP_NAME=vlagent $(MAKE) package-via-docker-pure
-
-package-vlagent-amd64:
-	APP_NAME=vlagent $(MAKE) package-via-docker-amd64
-
-package-vlagent-arm:
-	APP_NAME=vlagent $(MAKE) package-via-docker-arm
-
-package-vlagent-arm64:
-	APP_NAME=vlagent $(MAKE) package-via-docker-arm64
-
-package-vlagent-ppc64le:
-	APP_NAME=vlagent $(MAKE) package-via-docker-ppc64le
-
-package-vlagent-386:
-	APP_NAME=vlagent $(MAKE) package-via-docker-386
-
-publish-vlagent:
-	APP_NAME=vlagent $(MAKE) publish-via-docker
-
-vlagent-linux-amd64:
-	APP_NAME=vlagent CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlagent-linux-arm:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-vlagent-linux-arm64:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlagent-linux-ppc64le:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vlagent-linux-s390x:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vlagent-linux-loong64:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vlagent-linux-386:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vlagent-darwin-amd64:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlagent-darwin-arm64:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlagent-freebsd-amd64:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlagent-openbsd-amd64:
-	APP_NAME=vlagent CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlagent-windows-amd64:
-	GOARCH=amd64 APP_NAME=vlagent $(MAKE) app-local-windows-goarch
-
-vlagent-pure:
-	APP_NAME=vlagent $(MAKE) app-local-pure

app/vlagent/README.md

@@ -1,3 +1 @@
-See vlagent docs [here](https://docs.victoriametrics.com/victorialogs/vlagent/).
-
-vlagent docs can be edited at [docs/vlagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victorialogs/vlagent.md).
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlagent/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-EXPOSE 9429
-
-ENTRYPOINT ["/vlagent-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vlagent-prod

app/vlagent/main.go

@@ -1,97 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-)
-
-var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
-		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vlagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	remotewrite.InitSecretFlags()
-	logger.Init()
-
-	remotewrite.Init()
-	vlinsert.Init()
-
-	insertutil.SetLogRowsStorage(&remotewrite.Storage{})
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9429"}
-	}
-	logger.Infof("starting vlagent at %q...", listenAddrs)
-	startTime := time.Now()
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
-	logger.Infof("started vlagent in %.3f seconds", time.Since(startTime).Seconds())
-
-	pushmetrics.Init()
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
-
-	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	vlinsert.Stop()
-	remotewrite.Stop()
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
-	logger.Infof("successfully stopped vlagent in %.3f seconds", time.Since(startTime).Seconds())
-}
-
-// RequestHandler handles insert requests for VictoriaLogs
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
-			return false
-		}
-		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, "<h2>vlagent</h2>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/vlagent/'>https://docs.victoriametrics.com/victorialogs/vlagent/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints:</br>")
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"metrics", "available service metrics"},
-			{"flags", "command-line flags"},
-		})
-		return true
-	}
-	return vlinsert.RequestHandler(w, r)
-}
-
-func usage() {
-	const s = `
-vlagent collects logs via popular data ingestion protocols and routes it to VictoriaLogs.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/vlagent/ .
-`
-	flagutil.Usage(s)
-}

app/vlagent/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 9429
-ENTRYPOINT ["/vlagent-prod"]
-ARG TARGETARCH
-COPY vlagent-linux-${TARGETARCH}-prod ./vlagent-prod

app/vlagent/remotewrite/client.go

@@ -1,462 +0,0 @@
-package remotewrite
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
-		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data ")
-	sendTimeout      = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxTime")
-	retryMaxTime     = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
-	proxyURL         = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
-		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
-
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
-	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
-	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
-		"to the corresponding -remoteWrite.url")
-	tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
-	tlsCAFile  = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
-		"By default, system CA is used")
-	tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
-		"By default, the server name from -remoteWrite.url is used")
-
-	headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
-		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
-		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
-
-	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
-	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
-	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
-		"The file is re-read every second")
-	bearerToken     = flagutil.NewArrayString("remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
-	bearerTokenFile = flagutil.NewArrayString("remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
-		"The token is re-read from the file every second")
-
-	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
-	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
-	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
-	oauth2EndpointParams   = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
-		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
-	oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
-	oauth2Scopes   = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
-)
-
-type client struct {
-	sanitizedURL   string
-	remoteWriteURL string
-
-	fq *persistentqueue.FastQueue
-	hc *http.Client
-
-	retryMinInterval time.Duration
-	retryMaxTime     time.Duration
-
-	sendBlock func(block []byte) bool
-	authCfg   *promauth.Config
-
-	rl *ratelimiter.RateLimiter
-
-	bytesSent       *metrics.Counter
-	blocksSent      *metrics.Counter
-	requestDuration *metrics.Histogram
-	requestsOKCount *metrics.Counter
-	errorsCount     *metrics.Counter
-	packetsDropped  *metrics.Counter
-	rateLimit       *metrics.Gauge
-	retriesCount    *metrics.Counter
-	sendDuration    *metrics.FloatCounter
-
-	wg     sync.WaitGroup
-	stopCh chan struct{}
-}
-
-func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
-	authCfg, err := getAuthConfig(argIdx)
-	if err != nil {
-		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
-	}
-
-	tr := httputil.NewTransport(false, "vlagent_remotewrite")
-	tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
-	tr.MaxConnsPerHost = 2 * concurrency
-	tr.MaxIdleConnsPerHost = 2 * concurrency
-	tr.IdleConnTimeout = time.Minute
-	tr.WriteBufferSize = 64 * 1024
-
-	pURL := proxyURL.GetOptionalArg(argIdx)
-	if len(pURL) > 0 {
-		if !strings.Contains(pURL, "://") {
-			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
-		}
-		pu, err := url.Parse(pURL)
-		if err != nil {
-			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: %s", pURL, err)
-		}
-		tr.Proxy = http.ProxyURL(pu)
-	}
-	hc := &http.Client{
-		Transport: authCfg.NewRoundTripper(tr),
-		Timeout:   sendTimeout.GetOptionalArg(argIdx),
-	}
-	c := &client{
-		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
-		authCfg:          authCfg,
-		fq:               fq,
-		hc:               hc,
-		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
-		retryMaxTime:     retryMaxTime.GetOptionalArg(argIdx),
-		stopCh:           make(chan struct{}),
-	}
-	c.sendBlock = c.sendBlockHTTP
-	return c
-}
-
-func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
-	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
-	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
-		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
-		c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
-	}
-	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
-	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
-	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(rateLimit.GetOptionalArg(argIdx))
-	})
-	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vlagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
-	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
-	c.errorsCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_errors_total{url=%q}`, c.sanitizedURL))
-	c.packetsDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_packets_dropped_total{url=%q}`, c.sanitizedURL))
-	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
-	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vlagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
-	metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(*queues)
-	})
-	for i := 0; i < concurrency; i++ {
-		c.wg.Add(1)
-		go func() {
-			defer c.wg.Done()
-			c.runWorker()
-		}()
-	}
-	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
-}
-
-func (c *client) MustStop() {
-	close(c.stopCh)
-	c.wg.Wait()
-	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
-}
-
-func getAuthConfig(argIdx int) (*promauth.Config, error) {
-	headersValue := headers.GetOptionalArg(argIdx)
-	var hdrs []string
-	if headersValue != "" {
-		hdrs = strings.Split(headersValue, "^^")
-	}
-	username := basicAuthUsername.GetOptionalArg(argIdx)
-	password := basicAuthPassword.GetOptionalArg(argIdx)
-	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
-	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
-		basicAuthCfg = &promauth.BasicAuthConfig{
-			Username:     username,
-			Password:     promauth.NewSecret(password),
-			PasswordFile: passwordFile,
-		}
-	}
-
-	token := bearerToken.GetOptionalArg(argIdx)
-	tokenFile := bearerTokenFile.GetOptionalArg(argIdx)
-
-	var oauth2Cfg *promauth.OAuth2Config
-	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
-	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
-	if clientSecretFile != "" || clientSecret != "" {
-		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
-		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
-		}
-		oauth2Cfg = &promauth.OAuth2Config{
-			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
-			ClientSecret:     promauth.NewSecret(clientSecret),
-			ClientSecretFile: clientSecretFile,
-			EndpointParams:   endpointParams,
-			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
-			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
-		}
-	}
-
-	tlsCfg := &promauth.TLSConfig{
-		CAFile:             tlsCAFile.GetOptionalArg(argIdx),
-		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
-		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
-		ServerName:         tlsServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
-	}
-
-	opts := &promauth.Options{
-		BasicAuth:       basicAuthCfg,
-		BearerToken:     token,
-		BearerTokenFile: tokenFile,
-		OAuth2:          oauth2Cfg,
-		TLSConfig:       tlsCfg,
-		Headers:         hdrs,
-	}
-	authCfg, err := opts.NewConfig()
-	if err != nil {
-		return nil, fmt.Errorf("cannot populate auth config for remoteWrite idx: %d, err: %w", argIdx, err)
-	}
-	return authCfg, nil
-}
-
-func (c *client) runWorker() {
-	var ok bool
-	var block []byte
-	ch := make(chan bool, 1)
-	for {
-		block, ok = c.fq.MustReadBlock(block[:0])
-		if !ok {
-			return
-		}
-		if len(block) == 0 {
-			// skip empty data blocks from sending
-			continue
-		}
-		go func() {
-			startTime := time.Now()
-			ch <- c.sendBlock(block)
-			c.sendDuration.Add(time.Since(startTime).Seconds())
-		}()
-		select {
-		case ok := <-ch:
-			if ok {
-				// The block has been sent successfully
-				continue
-			}
-			// Return unsent block to the queue.
-			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
-			return
-		case <-c.stopCh:
-			// c must be stopped. Wait for a while in the hope the block will be sent.
-			graceDuration := 5 * time.Second
-			select {
-			case ok := <-ch:
-				if !ok {
-					// Return unsent block to the queue.
-					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
-				}
-			case <-time.After(graceDuration):
-				// Return unsent block to the queue.
-				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
-			}
-			return
-		}
-	}
-}
-
-func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
-	req, err := c.newRequest(url, body)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := c.hc.Do(req)
-	if err == nil {
-		return resp, nil
-	}
-	if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
-		return nil, err
-	}
-	// It is likely connection became stale or timed out during the first request.
-	// Make another attempt in hope request will succeed.
-	// If not, the error should be handled by the caller as usual.
-	// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
-	req, err = c.newRequest(url, body)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	resp, err = c.hc.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	return resp, nil
-}
-
-func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
-	reqBody := bytes.NewBuffer(body)
-	req, err := http.NewRequest(http.MethodPost, url, reqBody)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
-	}
-	err = c.authCfg.SetHeaders(req, true)
-	if err != nil {
-		return nil, err
-	}
-	h := req.Header
-	h.Set("User-Agent", "vlagent")
-	h.Set("Content-Encoding", "zstd")
-	h.Set("Content-Type", "application/octet-stream")
-
-	return req, nil
-}
-
-// sendBlockHTTP sends the given block to c.remoteWriteURL.
-//
-// The function returns false only if c.stopCh is closed.
-// Otherwise, it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
-	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
-	retriesCount := 0
-
-again:
-	startTime := time.Now()
-	resp, err := c.doRequest(c.remoteWriteURL, block)
-	c.requestDuration.UpdateDuration(startTime)
-	if err != nil {
-		c.errorsCount.Inc()
-		retryDuration *= 2
-		if retryDuration > maxRetryDuration {
-			retryDuration = maxRetryDuration
-		}
-		remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
-			len(block), c.sanitizedURL, err, retryDuration.Seconds())
-		t := timerpool.Get(retryDuration)
-		select {
-		case <-c.stopCh:
-			timerpool.Put(t)
-			return false
-		case <-t.C:
-			timerpool.Put(t)
-		}
-		c.retriesCount.Inc()
-		goto again
-	}
-
-	statusCode := resp.StatusCode
-	if statusCode/100 == 2 {
-		_ = resp.Body.Close()
-		c.requestsOKCount.Inc()
-		c.bytesSent.Add(len(block))
-		c.blocksSent.Inc()
-		return true
-	}
-
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vlagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
-	if statusCode == 400 || statusCode == 404 {
-		logBlockRejected(block, c.sanitizedURL, resp)
-		_ = resp.Body.Close()
-		c.packetsDropped.Inc()
-		return true
-	}
-	// Unexpected status code returned
-	retriesCount++
-	retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
-	retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
-
-	// Handle response
-	body, err := io.ReadAll(resp.Body)
-	_ = resp.Body.Close()
-	if err != nil {
-		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
-	} else {
-		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
-			"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
-	}
-	t := timerpool.Get(retryDuration)
-	select {
-	case <-c.stopCh:
-		timerpool.Put(t)
-		return false
-	case <-t.C:
-		timerpool.Put(t)
-	}
-	c.retriesCount.Inc()
-	goto again
-}
-
-var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
-var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
-
-// getRetryDuration returns retry duration.
-// retryAfterDuration has the highest priority.
-// If retryAfterDuration is not specified, retryDuration gets doubled.
-// retryDuration can't exceed maxRetryDuration.
-//
-// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
-func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
-	// retryAfterDuration has the highest priority duration
-	if retryAfterDuration > 0 {
-		return timeutil.AddJitterToDuration(retryAfterDuration)
-	}
-
-	// default backoff retry policy
-	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
-	}
-
-	return retryDuration
-}
-
-func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
-			"failed to read response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, err)
-	} else {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, string(body))
-	}
-}
-
-// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
-// retryAfterString should be in either HTTP-date or a number of seconds.
-// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
-func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
-	if retryAfterString == "" {
-		return retryAfterDuration
-	}
-
-	defer func() {
-		v := retryAfterDuration.Seconds()
-		logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
-	}()
-
-	// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
-	if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
-		return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
-	}
-	// Retry-After could be in seconds.
-	if seconds, err := strconv.Atoi(retryAfterString); err == nil {
-		return time.Duration(seconds) * time.Second
-	}
-
-	return 0
-}

app/vlagent/remotewrite/client_test.go

@@ -1,99 +0,0 @@
-package remotewrite
-
-import (
-	"math"
-	"net/http"
-	"testing"
-	"time"
-)
-
-func TestCalculateRetryDuration(t *testing.T) {
-	// `testFunc` call `calculateRetryDuration` for `n` times
-	// and evaluate if the result of `calculateRetryDuration` is
-	// 1. >= expectMinDuration
-	// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
-	f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
-		t.Helper()
-
-		for i := 0; i < n; i++ {
-			retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
-		}
-
-		expectMaxDuration := helper(expectMinDuration)
-		expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
-
-		if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
-			t.Fatalf(
-				"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
-				expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
-				retryDuration.Milliseconds(),
-			)
-		}
-	}
-
-	// Call calculateRetryDuration for 1 time.
-	{
-		// default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-		// default backoff policy exceed max limit"
-		f(0, 10*time.Minute, 1, time.Minute)
-
-		// retry after > default backoff policy
-		f(10*time.Second, 1*time.Second, 1, 10*time.Second)
-		// retry after < default backoff policy
-		f(1*time.Second, 10*time.Second, 1, 1*time.Second)
-		// retry after invalid and < default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-
-	}
-
-	// Call calculateRetryDuration for multiple times.
-	{
-		// default backoff policy 2 times
-		f(0, time.Second, 2, 4*time.Second)
-		// default backoff policy 3 times
-		f(0, time.Second, 3, 8*time.Second)
-		// default backoff policy N times exceed max limit
-		f(0, time.Second, 10, time.Minute)
-
-		// retry after 120s 1 times
-		f(120*time.Second, time.Second, 1, 120*time.Second)
-		// retry after 120s 2 times
-		f(120*time.Second, time.Second, 2, 120*time.Second)
-	}
-}
-
-func TestParseRetryAfterHeader(t *testing.T) {
-	f := func(retryAfterString string, expectResult time.Duration) {
-		t.Helper()
-
-		result := parseRetryAfterHeader(retryAfterString)
-		// expect `expectResult == result` when retryAfterString is in seconds or invalid
-		// expect the difference between result and expectResult to be lower than 10%
-		if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
-			t.Fatalf(
-				"incorrect retry after duration, want (ms): %d, got (ms): %d",
-				expectResult.Milliseconds(), result.Milliseconds(),
-			)
-		}
-	}
-
-	// retry after header in seconds
-	f("10", 10*time.Second)
-	// retry after header in date time
-	f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
-	// retry after header invalid
-	f("invalid-retry-after", 0)
-	// retry after header not in GMT
-	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
-}
-
-// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
-func helper(d time.Duration) time.Duration {
-	dv := d / 10
-	if dv > 10*time.Second {
-		dv = 10 * time.Second
-	}
-
-	return d + dv
-}

app/vlagent/remotewrite/pendinglogrows.go

@@ -1,158 +0,0 @@
-package remotewrite
-
-import (
-	"flag"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage.")
-	flushInterval        = flag.Duration("remoteWrite.flushInterval", time.Second, "Interval for flushing the data to remote storage. "+
-		"This option takes effect only when less than 2MB of data per second are pushed to -remoteWrite.url")
-)
-
-type pendingLogs struct {
-	lastFlushTime atomic.Uint64
-
-	// The queue to send blocks to.
-	fq *persistentqueue.FastQueue
-
-	// mu protects wr
-	mu sync.Mutex
-	wr writeRequest
-
-	stopCh            chan struct{}
-	periodicFlusherWG sync.WaitGroup
-}
-
-func newPendingLogs(fq *persistentqueue.FastQueue) *pendingLogs {
-	pl := &pendingLogs{
-		fq:     fq,
-		stopCh: make(chan struct{}),
-	}
-
-	pl.periodicFlusherWG.Add(1)
-	go func() {
-		defer pl.periodicFlusherWG.Done()
-		pl.periodicFlusher()
-	}()
-
-	return pl
-}
-
-func (pl *pendingLogs) add(lr *logstorage.LogRows) {
-	lr.ForEachRow(func(_ uint64, r *logstorage.InsertRow) {
-		pl.addLogRow(r)
-	})
-}
-
-func (pl *pendingLogs) addLogRow(r *logstorage.InsertRow) {
-	bb := bbPool.Get()
-	bb.B = r.Marshal(bb.B)
-
-	pl.mu.Lock()
-	_, _ = pl.wr.pendingData.Write(bb.B)
-	pl.wr.pendingLogRowsCount++
-	if len(pl.wr.pendingData.B) > maxUnpackedBlockSize.IntN() {
-		pl.mustFlushLocked()
-	}
-	pl.mu.Unlock()
-	bbPool.Put(bb)
-}
-
-func (pl *pendingLogs) mustFlushLocked() {
-	pl.lastFlushTime.Store(fasttime.UnixTimestamp())
-	pl.wr.push(func(b []byte) {
-		if !pl.fq.TryWriteBlock(b) {
-			logger.Fatalf("BUG: TryWriteBlock cannot return false")
-		}
-	})
-	pl.wr.reset()
-}
-
-func (pl *pendingLogs) periodicFlusher() {
-	flushSeconds := int64(flushInterval.Seconds())
-	if flushSeconds <= 0 {
-		flushSeconds = 1
-	}
-	d := timeutil.AddJitterToDuration(*flushInterval)
-	ticker := time.NewTicker(d)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-pl.stopCh:
-			pl.mu.Lock()
-			pl.mustFlushOnStop()
-			pl.mu.Unlock()
-			return
-		case <-ticker.C:
-			if fasttime.UnixTimestamp()-pl.lastFlushTime.Load() < uint64(flushSeconds) {
-				continue
-			}
-		}
-		pl.mu.Lock()
-		pl.mustFlushLocked()
-		pl.mu.Unlock()
-	}
-}
-
-// mustFlushOnStop force pushes wr data
-//
-// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
-func (pl *pendingLogs) mustFlushOnStop() {
-	pl.wr.push(pl.fq.MustWriteBlockIgnoreDisabledPQ)
-	pl.wr.reset()
-}
-
-func (pl *pendingLogs) mustStop() {
-	close(pl.stopCh)
-	pl.periodicFlusherWG.Wait()
-}
-
-type writeRequest struct {
-	pendingData         bytesutil.ByteBuffer
-	pendingLogRowsCount int64
-}
-
-func (wr *writeRequest) push(pushBlock func([]byte)) {
-	if len(wr.pendingData.B) == 0 {
-		return
-	}
-	b := wr.pendingData.B
-
-	zb := compressBufPool.Get()
-	zb.B = zstd.CompressLevel(zb.B[:0], b, 1)
-	zbLen := len(zb.B)
-	pushBlock(zb.B)
-	compressBufPool.Put(zb)
-	blockSizeBytes.Update(float64(zbLen))
-	blockSizeLogRows.Update(float64(wr.pendingLogRowsCount))
-}
-
-func (wr *writeRequest) reset() {
-	wr.pendingData.Reset()
-	wr.pendingLogRowsCount = 0
-}
-
-var (
-	blockSizeBytes   = metrics.NewHistogram(`vlagent_remotewrite_block_size_bytes`)
-	blockSizeLogRows = metrics.NewHistogram(`vlagent_remotewrite_block_size_rows`)
-)
-
-var (
-	compressBufPool bytesutil.ByteBufferPool
-	bbPool          bytesutil.ByteBufferPool
-)

app/vlagent/remotewrite/remotewrite.go

@@ -1,277 +0,0 @@
-package remotewrite
-
-import (
-	"flag"
-	"fmt"
-	"net/url"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
-)
-
-var (
-	remoteWriteURLs = flagutil.NewArrayString("remoteWrite.url", "Remote storage URL to write data to. It must support VictoriaLogs native protocol. "+
-		"Example url: http://<victorialogs-host>:9428/internal/insert. "+
-		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems.")
-	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
-		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
-		"Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. "+
-		"Disk usage is unlimited if the value is set to 0")
-
-	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vlagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
-		"See also -remoteWrite.maxDiskUsagePerURL")
-	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
-
-	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
-		"It is hidden by default, since it can contain sensitive info such as auth key")
-)
-
-// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
-var rwctxsGlobal []*remoteWriteCtx
-
-// Storage implements insertutil.LogRowsStorage interface
-type Storage struct{}
-
-// MustAddRows implements insertutil.LogRowsStorage interface
-func (*Storage) MustAddRows(lr *logstorage.LogRows) {
-	pushToRemoteStorages(lr)
-}
-
-// CanWriteData implements insertutil.LogRowsStorage interface
-func (*Storage) CanWriteData() error {
-	return nil
-}
-
-// maxQueues limits the maximum value for `-remoteWrite.queues`. There is no sense in setting too high value,
-// since it may lead to high memory usage due to big number of buffers.
-var maxQueues = cgroup.AvailableCPUs() * 16
-
-const persistentQueueDirname = "persistent-queue"
-
-// InitSecretFlags must be called after flag.Parse and before any logging.
-func InitSecretFlags() {
-	if !*showRemoteWriteURL {
-		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
-		flagutil.RegisterSecretFlag("remoteWrite.url")
-	}
-}
-
-// Init initializes remotewrite.
-//
-// It must be called after flag.Parse().
-//
-// Stop must be called for graceful shutdown.
-func Init() {
-	if len(*remoteWriteURLs) == 0 {
-		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
-	}
-	if *queues > maxQueues {
-		*queues = maxQueues
-	}
-	if *queues <= 0 {
-		*queues = 1
-	}
-	initRemoteWriteCtxs(*remoteWriteURLs)
-	dropDanglingQueues()
-}
-
-// Stop stops remotewrite.
-//
-// It is expected that nobody calls TryPush during and after the call to this func.
-func Stop() {
-	for _, rwctx := range rwctxsGlobal {
-		rwctx.mustStop()
-	}
-	rwctxsGlobal = nil
-}
-
-func dropDanglingQueues() {
-	// Remove dangling persistent queues, if any.
-	// This is required for the case when the number of queues has been changed or URL have been changed.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4014
-	//
-	// In case if there were many persistent queues with identical *remoteWriteURLs
-	// the queue with the last index will be dropped.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
-	for _, rwctx := range rwctxsGlobal {
-		existingQueues[rwctx.fq.Dirname()] = struct{}{}
-	}
-
-	queuesDir := filepath.Join(*tmpDataPath, persistentQueueDirname)
-	files := fs.MustReadDir(queuesDir)
-	removed := 0
-	for _, f := range files {
-		dirname := f.Name()
-		if _, ok := existingQueues[dirname]; !ok {
-			logger.Infof("removing dangling queue %q", dirname)
-			fullPath := filepath.Join(queuesDir, dirname)
-			fs.MustRemoveAll(fullPath)
-			removed++
-		}
-	}
-	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
-	}
-}
-
-func initRemoteWriteCtxs(urls []string) {
-	if len(urls) == 0 {
-		logger.Panicf("BUG: urls must be non-empty")
-	}
-
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / 10000
-	if maxInmemoryBlocks / *queues > 100 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 100 * *queues
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
-	rwctxs := make([]*remoteWriteCtx, len(urls))
-	rwctxIdx := make([]int, len(urls))
-	for i, remoteWriteURLRaw := range urls {
-		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
-		if err != nil {
-			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
-		}
-		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
-		if *showRemoteWriteURL {
-			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
-		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
-		rwctxIdx[i] = i
-	}
-
-	rwctxsGlobal = rwctxs
-}
-
-func pushToRemoteStorages(lr *logstorage.LogRows) {
-	rwctxs := rwctxsGlobal
-	if len(rwctxs) == 1 {
-		// fast path
-		rwctxs[0].push(lr)
-		return
-	}
-	// Push samples to remote storage systems in parallel in order to reduce
-	// the time needed for sending the data to multiple remote storage systems.
-	var wg sync.WaitGroup
-	for _, rwctx := range rwctxs {
-		wg.Add(1)
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
-			rwctx.push(lr)
-
-		}(rwctx)
-	}
-	wg.Wait()
-}
-
-type remoteWriteCtx struct {
-	idx int
-	fq  *persistentqueue.FastQueue
-	c   *client
-
-	pls        []*pendingLogs
-	pssNextIdx atomic.Uint64
-}
-
-func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
-	// protocol version is required by victoria-logs
-	q := remoteWriteURL.Query()
-	q.Set("version", netinsert.ProtocolVersion)
-	remoteWriteURL.RawQuery = q.Encode()
-
-	// strip query params, otherwise changing params resets pq
-	pqURL := *remoteWriteURL
-	pqURL.RawQuery = ""
-	pqURL.Fragment = ""
-	h := xxhash.Sum64([]byte(pqURL.String()))
-	queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
-	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArg(argIdx)
-	if maxPendingBytes != 0 && maxPendingBytes < persistentqueue.DefaultChunkFileSize {
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195
-		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
-		maxPendingBytes = persistentqueue.DefaultChunkFileSize
-	}
-
-	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, false)
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
-		return float64(fq.GetPendingBytes())
-	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_pending_inmemory_blocks{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
-		return float64(fq.GetInmemoryQueueLen())
-	})
-	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vlagent_remotewrite_queue_blocked{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
-		if fq.IsWriteBlocked() {
-			return 1
-		}
-		return 0
-	})
-
-	var c *client
-	switch remoteWriteURL.Scheme {
-	case "http", "https":
-		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
-	default:
-		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
-	}
-	c.init(argIdx, *queues, sanitizedURL)
-
-	// Initialize pss
-	plsLen := *queues
-	if n := cgroup.AvailableCPUs(); plsLen > n {
-		// There is no sense in running more than availableCPUs concurrent pendingLogs,
-		// since every pendingLogs can saturate up to a single CPU.
-		plsLen = n
-	}
-	pls := make([]*pendingLogs, plsLen)
-	for i := range pls {
-		pls[i] = newPendingLogs(fq)
-	}
-
-	rwctx := &remoteWriteCtx{
-		idx: argIdx,
-		fq:  fq,
-		c:   c,
-		pls: pls,
-	}
-
-	return rwctx
-}
-
-func (rwctx *remoteWriteCtx) push(lr *logstorage.LogRows) {
-	pls := rwctx.pls
-	idx := rwctx.pssNextIdx.Add(1) % uint64(len(pls))
-	pls[idx].add(lr)
-}
-
-func (rwctx *remoteWriteCtx) mustStop() {
-	for _, ps := range rwctx.pls {
-		ps.mustStop()
-	}
-	rwctx.idx = 0
-	rwctx.pls = nil
-	rwctx.fq.UnblockAllReaders()
-	rwctx.c.MustStop()
-	rwctx.c = nil
-
-	rwctx.fq.MustClose()
-	rwctx.fq = nil
-}

app/vlinsert/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlinsert/datadog/datadog.go

@@ -1,260 +0,0 @@
-package datadog
-
-import (
-	"bytes"
-	"fmt"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#stream-fields")
-	datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Comma-separated list of fields to ignore for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#dropping-fields")
-
-	maxRequestSize = flagutil.NewBytes("datadog.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single DataDog request")
-)
-
-var parserPool fastjson.ParserPool
-
-// RequestHandler processes Datadog insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/datadog/api/v1/validate":
-		fmt.Fprintf(w, `{}`)
-		return true
-	case "/insert/datadog/api/v2/logs":
-		return datadogLogsIngestion(w, r)
-	default:
-		return false
-	}
-}
-
-func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	startTime := time.Now()
-	v2LogsRequestsTotal.Inc()
-
-	var ts int64
-	if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
-		var err error
-		ts, err = strconv.ParseInt(tsValue, 10, 64)
-		if err != nil {
-			httpserver.Errorf(w, r, "could not parse dd-message-timestamp header value: %s", err)
-			return true
-		}
-		ts *= 1e6
-	} else {
-		ts = startTime.UnixNano()
-	}
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = *datadogStreamFields
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *datadogIgnoreFields
-	}
-
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("datadog", false)
-		err := readLogsRequest(ts, data, lmp)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read DataDog protocol data: %s", err)
-		return true
-	}
-
-	// update v2LogsRequestDuration only for successfully parsed requests
-	// There is no need in updating v2LogsRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	v2LogsRequestDuration.UpdateDuration(startTime)
-	w.WriteHeader(http.StatusAccepted)
-	fmt.Fprintf(w, `{}`)
-	return true
-}
-
-var (
-	v2LogsRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
-	v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
-)
-
-// datadog message field has two formats:
-//   - regular log message with string text
-//   - nested json format for serverless plugins
-//     which has the following format:
-//     {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
-//
-// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
-func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage.Field, error) {
-	switch v.Type() {
-	case fastjson.TypeString:
-		val := v.GetStringBytes()
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: bytesutil.ToUnsafeString(val),
-		})
-	case fastjson.TypeObject:
-		var firstErr error
-		v.GetObject().Visit(func(k []byte, v *fastjson.Value) {
-			if firstErr != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "_msg",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "status":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "status",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "lamdba":
-				obj, err := v.Object()
-				if err != nil {
-					firstErr = err
-					firstErr = fmt.Errorf("unexpected lambda value type for %q:%q; want object", k, v)
-					return
-				}
-				obj.Visit(func(k []byte, v *fastjson.Value) {
-					if firstErr != nil {
-						return
-					}
-					val, err := v.StringBytes()
-					if err != nil {
-						firstErr = fmt.Errorf("unexpected lambda label value type for %q:%q; want string", k, v)
-						return
-					}
-					fields = append(fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(val),
-					})
-				})
-
-			}
-		})
-	default:
-		return fields, fmt.Errorf("unsupported message type %q", v.Type().String())
-	}
-	return fields, nil
-}
-
-// readLogsRequest parses data according to DataDog logs format
-// https://docs.datadoghq.com/api/latest/logs/#send-logs
-func readLogsRequest(ts int64, data []byte, lmp insertutil.LogMessageProcessor) error {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-	records, err := v.Array()
-	if err != nil {
-		return fmt.Errorf("cannot extract array from parsed JSON: %w", err)
-	}
-
-	var fields []logstorage.Field
-	for _, r := range records {
-		o, err := r.Object()
-		if err != nil {
-			return fmt.Errorf("could not extract log record: %w", err)
-		}
-		o.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				fields, err = appendMsgFields(fields, v)
-				if err != nil {
-					return
-				}
-			case "timestamp":
-				val, e := v.Int64()
-				if e != nil {
-					err = fmt.Errorf("failed to parse timestamp for %q:%q", k, v)
-				}
-				if val > 0 {
-					ts = val * 1e6
-				}
-			case "ddtags":
-				// https://docs.datadoghq.com/getting_started/tagging/
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				var pair []byte
-				idx := 0
-				for idx >= 0 {
-					idx = bytes.IndexByte(val, ',')
-					if idx < 0 {
-						pair = val
-					} else {
-						pair = val[:idx]
-						val = val[idx+1:]
-					}
-					if len(pair) > 0 {
-						n := bytes.IndexByte(pair, ':')
-						if n < 0 {
-							// No tag value.
-							fields = append(fields, logstorage.Field{
-								Name:  bytesutil.ToUnsafeString(pair),
-								Value: "no_label_value",
-							})
-						}
-						fields = append(fields, logstorage.Field{
-							Name:  bytesutil.ToUnsafeString(pair[:n]),
-							Value: bytesutil.ToUnsafeString(pair[n+1:]),
-						})
-					}
-				}
-			default:
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				fields = append(fields, logstorage.Field{
-					Name:  bytesutil.ToUnsafeString(k),
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			}
-		})
-		if err != nil {
-			return err
-		}
-		lmp.AddRow(ts, fields, nil)
-		fields = fields[:0]
-	}
-	return nil
-}

app/vlinsert/datadog/datadog_test.go

@@ -1,104 +0,0 @@
-package datadog
-
-import (
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestReadLogsRequestFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-
-		lmp := &insertutil.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if err := lmp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`["create":{}]`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadLogsRequestSuccess(t *testing.T) {
-	f := func(data string, rowsExpected int, resultExpected string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-		var timestampsExpected []int64
-		for i := 0; i < rowsExpected; i++ {
-			timestampsExpected = append(timestampsExpected, ts)
-		}
-		lmp := &insertutil.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := lmp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	// Verify non-empty data
-	data := `[
-		{
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"bar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-                        "ddtags":"tag1:value1,tag2:value2",
-                        "hostname":"127.0.0.1",
-                        "message":{"message": "nested"},
-                        "service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"foobar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"baz",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"xyz",
-			"service":"test"
-		}, {
-			"ddsource": "nginx",
-			"ddtags":"tag1:value1,tag2:value2,",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-                }, {
-			"ddsource":"nginx",
-			"ddtags":",tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-		}
-	]`
-	rowsExpected := 7
-	resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"nested","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}`
-	f(data, rowsExpected, resultExpected)
-}

app/vlinsert/elasticsearch/bulk_response.qtpl

@@ -1,20 +0,0 @@
-{% stripspace %}
-
-{% func BulkResponse(n int, tookMs int64) %}
-{
-	"took":{%dl tookMs %},
-	"errors":false,
-	"items":[
-		{% for i := 0; i < n; i++ %}
-		{
-			"create":{
-				"status":201
-			}
-		}
-		{% if i+1 < n %},{% endif %}
-		{% endfor %}
-	]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlinsert/elasticsearch/bulk_response.qtpl.go

@@ -1,69 +0,0 @@
-// Code generated by qtc from "bulk_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-package elasticsearch
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-func StreamBulkResponse(qw422016 *qt422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-	qw422016.N().S(`{"took":`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().DL(tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().S(`,"errors":false,"items":[`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-	for i := 0; i < n; i++ {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-		qw422016.N().S(`{"create":{"status":201}}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		if i+1 < n {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	qw422016.N().S(`]}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func WriteBulkResponse(qq422016 qtio422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	StreamBulkResponse(qw422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func BulkResponse(n int, tookMs int64) string {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	WriteBulkResponse(qb422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	return qs422016
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -1,248 +0,0 @@
-package elasticsearch
-
-import (
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-)
-
-var (
-	elasticsearchVersion = flag.String("elasticsearch.version", "8.9.0", "Elasticsearch version to report to client")
-)
-
-// RequestHandler processes Elasticsearch insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	// This header is needed for Logstash
-	w.Header().Set("X-Elastic-Product", "Elasticsearch")
-
-	if strings.HasPrefix(path, "/insert/elasticsearch/_ilm/policy") {
-		// Return fake response for Elasticsearch ilm request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_index_template") {
-		// Return fake response for Elasticsearch index template request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_ingest") {
-		// Return fake response for Elasticsearch ingest pipeline request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/_nodes") {
-		// Return fake response for Elasticsearch nodes discovery request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/insert/elasticsearch/logstash") || strings.HasPrefix(path, "/insert/elasticsearch/_logstash") {
-		// Return fake response for Logstash APIs requests.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	switch path {
-	// some clients may omit trailing slash
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
-	case "/insert/elasticsearch/", "/insert/elasticsearch":
-		switch r.Method {
-		case http.MethodGet:
-			// Return fake response for Elasticsearch ping request.
-			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
-			fmt.Fprintf(w, `{
-			"version": {
-				"number": %q
-			}
-		}`, *elasticsearchVersion)
-		case http.MethodHead:
-			// Return empty response for Logstash ping request.
-		}
-
-		return true
-	case "/insert/elasticsearch/_license":
-		// Return fake response for Elasticsearch license request.
-		fmt.Fprintf(w, `{
-			"license": {
-				"uid": "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
-				"type": "oss",
-				"status": "active",
-				"expiry_date_in_millis" : 4000000000000
-			}
-		}`)
-		return true
-	case "/insert/elasticsearch/_bulk":
-		startTime := time.Now()
-		bulkRequestsTotal.Inc()
-
-		cp, err := insertutil.GetCommonParams(r)
-		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		if err := insertutil.CanWriteData(); err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk", true)
-		encoding := r.Header.Get("Content-Encoding")
-		streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-		n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeFields, cp.MsgFields, lmp)
-		lmp.MustClose()
-		if err != nil {
-			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
-			return true
-		}
-
-		tookMs := time.Since(startTime).Milliseconds()
-		bw := bufferedwriter.Get(w)
-		defer bufferedwriter.Put(bw)
-		WriteBulkResponse(bw, n, tookMs)
-		_ = bw.Flush()
-
-		// update bulkRequestDuration only for successfully parsed requests
-		// There is no need in updating bulkRequestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		bulkRequestDuration.UpdateDuration(startTime)
-
-		return true
-	default:
-		return false
-	}
-}
-
-var (
-	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
-	bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
-)
-
-func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
-	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
-
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return 0, fmt.Errorf("cannot decode Elasticsearch protocol data: %w", err)
-	}
-	defer protoparserutil.PutUncompressedReader(reader)
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lr := insertutil.NewLineReader(streamName, wcr)
-
-	n := 0
-	for {
-		ok, err := readBulkLine(lr, timeFields, msgFields, lmp)
-		wcr.DecConcurrency()
-		if err != nil || !ok {
-			return n, err
-		}
-		n++
-	}
-}
-
-func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
-	var line []byte
-
-	// Read the command, must be "create" or "index"
-	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
-		}
-		line = lr.Line
-	}
-	lineStr := bytesutil.ToUnsafeString(line)
-	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
-		return false, fmt.Errorf(`unexpected command %q; expecting "create" or "index"`, line)
-	}
-
-	// Decode log message
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return false, err
-		}
-		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
-	}
-	line = lr.Line
-	if len(line) == 0 {
-		// Special case - the line could be too long, so it was skipped.
-		// Continue parsing next lines.
-		return true, nil
-	}
-	p := logstorage.GetJSONParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-
-	ts, err := extractTimestampFromFields(timeFields, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
-	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
-	logstorage.PutJSONParser(p)
-
-	return true, nil
-}
-
-func extractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			timestamp, err := parseElasticsearchTimestamp(f.Value)
-			if err != nil {
-				return 0, err
-			}
-			f.Value = ""
-			return timestamp, nil
-		}
-	}
-	return 0, nil
-}
-
-func parseElasticsearchTimestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case - zero or empty timestamp must be substituted
-		// with the current time by the caller.
-		return 0, nil
-	}
-	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in seconds or milliseconds
-		return insertutil.ParseUnixTimestamp(s)
-	}
-	if len(s) == len("YYYY-MM-DD") {
-		t, err := time.Parse("2006-01-02", s)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse date %q: %w", s, err)
-		}
-		return t.UnixNano(), nil
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot parse timestamp %q", s)
-	}
-	return nsecs, nil
-}

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -1,128 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/golang/snappy"
-	"github.com/klauspost/compress/gzip"
-	"github.com/klauspost/compress/zlib"
-	"github.com/klauspost/compress/zstd"
-	"io"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestReadBulkRequest_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", []string{"_time"}, []string{"_msg"}, tlp)
-		if err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if rows != 0 {
-			t.Fatalf("unexpected non-zero rows=%d", rows)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`{"create":{}}`)
-	f(`{"creat":{}}
-{}`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadBulkRequest_Success(t *testing.T) {
-	f := func(data, encoding, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		timeFields := []string{"non_existing_foo", timeField, "non_existing_bar"}
-		msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		// Read the request without compression
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", timeFields, msgFields, tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-
-		// Read the request with compression
-		tlp = &insertutil.TestLogMessageProcessor{}
-		if encoding != "" {
-			data = compressData(data, encoding)
-		}
-		r = bytes.NewBufferString(data)
-		rows, err = readBulkRequest("test", r, encoding, timeFields, msgFields, tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("verification failure after compression: %s", err)
-		}
-	}
-
-	// Verify an empty data
-	f("", "gzip", "_time", "_msg", nil, "")
-	f("\n", "gzip", "_time", "_msg", nil, "")
-	f("\n\n", "gzip", "_time", "_msg", nil, "")
-
-	// Verify non-empty data
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06 04:48:12.735+01:00","message":"baz"}
-{"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"1686026893735","x":"y"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty","@timestamp":"1686026893"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty float","@timestamp":"1686026123.62"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000, 1686026123620000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}
-{"_msg":"qwe rty"}
-{"_msg":"qwe rty float"}`
-	f(data, "zstd", timeField, msgField, timestampsExpected, resultExpected)
-}
-
-func compressData(s string, encoding string) string {
-	var bb bytes.Buffer
-	var zw io.WriteCloser
-	switch encoding {
-	case "gzip":
-		zw = gzip.NewWriter(&bb)
-	case "zstd":
-		zw, _ = zstd.NewWriter(&bb)
-	case "snappy":
-		return string(snappy.Encode(nil, []byte(s)))
-	case "deflate":
-		zw = zlib.NewWriter(&bb)
-	default:
-		panic(fmt.Errorf("%q encoding is not supported", encoding))
-	}
-	if _, err := zw.Write([]byte(s)); err != nil {
-		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
-	}
-	if err := zw.Close(); err != nil {
-		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
-	}
-	return bb.String()
-}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -1,59 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-)
-
-func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("encoding:none", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "")
-	})
-	b.Run("encoding:gzip", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "gzip")
-	})
-	b.Run("encoding:zstd", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "zstd")
-	})
-	b.Run("encoding:deflate", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "deflate")
-	})
-	b.Run("encoding:snappy", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "snappy")
-	})
-}
-
-func benchmarkReadBulkRequest(b *testing.B, encoding string) {
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	if encoding != "" {
-		data = compressData(data, encoding)
-	}
-	dataBytes := bytesutil.ToUnsafeBytes(data)
-
-	timeFields := []string{"@timestamp"}
-	msgFields := []string{"message"}
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		for pb.Next() {
-			r.Reset(dataBytes)
-			_, err := readBulkRequest("test", r, encoding, timeFields, msgFields, blp)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}

app/vlinsert/insertutil/common_params.go

@@ -1,345 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	defaultMsgValue = flag.String("defaultMsgValue", "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field",
-		"Default value for _msg field if the ingested log entry doesn't contain it; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field")
-)
-
-// CommonParams contains common HTTP parameters used by log ingestion APIs.
-//
-// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
-type CommonParams struct {
-	TenantID         logstorage.TenantID
-	TimeFields       []string
-	MsgFields        []string
-	StreamFields     []string
-	IgnoreFields     []string
-	DecolorizeFields []string
-	ExtraFields      []logstorage.Field
-
-	IsTimeFieldSet  bool
-	Debug           bool
-	DebugRequestURI string
-	DebugRemoteAddr string
-}
-
-// GetCommonParams returns CommonParams from r.
-func GetCommonParams(r *http.Request) (*CommonParams, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, err
-	}
-
-	var isTimeFieldSet bool
-	timeFields := []string{"_time"}
-	if tfs := httputil.GetArray(r, "_time_field", "VL-Time-Field"); len(tfs) > 0 {
-		isTimeFieldSet = true
-		timeFields = tfs
-	}
-
-	msgFields := httputil.GetArray(r, "_msg_field", "VL-Msg-Field")
-	streamFields := httputil.GetArray(r, "_stream_fields", "VL-Stream-Fields")
-	ignoreFields := httputil.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
-	decolorizeFields := httputil.GetArray(r, "decolorize_fields", "VL-Decolorize-Fields")
-
-	extraFields, err := getExtraFields(r)
-	if err != nil {
-		return nil, err
-	}
-
-	debug := false
-	if dv := httputil.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
-		debug, err = strconv.ParseBool(dv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
-		}
-	}
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-	if debug {
-		debugRequestURI = httpserver.GetRequestURI(r)
-		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
-	}
-
-	cp := &CommonParams{
-		TenantID:         tenantID,
-		TimeFields:       timeFields,
-		MsgFields:        msgFields,
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-
-		IsTimeFieldSet:  isTimeFieldSet,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
-	}
-
-	return cp, nil
-}
-
-func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
-	efs := httputil.GetArray(r, "extra_fields", "VL-Extra-Fields")
-	if len(efs) == 0 {
-		return nil, nil
-	}
-
-	extraFields := make([]logstorage.Field, len(efs))
-	for i, ef := range efs {
-		n := strings.Index(ef, "=")
-		if n <= 0 || n == len(ef)-1 {
-			return nil, fmt.Errorf(`invalid extra_field format: %q; must be in the form "field=value"`, ef)
-		}
-		extraFields[i] = logstorage.Field{
-			Name:  ef[:n],
-			Value: ef[n+1:],
-		}
-	}
-	return extraFields, nil
-}
-
-// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) *CommonParams {
-	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
-	if streamFields == nil {
-		streamFields = []string{
-			"hostname",
-			"app_name",
-			"proc_id",
-		}
-	}
-	cp := &CommonParams{
-		TenantID: tenantID,
-		TimeFields: []string{
-			"timestamp",
-		},
-		MsgFields: []string{
-			"message",
-		},
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-	}
-
-	return cp
-}
-
-// LogRowsStorage is an interface for ingesting logs into the storage.
-type LogRowsStorage interface {
-	// MustAddRows must add lr to the underlying storage.
-	MustAddRows(lr *logstorage.LogRows)
-
-	// CanWriteData must returns non-nil error if logs cannot be added to the underlying storage.
-	CanWriteData() error
-}
-
-var logRowsStorage LogRowsStorage
-
-// SetLogRowsStorage sets the storage for writing data to via LogMessageProcessor.
-//
-// This function must be called before using LogMessageProcessor and CanWriteData from this package.
-func SetLogRowsStorage(storage LogRowsStorage) {
-	logRowsStorage = storage
-}
-
-// CanWriteData returns non-nil error if data cannot be written to the underlying storage.
-func CanWriteData() error {
-	return logRowsStorage.CanWriteData()
-}
-
-// LogMessageProcessor is an interface for log message processors.
-type LogMessageProcessor interface {
-	// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
-	//
-	// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
-	//
-	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can reuse them.
-	AddRow(timestamp int64, fields, streamFields []logstorage.Field)
-
-	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
-	MustClose()
-}
-
-type logMessageProcessor struct {
-	mu            sync.Mutex
-	wg            sync.WaitGroup
-	stopCh        chan struct{}
-	lastFlushTime time.Time
-
-	cp *CommonParams
-	lr *logstorage.LogRows
-
-	rowsIngestedTotal  *metrics.Counter
-	bytesIngestedTotal *metrics.Counter
-	flushDuration      *metrics.Summary
-}
-
-func (lmp *logMessageProcessor) initPeriodicFlush() {
-	lmp.lastFlushTime = time.Now()
-
-	lmp.wg.Add(1)
-	go func() {
-		defer lmp.wg.Done()
-
-		d := timeutil.AddJitterToDuration(time.Second)
-		ticker := time.NewTicker(d)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-lmp.stopCh:
-				return
-			case <-ticker.C:
-				lmp.mu.Lock()
-				if time.Since(lmp.lastFlushTime) >= d {
-					lmp.flushLocked()
-				}
-				lmp.mu.Unlock()
-			}
-		}
-	}()
-}
-
-// AddRow adds new log message to lmp with the given timestamp and fields.
-//
-// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
-func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// InsertRowProcessor is used by native data ingestion protocol parser.
-type InsertRowProcessor interface {
-	// AddInsertRow must add r to the underlying storage.
-	AddInsertRow(r *logstorage.InsertRow)
-}
-
-// AddInsertRow adds r to lmp.
-func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(r.Fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(r.Fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAddInsertRow(r)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// flushLocked must be called under locked lmp.mu.
-func (lmp *logMessageProcessor) flushLocked() {
-	start := time.Now()
-	lmp.lastFlushTime = start
-	logRowsStorage.MustAddRows(lmp.lr)
-	lmp.lr.ResetKeepSettings()
-	lmp.flushDuration.UpdateDuration(start)
-}
-
-// MustClose flushes the remaining data to the underlying storage and closes lmp.
-func (lmp *logMessageProcessor) MustClose() {
-	close(lmp.stopCh)
-	lmp.wg.Wait()
-
-	lmp.flushLocked()
-	logstorage.PutLogRows(lmp.lr)
-	lmp.lr = nil
-	messageProcessorCount.Add(-1)
-}
-
-// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
-//
-// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
-func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode bool) LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.DecolorizeFields, cp.ExtraFields, *defaultMsgValue)
-	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
-	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
-	flushDuration := metrics.GetOrCreateSummary(fmt.Sprintf("vl_insert_flush_duration_seconds{type=%q}", protocolName))
-	lmp := &logMessageProcessor{
-		cp: cp,
-		lr: lr,
-
-		rowsIngestedTotal:  rowsIngestedTotal,
-		bytesIngestedTotal: bytesIngestedTotal,
-		flushDuration:      flushDuration,
-
-		stopCh: make(chan struct{}),
-	}
-
-	if isStreamMode {
-		lmp.initPeriodicFlush()
-	}
-
-	messageProcessorCount.Add(1)
-	return lmp
-}
-
-var (
-	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
-	_                             = metrics.NewGauge(`vl_insert_processors_count`, func() float64 { return float64(messageProcessorCount.Load()) })
-	messageProcessorCount         atomic.Int64
-)

app/vlinsert/insertutil/flags.go

@@ -1,17 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-)
-
-var (
-	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
-
-	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
-)

app/vlinsert/insertutil/line_reader.go

@@ -1,158 +0,0 @@
-package insertutil
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-)
-
-// LineReader reads newline-delimited lines from the underlying reader
-type LineReader struct {
-	// Line contains the next line read after the call to NextLine
-	//
-	// The Line contents is valid until the next call to NextLine.
-	Line []byte
-
-	// name is the LineReader name
-	name string
-
-	// r is the underlying reader to read data from
-	r io.Reader
-
-	// buf is a buffer for reading the next line
-	buf []byte
-
-	// bufOffset is the offset at buf to read the next line from
-	bufOffset int
-
-	// err is the last error when reading data from r
-	err error
-
-	// eofReached is set to true when all the data is read from r
-	eofReached bool
-}
-
-// NewLineReader returns LineReader for r.
-func NewLineReader(name string, r io.Reader) *LineReader {
-	return &LineReader{
-		name: name,
-		r:    r,
-	}
-}
-
-// NextLine reads the next line from the underlying reader.
-//
-// It returns true if the next line is successfully read into Line.
-// If the line length exceeds MaxLineSizeBytes, then this line is skipped
-// and an empty line is returned instead.
-//
-// If false is returned, then no more lines left to read from r.
-// Check for Err in this case.
-func (lr *LineReader) NextLine() bool {
-	for {
-		lr.Line = nil
-		if lr.bufOffset >= len(lr.buf) {
-			if lr.err != nil || lr.eofReached {
-				return false
-			}
-			if !lr.readMoreData() {
-				return false
-			}
-			if lr.bufOffset >= len(lr.buf) && lr.eofReached {
-				return false
-			}
-		}
-
-		buf := lr.buf[lr.bufOffset:]
-		if n := bytes.IndexByte(buf, '\n'); n >= 0 {
-			lr.Line = buf[:n]
-			lr.bufOffset += n + 1
-			return true
-		}
-		if lr.eofReached {
-			lr.Line = buf
-			lr.bufOffset += len(buf)
-			return true
-		}
-		if !lr.readMoreData() {
-			return false
-		}
-	}
-}
-
-// Err returns the last error after NextLine call.
-func (lr *LineReader) Err() error {
-	if lr.err == nil {
-		return nil
-	}
-	return fmt.Errorf("%s: %s", lr.name, lr.err)
-}
-
-func (lr *LineReader) readMoreData() bool {
-	if lr.bufOffset > 0 {
-		lr.buf = append(lr.buf[:0], lr.buf[lr.bufOffset:]...)
-		lr.bufOffset = 0
-	}
-
-	bufLen := len(lr.buf)
-	if bufLen >= MaxLineSizeBytes.IntN() {
-		ok, skippedBytes := lr.skipUntilNextLine()
-		logger.Warnf("%s: the line length exceeds -insert.maxLineSizeBytes=%d; skipping it; total skipped bytes=%d",
-			lr.name, MaxLineSizeBytes.IntN(), skippedBytes)
-		tooLongLinesSkipped.Inc()
-		return ok
-	}
-
-	lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-	n, err := lr.r.Read(lr.buf[bufLen:])
-	lr.buf = lr.buf[:bufLen+n]
-	if err != nil {
-		if errors.Is(err, io.EOF) {
-			lr.eofReached = true
-			return true
-		}
-		lr.err = fmt.Errorf("cannot read the next line: %s", err)
-	}
-	return n > 0
-}
-
-var tooLongLinesSkipped = metrics.NewCounter("vl_too_long_lines_skipped_total")
-
-func (lr *LineReader) skipUntilNextLine() (bool, int) {
-
-	// Initialize skipped bytes count with MaxLineSizeBytes because
-	// we've already read that many bytes without encountering a newline,
-	// indicating the line size exceeds the maximum allowed limit.
-	skipSizeBytes := MaxLineSizeBytes.IntN()
-
-	for {
-		lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-		n, err := lr.r.Read(lr.buf)
-		skipSizeBytes += n
-		lr.buf = lr.buf[:n]
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				lr.eofReached = true
-				lr.buf = lr.buf[:0]
-				return true, skipSizeBytes
-			}
-			lr.err = fmt.Errorf("cannot skip the current line: %s", err)
-			return false, skipSizeBytes
-		}
-		if n := bytes.IndexByte(lr.buf, '\n'); n >= 0 {
-			// Include skipped bytes before \n, including the newline itself.
-			skipSizeBytes += n + 1 - len(lr.buf)
-			// Include \n in the buf, so too long line is replaced with an empty line.
-			// This is needed for maintaining synchorinzation consistency between lines
-			// in protocols such as Elasticsearch bulk import.
-			lr.buf = append(lr.buf[:0], lr.buf[n:]...)
-			return true, skipSizeBytes
-		}
-	}
-}

app/vlinsert/insertutil/line_reader_test.go

@@ -1,164 +0,0 @@
-package insertutil
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"reflect"
-	"testing"
-)
-
-func TestLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if len(lr.Line) > 0 {
-			t.Fatalf("unexpected non-empty line after failed NextLine(): %q", lr.Line)
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", []string{""})
-	f("\n\n", []string{"", ""})
-	f("foo", []string{"foo"})
-	f("foo\n", []string{"foo"})
-	f("\nfoo", []string{"", "foo"})
-	f("foo\n\n", []string{"foo", ""})
-	f("foo\nbar", []string{"foo", "bar"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\n\nbar\n\n", []string{"", "foo", "", "bar", ""})
-}
-
-func TestLineReader_SkipUntilNextLine(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error for data=%q: %s", data, err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines for data=%q\ngot\n%q\nwant\n%q", data, lines, linesExpected)
-		}
-	}
-
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLineLen := MaxLineSizeBytes.IntN() + overflow
-		longLine := string(make([]byte, longLineLen))
-
-		// Single long line
-		data := longLine
-		f(data, nil)
-
-		// Multiple long lines
-		data = longLine + "\n" + longLine
-		f(data, []string{""})
-
-		data = longLine + "\n" + longLine + "\n"
-		f(data, []string{"", ""})
-
-		// Long line in the middle
-		data = "foo\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "bar"})
-
-		// Multiple long lines in the middle
-		data = "foo\n" + longLine + "\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "", "bar"})
-
-		// Long line in the end
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		// Long line in the end
-		data = "foo\n" + longLine + "\n"
-		f(data, []string{"foo", ""})
-	}
-}
-
-func TestLineReader_Failure(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		fr := &failureReader{
-			r: bytes.NewBufferString(data),
-		}
-		lr := NewLineReader("foo", fr)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error on the second call")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("foo", nil)
-	f("foo\n", []string{"foo"})
-	f("\n", []string{""})
-	f("foo\nbar", []string{"foo"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\nbar\n\n", []string{"", "foo", "bar", ""})
-
-	// long line
-	longLineLen := MaxLineSizeBytes.IntN()
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLine := string(make([]byte, longLineLen+overflow))
-
-		data := longLine
-		f(data, nil)
-
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		data = longLine + "\nfoo"
-		f(data, []string{""})
-
-		data = longLine + "\nfoo\n"
-		f(data, []string{"", "foo"})
-	}
-}
-
-type failureReader struct {
-	r io.Reader
-}
-
-func (r *failureReader) Read(p []byte) (int, error) {
-	n, _ := r.r.Read(p)
-	if n > 0 {
-		return n, nil
-	}
-	return 0, fmt.Errorf("some error")
-}

app/vlinsert/insertutil/testutils.go

@@ -1,56 +0,0 @@
-package insertutil
-
-import (
-	"fmt"
-	"reflect"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// TestLogMessageProcessor implements LogMessageProcessor for testing.
-type TestLogMessageProcessor struct {
-	timestamps []int64
-	rows       []string
-}
-
-// AddRow adds row with the given timestamp and fields to tlp
-func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("BUG: streamFields must be nil; got %v", streamFields))
-	}
-	tlp.timestamps = append(tlp.timestamps, timestamp)
-	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
-}
-
-// MustClose closes tlp.
-func (tlp *TestLogMessageProcessor) MustClose() {
-}
-
-// Verify verifies the number of rows, timestamps and results after AddRow calls.
-func (tlp *TestLogMessageProcessor) Verify(timestampsExpected []int64, resultExpected string) error {
-	result := strings.Join(tlp.rows, "\n")
-	if len(tlp.rows) != len(timestampsExpected) {
-		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), len(timestampsExpected), result, resultExpected)
-	}
-
-	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
-		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
-	}
-	if result != resultExpected {
-		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-	}
-
-	return nil
-}
-
-// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
-type BenchmarkLogMessageProcessor struct{}
-
-// AddRow implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _, _ []logstorage.Field) {
-}
-
-// MustClose implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) MustClose() {
-}

app/vlinsert/insertutil/timestamp.go

@@ -1,106 +0,0 @@
-package insertutil
-
-import (
-	"fmt"
-	"math"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ExtractTimestampFromFields extracts timestamp in nanoseconds from the first field the name from timeFields at fields.
-//
-// The value for the corresponding timeFields is set to empty string after returning from the function,
-// so it could be ignored during data ingestion.
-//
-// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			nsecs, err := parseTimestamp(f.Value)
-			if err != nil {
-				return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", f.Name, err)
-			}
-			f.Value = ""
-			if nsecs == 0 {
-				nsecs = time.Now().UnixNano()
-			}
-			return nsecs, nil
-		}
-	}
-	return time.Now().UnixNano(), nil
-}
-
-func parseTimestamp(s string) (int64, error) {
-	// "-" is a nil timestamp value, if the syslog
-	// application is incapable of obtaining system time
-	// https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
-	if s == "" || s == "0" || s == "-" {
-		return time.Now().UnixNano(), nil
-	}
-	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
-		return ParseUnixTimestamp(s)
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
-	}
-	return nsecs, nil
-}
-
-// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
-func ParseUnixTimestamp(s string) (int64, error) {
-	if strings.IndexByte(s, '.') >= 0 {
-		// Parse timestamp as floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-		}
-		if f < (1<<31) && f >= (-1<<31) {
-			// The timestamp is in seconds.
-			return int64(f * 1e9), nil
-		}
-		if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
-			// The timestamp is in milliseconds.
-			return int64(f * 1e6), nil
-		}
-		if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
-			// The timestamp is in microseconds.
-			return int64(f * 1e3), nil
-		}
-		// The timestamp is in nanoseconds
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		return int64(f), nil
-	}
-
-	// Parse timestamp as integer
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-	}
-	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds.
-		return n * 1e9, nil
-	}
-	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
-		// The timestamp is in milliseconds.
-		return n * 1e6, nil
-	}
-	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
-		// The timestamp is in microseconds.
-		return n * 1e3, nil
-	}
-	// The timestamp is in nanoseconds
-	return n, nil
-}

app/vlinsert/insertutil/timestamp_test.go

@@ -1,185 +0,0 @@
-package insertutil
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestParseUnixTimestamp_Success(t *testing.T) {
-	f := func(s string, timestampExpected int64) {
-		t.Helper()
-
-		timestamp, err := ParseUnixTimestamp(s)
-		if err != nil {
-			t.Fatalf("unexpected error in ParseUnixTimestamp(%q): %s", s, err)
-		}
-		if timestamp != timestampExpected {
-			t.Fatalf("unexpected timestamp returned from ParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
-		}
-	}
-
-	f("0", 0)
-
-	// nanoseconds
-	f("-1234567890123456789", -1234567890123456789)
-	f("1234567890123456789", 1234567890123456789)
-
-	// microseconds
-	f("-1234567890123456", -1234567890123456000)
-	f("1234567890123456", 1234567890123456000)
-	f("1234567890123456.789", 1234567890123456768)
-
-	// milliseconds
-	f("-1234567890123", -1234567890123000000)
-	f("1234567890123", 1234567890123000000)
-	f("1234567890123.456", 1234567890123456000)
-
-	// seconds
-	f("-1234567890", -1234567890000000000)
-	f("1234567890", 1234567890000000000)
-	f("-1234567890.123456", -1234567890123456000)
-}
-
-func TestParseUnixTimestamp_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := ParseUnixTimestamp(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error in ParseUnixTimestamp(%q)", s)
-		}
-	}
-
-	// non-numeric timestamp
-	f("")
-	f("foobar")
-	f("foo.bar")
-
-	// too big timestamp
-	f("12345678901234567890")
-	f("-12345678901234567890")
-	f("12345678901234567890.235424")
-	f("-12345678901234567890.235424")
-}
-
-func TestExtractTimestampFromFields_Success(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs != nsecsExpected {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
-		}
-
-		for _, f := range fields {
-			if f.Name == timeField {
-				if f.Value != "" {
-					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
-				}
-			}
-		}
-	}
-
-	// UTC time
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20Z"},
-	}, 1718753840000000000)
-
-	// Time with timezone
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
-	}, 1718725040000000000)
-
-	// SQL datetime format
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
-	}, 1718773640123000000)
-
-	// Time with nanosecond precision
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
-		{Name: "foo", Value: "bar"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in nanoseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456789"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in microseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456"},
-	}, 1718773640123456000)
-
-	// Unix timestamp in milliseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123"},
-	}, 1718773640123000000)
-
-	// Unix timestamp in seconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640"},
-	}, 1718773640000000000)
-}
-
-func TestExtractTimestampFromFields_Now(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs < 1 {
-			t.Fatalf("expected generated timestamp, got error: %s", err)
-		}
-	}
-
-	// RFC5424 allows `-` for nil timestamp (log ingestion time)
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "-"},
-	})
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: ""},
-	})
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "0"},
-	})
-}
-
-func TestExtractTimestampFromFields_Error(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		fields := []logstorage.Field{
-			{Name: "time", Value: s},
-		}
-		nsecs, err := ExtractTimestampFromFields([]string{"time"}, fields)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if nsecs != 0 {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
-		}
-	}
-
-	// invalid time
-	f("foobar")
-
-	// incomplete time
-	f("2024-06-18")
-	f("2024-06-18T23:37")
-}

app/vlinsert/internalinsert/internalinsert.go

@@ -1,88 +0,0 @@
-package internalinsert
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
-)
-
-// RequestHandler processes /internal/insert requests.
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-	version := r.FormValue("version")
-	if version != netinsert.ProtocolVersion {
-		httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("internalinsert", false)
-		irp := lmp.(insertutil.InsertRowProcessor)
-		err := parseData(irp, data)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
-		return
-	}
-
-	requestDuration.UpdateDuration(startTime)
-}
-
-func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
-	r := logstorage.GetInsertRow()
-	src := data
-	i := 0
-	for len(src) > 0 {
-		tail, err := r.UnmarshalInplace(src)
-		if err != nil {
-			return fmt.Errorf("cannot parse row #%d: %s", i, err)
-		}
-		src = tail
-		i++
-
-		irp.AddInsertRow(r)
-	}
-	logstorage.PutInsertRow(r)
-
-	return nil
-}
-
-var (
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
-
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/internal/insert"}`)
-)

app/vlinsert/journald/journald.go

@@ -1,377 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"slices"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
-const maxFieldNameLen = 64
-
-var (
-	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
-	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Comma-separated list of fields to ignore for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#dropping-fields")
-	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Field to use as a log timestamp for logs ingested via journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
-	journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
-	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include Journald fields with double underscore prefixes")
-)
-
-func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		tenantID, err := logstorage.ParseTenantID(*journaldTenantID)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse -journald.tenantID=%q for journald: %w", *journaldTenantID, err)
-		}
-		cp.TenantID = tenantID
-	}
-
-	if !cp.IsTimeFieldSet {
-		cp.TimeFields = []string{*journaldTimeField}
-	}
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = getStreamFields()
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *journaldIgnoreFields
-	}
-	cp.MsgFields = []string{"MESSAGE"}
-	return cp, nil
-}
-
-func getStreamFields() []string {
-	if len(*journaldStreamFields) > 0 {
-		return *journaldStreamFields
-	}
-	return defaultStreamFields
-}
-
-var defaultStreamFields = []string{
-	"_MACHINE_ID",
-	"_HOSTNAME",
-	"_SYSTEMD_UNIT",
-}
-
-// RequestHandler processes Journald Export insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/journald/upload":
-		if r.Header.Get("Content-Type") != "application/vnd.fdo.journal" {
-			httpserver.Errorf(w, r, "only application/vnd.fdo.journal encoding is supported for Journald")
-			return true
-		}
-		handleJournald(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-// handleJournald parses Journal binary entries
-func handleJournald(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsTotal.Inc()
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-
-	if err := insertutil.CanWriteData(); err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
-	if err != nil {
-		errorsTotal.Inc()
-		logger.Errorf("cannot decode journald request: %s", err)
-		return
-	}
-
-	lmp := cp.NewLogMessageProcessor("journald", true)
-	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, lmp, cp)
-	protoparserutil.PutUncompressedReader(reader)
-	lmp.MustClose()
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
-		return
-	}
-
-	// systemd starting release v258 will support compression, which starts working after negotiation: it expects supported compression
-	// algorithms list in Accept-Encoding response header in a format "<algorithm_1>[:<priority_1>][;<algorithm_2>:<priority_2>]"
-	// See https://github.com/systemd/systemd/pull/34822
-	w.Header().Set("Accept-Encoding", "zstd")
-
-	// update requestDuration only for successfully parsed requests
-	// There is no need in updating requestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
-	errorsTotal     = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
-)
-
-func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lr := insertutil.NewLineReader("journald", wcr)
-
-	for {
-		err := readJournaldLogEntry(streamName, lr, lmp, cp)
-		wcr.DecConcurrency()
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				return nil
-			}
-			return fmt.Errorf("%s: %w", streamName, err)
-		}
-	}
-}
-
-type fieldsBuf struct {
-	fields []logstorage.Field
-
-	buf   []byte
-	name  []byte
-	value []byte
-}
-
-func (fb *fieldsBuf) reset() {
-	fb.fields = fb.fields[:0]
-	fb.buf = fb.buf[:0]
-	fb.name = fb.name[:0]
-	fb.value = fb.value[:0]
-}
-
-func (fb *fieldsBuf) addField(name, value string) {
-	bufLen := len(fb.buf)
-	fb.buf = append(fb.buf, name...)
-	nameCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
-
-	bufLen = len(fb.buf)
-	fb.buf = append(fb.buf, value...)
-	valueCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
-
-	fb.fields = append(fb.fields, logstorage.Field{
-		Name:  nameCopy,
-		Value: valueCopy,
-	})
-}
-
-func (fb *fieldsBuf) appendNextLineToValue(lr *insertutil.LineReader) error {
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return err
-		}
-		return fmt.Errorf("unexpected end of stream")
-	}
-	fb.value = append(fb.value, lr.Line...)
-	fb.value = append(fb.value, '\n')
-	return nil
-}
-
-func getFieldsBuf() *fieldsBuf {
-	fb := fieldsBufPool.Get()
-	if fb == nil {
-		return &fieldsBuf{}
-	}
-	return fb.(*fieldsBuf)
-}
-
-func putFieldsBuf(fb *fieldsBuf) {
-	fb.reset()
-	fieldsBufPool.Put(fb)
-}
-
-var fieldsBufPool sync.Pool
-
-// readJournaldLogEntry reads a single log entry in Journald format.
-//
-// See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
-	var ts int64
-	var name, value string
-
-	fb := getFieldsBuf()
-	defer putFieldsBuf(fb)
-
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
-			return fmt.Errorf("cannot read the first field: %w", err)
-		}
-		return io.EOF
-	}
-
-	for {
-		line := lr.Line
-		if len(line) == 0 {
-			// The end of a single log entry. Write it to the storage
-			if len(fb.fields) > 0 {
-				if ts == 0 {
-					ts = time.Now().UnixNano()
-				}
-				lmp.AddRow(ts, fb.fields, nil)
-			}
-			return nil
-		}
-
-		// line could be either "key=value" or "key"
-		// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-		if n := bytes.IndexByte(line, '='); n >= 0 {
-			// line = "key=value"
-			fb.name = append(fb.name[:0], line[:n]...)
-			name = bytesutil.ToUnsafeString(fb.name)
-
-			fb.value = append(fb.value[:0], line[n+1:]...)
-			value = bytesutil.ToUnsafeString(fb.value)
-		} else {
-			// line = "key"
-			// Parse the binary-encoded value from the next line according to "key\n<little_endian_size_64>value\n" format
-			fb.name = append(fb.name[:0], line...)
-			name = bytesutil.ToUnsafeString(fb.name)
-
-			fb.value = fb.value[:0]
-			for len(fb.value) < 8 {
-				if err := fb.appendNextLineToValue(lr); err != nil {
-					return fmt.Errorf("cannot read value size: %w", err)
-				}
-			}
-			size := binary.LittleEndian.Uint64(fb.value[:8])
-
-			// Read the value until its length exceeds the given size - the last char in the read value will always be '\n'
-			// because it is appended by appendNextLineToValue().
-			for uint64(len(fb.value[8:])) <= size {
-				if err := fb.appendNextLineToValue(lr); err != nil {
-					return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
-				}
-			}
-			value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
-			if uint64(len(value)) != size {
-				return fmt.Errorf("unexpected %q value size; got %d bytes; want %d bytes; value: %q", fb.name, len(value), size, value)
-			}
-		}
-
-		if !lr.NextLine() {
-			if err := lr.Err(); err != nil {
-				return fmt.Errorf("cannot read the next log field: %w", err)
-			}
-
-			// add the last log field below before the return
-		}
-
-		if len(name) > maxFieldNameLen {
-			logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
-			continue
-		}
-		if !isValidFieldName(name) {
-			logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
-			continue
-		}
-
-		if slices.Contains(cp.TimeFields, name) {
-			t, err := strconv.ParseInt(value, 10, 64)
-			if err != nil {
-				logger.Errorf("%s: cannot parse timestamp from the field %q: %w; using the current timestamp", streamName, name, err)
-				ts = 0
-			} else {
-				// Convert journald microsecond timestamp to nanoseconds
-				ts = t * 1e3
-			}
-			continue
-		}
-
-		if slices.Contains(cp.MsgFields, name) {
-			name = "_msg"
-		}
-
-		if name == "PRIORITY" {
-			priority := journaldPriorityToLevel(value)
-			fb.addField("level", priority)
-		}
-
-		if !strings.HasPrefix(name, "__") || *journaldIncludeEntryMetadata {
-			fb.addField(name, value)
-		}
-	}
-}
-
-func journaldPriorityToLevel(priority string) string {
-	// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
-	// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
-	switch priority {
-	case "0":
-		return "emerg"
-	case "1":
-		return "alert"
-	case "2":
-		return "critical"
-	case "3":
-		return "error"
-	case "4":
-		return "warning"
-	case "5":
-		return "notice"
-	case "6":
-		return "info"
-	case "7":
-		return "debug"
-	default:
-		return priority
-	}
-}
-
-func isValidFieldName(s string) bool {
-	if len(s) == 0 {
-		return false
-	}
-	c := s[0]
-	if !(c >= 'A' && c <= 'Z' || c == '_') {
-		return false
-	}
-
-	for i := 1; i < len(s); i++ {
-		c := s[i]
-		if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
-			return false
-		}
-	}
-	return true
-}

app/vlinsert/journald/journald_test.go

@@ -1,166 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"net/http"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestIsValidFieldName(t *testing.T) {
-	f := func(name string, resultExpected bool) {
-		t.Helper()
-
-		result := isValidFieldName(name)
-		if result != resultExpected {
-			t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
-		}
-	}
-
-	f("", false)
-	f("a", false)
-	f("1", false)
-	f("_", true)
-	f("X", true)
-	f("Xa", false)
-	f("X_343", true)
-	f("X_0123456789_AZ", true)
-	f("SDDFD sdf", false)
-}
-
-func TestGetCommonParams_TimeField(t *testing.T) {
-	f := func(timeFieldHeader, expectedTimeField string) {
-		t.Helper()
-
-		req, err := http.NewRequest("POST", "/insert/journald/upload", nil)
-		if err != nil {
-			t.Fatalf("unexpected error creating request: %s", err)
-		}
-
-		if timeFieldHeader != "" {
-			req.Header.Set("VL-Time-Field", timeFieldHeader)
-		}
-
-		cp, err := getCommonParams(req)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if len(cp.TimeFields) != 1 || cp.TimeFields[0] != expectedTimeField {
-			t.Fatalf("unexpected TimeFields; got %v; want [%s]", cp.TimeFields, expectedTimeField)
-		}
-	}
-
-	// Test default behavior - when no custom time field is specified, journald uses __REALTIME_TIMESTAMP
-	f("", "__REALTIME_TIMESTAMP")
-
-	// Test custom time field - when a custom time field is specified via HTTP header, it's respected
-	f("custom_time", "custom_time")
-}
-
-func TestPushJournald_Success(t *testing.T) {
-	f := func(src string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		cp, err := getCommonParams(r)
-		if err != nil {
-			t.Fatalf("cannot create commonParams: %s", err)
-		}
-
-		buf := bytes.NewBufferString(src)
-		if err := processStreamInternal("test", buf, tlp, cp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Single event
-	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n",
-		[]int64{91723819283000},
-		"{\"_msg\":\"Test message\"}",
-	)
-
-	// Multiple events
-	f("__REALTIME_TIMESTAMP=91723819283\nPRIORITY=3\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
-		[]int64{91723819283000, 91723819284000},
-		"{\"level\":\"error\",\"PRIORITY\":\"3\",\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
-	)
-
-	// Parse binary data
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\nE=JobStateChanged\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
-		[]int64{1729698775704404000},
-		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
-	)
-
-	// Parse binary data with trailing newline
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x14\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"foo\nbar\n\n\nasda\nasda\n","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"123456789\n","_PID":"2763"}`,
-	)
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890\n_PID=2763\n\n",
-		[]int64{1729698775704404000},
-		`{"_CMDLINE":"python3","_msg":"1234567890","_PID":"2763"}`,
-	)
-
-	// Empty field name must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
-	f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
-
-	// field name starting with number must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
-
-	// field name exceeding 64 bytes limit must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n", nil, "")
-
-	// field name with invalid chars must be ignored
-	f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n", nil, "")
-}
-
-func TestPushJournald_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		cp, err := getCommonParams(r)
-		if err != nil {
-			t.Fatalf("cannot create commonParams: %s", err)
-		}
-
-		buf := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", buf, tlp, cp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// too short binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasd")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00_PID=2763\n\n")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890_PID=2763\n\n")
-	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n_PID=2763\n\n")
-
-	// too long binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")
-}

app/vlinsert/journald/journald_timing_test.go

@@ -1,82 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"encoding/binary"
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func BenchmarkIsValidFieldName(b *testing.B) {
-	b.ReportAllocs()
-	b.SetBytes(int64(len(benchmarkFields)))
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			for _, field := range benchmarkFields {
-				if !isValidFieldName(field) {
-					panic(fmt.Errorf("cannot validate field %q", field))
-				}
-			}
-		}
-	})
-}
-
-var benchmarkFields = strings.Split(
-	"E,_BOOT_ID,_UID,_GID,_MACHINE_ID,_HOSTNAME,_RUNTIME_SCOPE,_TRANSPORT,_CAP_EFFECTIVE,_SYSTEMD_CGROUP,_SYSTEMD_UNIT,"+
-		"_SYSTEMD_SLICE,CODE_FILE,CODE_LINE,CODE_FUNC,SYSLOG_IDENTIFIER,_COMM,_EXE,_CMDLINE,MESSAGE,_PID,_SOURCE_REALTIME_TIMESTAMP,_REALTIME_TIMESTAMP",
-	",")
-
-func BenchmarkPushJournaldPerformance(b *testing.B) {
-	cp := &insertutil.CommonParams{
-		TimeFields: []string{"__REALTIME_TIMESTAMP"},
-		MsgFields:  []string{"MESSAGE"},
-	}
-	const dataChunkSize = 1024 * 1024
-
-	data := generateJournaldData(dataChunkSize)
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		blp := &insertutil.BenchmarkLogMessageProcessor{}
-		for pb.Next() {
-			r.Reset(data)
-			if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func generateJournaldData(size int) []byte {
-	var buf []byte
-	timestamp := time.Now().UnixMicro()
-	binaryMsg := []byte("binary message data for performance test")
-	var sizeBuf [8]byte
-
-	for len(buf) < size {
-		timestamp++
-
-		var entry string
-		// Generate a mix of simple and binary messages
-		if timestamp%10 == 0 {
-			// Generate binary message
-			binary.LittleEndian.PutUint64(sizeBuf[:], uint64(len(binaryMsg)))
-			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE\n%s%s\n\n",
-				timestamp,
-				sizeBuf[:],
-				binaryMsg,
-			)
-		} else {
-			// Generate simple message
-			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE=Performance test message %d\n\n", timestamp, timestamp)
-		}
-		buf = append(buf, entry...)
-	}
-	return buf
-}

app/vlinsert/jsonline/jsonline.go

@@ -1,123 +0,0 @@
-package jsonline
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// RequestHandler processes jsonline insert requests
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-	w.Header().Add("Content-Type", "application/json")
-
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
-	if err != nil {
-		logger.Errorf("cannot decode jsonline request: %s", err)
-		return
-	}
-	defer protoparserutil.PutUncompressedReader(reader)
-
-	lmp := cp.NewLogMessageProcessor("jsonline", true)
-	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, cp.TimeFields, cp.MsgFields, lmp)
-	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot process jsonline request; error: %s", err)
-		return
-	}
-
-	requestDuration.UpdateDuration(startTime)
-}
-
-func processStreamInternal(streamName string, r io.Reader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lr := insertutil.NewLineReader(streamName, wcr)
-
-	n := 0
-	errors := 0
-	var lastError error
-	for {
-		ok, err := readLine(lr, timeFields, msgFields, lmp)
-		wcr.DecConcurrency()
-		if err != nil {
-			lastError = err
-			errors++
-			logger.Warnf("jsonline: cannot read line #%d in /jsonline request: %s", n, err)
-		}
-		if !ok {
-			break
-		}
-		n++
-	}
-	errorsTotal.Add(errors)
-
-	if errors > 0 && n == errors {
-		// Return an error if no logs were processed and there were errors
-		return lastError
-	}
-
-	return nil
-}
-
-func readLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
-	var line []byte
-	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
-		}
-		line = lr.Line
-	}
-
-	p := logstorage.GetJSONParser()
-	defer logstorage.PutJSONParser(p)
-
-	if err := p.ParseLogMessage(line); err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
-	}
-	ts, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
-	if err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
-	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
-
-	return true, nil
-}
-
-var (
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
-
-	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
-)

app/vlinsert/jsonline/jsonline_test.go

@@ -1,97 +0,0 @@
-package jsonline
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestProcessStreamInternalSuccess(t *testing.T) {
-	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		timeFields := []string{timeField}
-		msgFields := []string{msgField}
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, timeFields, msgFields, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06 04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// Non-existing msgField
-	data = `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-`
-	timeField = "@timestamp"
-	msgField = "foobar"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
-{"message":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// invalid lines among valid lines
-	data = `
-dsfodmasd
-
-{"time":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-invalid line
-{"time":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-asbsdf
-
-`
-	timeField = "time"
-	msgField = "message"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternalFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := strings.NewReader(data)
-		if err := processStreamInternal("test", r, []string{"time"}, nil, tlp); err == nil {
-			t.Fatalf("expected error, got nil")
-		}
-
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	// invalid json
-	f("foobar")
-
-	f(`foo
-bar`)
-
-	f(`
-foo
-
-`)
-
-	// invalid timestamp field
-	f(`{"time":"foobar"}`)
-}

app/vlinsert/loki/loki.go

@@ -1,86 +0,0 @@
-package loki
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whether to disable automatic parsing of JSON-encoded log fields inside Loki log message into distinct log fields")
-
-// RequestHandler processes Loki insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/insert/loki/api/v1/push":
-		handleInsert(r, w)
-		return true
-	case "/insert/loki/ready":
-		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("ready"))
-		return true
-	default:
-		return false
-	}
-}
-
-// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-func handleInsert(r *http.Request, w http.ResponseWriter) {
-	contentType := r.Header.Get("Content-Type")
-	switch contentType {
-	case "application/json":
-		handleJSON(r, w)
-	default:
-		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		handleProtobuf(r, w)
-	}
-}
-
-type commonParams struct {
-	cp *insertutil.CommonParams
-
-	// Whether to parse JSON inside plaintext log message.
-	//
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8486
-	parseMessage bool
-}
-
-func getCommonParams(r *http.Request) (*commonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// If parsed tenant is (0,0) it is likely to be default tenant
-	// Try parsing tenant from Loki headers
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		org := r.Header.Get("X-Scope-OrgID")
-		if org != "" {
-			tenantID, err := logstorage.ParseTenantID(org)
-			if err != nil {
-				return nil, err
-			}
-			cp.TenantID = tenantID
-		}
-
-	}
-
-	parseMessage := !*disableMessageParsing
-	if rv := httputil.GetRequestValue(r, "disable_message_parsing", "VL-Loki-Disable-Message-Parsing"); rv != "" {
-		bv, err := strconv.ParseBool(rv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse dusable_message_parsing=%q: %s", rv, err)
-		}
-		parseMessage = !bv
-	}
-
-	return &commonParams{
-		cp:           cp,
-		parseMessage: parseMessage,
-	}, nil
-}

app/vlinsert/loki/loki_json.go

@@ -1,225 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var maxRequestSize = flagutil.NewBytes("loki.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single Loki request")
-
-var parserPool fastjson.ParserPool
-
-func handleJSON(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsJSONTotal.Inc()
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_json", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseJSONRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki json data: %s", err)
-		return
-	}
-
-	// update requestJSONDuration only for successfully parsed requests
-	// There is no need in updating requestJSONDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestJSONDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
-}
-
-var (
-	requestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
-)
-
-func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-
-	streamsV := v.Get("streams")
-	if streamsV == nil {
-		return fmt.Errorf("missing `streams` item in the parsed JSON")
-	}
-	streams, err := streamsV.Array()
-	if err != nil {
-		return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
-	}
-
-	fields := getFields()
-	defer putFields(fields)
-
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
-	}
-
-	currentTimestamp := time.Now().UnixNano()
-
-	for _, stream := range streams {
-		// populate common labels from `stream` dict
-		fields.fields = fields.fields[:0]
-		labelsV := stream.Get("stream")
-		var labels *fastjson.Object
-		if labelsV != nil {
-			o, err := labelsV.Object()
-			if err != nil {
-				return fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
-			}
-			labels = o
-		}
-		labels.Visit(func(k []byte, v *fastjson.Value) {
-			vStr, errLocal := v.StringBytes()
-			if errLocal != nil {
-				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-				return
-			}
-			fields.fields = append(fields.fields, logstorage.Field{
-				Name:  bytesutil.ToUnsafeString(k),
-				Value: bytesutil.ToUnsafeString(vStr),
-			})
-		})
-		if err != nil {
-			return fmt.Errorf("error when parsing `stream` object: %w", err)
-		}
-
-		// populate messages from `values` array
-		linesV := stream.Get("values")
-		if linesV == nil {
-			return fmt.Errorf("missing `values` item in the parsed `stream` object %q", stream)
-		}
-		lines, err := linesV.Array()
-		if err != nil {
-			return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
-		}
-
-		commonFieldsLen := len(fields.fields)
-		for _, line := range lines {
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			lineA, err := line.Array()
-			if err != nil {
-				return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
-			}
-			if len(lineA) < 2 || len(lineA) > 3 {
-				return fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2 or 3", line, len(lineA))
-			}
-
-			// parse timestamp
-			timestamp, err := lineA[0].StringBytes()
-			if err != nil {
-				return fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
-			}
-			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
-			if err != nil {
-				return fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
-			}
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-			if len(lineA) > 2 {
-				structuredMetadata, err := lineA[2].Object()
-				if err != nil {
-					return fmt.Errorf("unexpected structured metadata type for %q; want JSON object", lineA[2])
-				}
-
-				structuredMetadata.Visit(func(k []byte, v *fastjson.Value) {
-					vStr, errLocal := v.StringBytes()
-					if errLocal != nil {
-						err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-						return
-					}
-
-					fields.fields = append(fields.fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(vStr),
-					})
-				})
-				if err != nil {
-					return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
-				}
-			}
-
-			// parse log message
-			msg, err := lineA[1].StringBytes()
-			if err != nil {
-				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
-			}
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, bytesutil.ToUnsafeString(msg))
-
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
-			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
-		}
-	}
-
-	return nil
-}
-
-func addMsgField(dst []logstorage.Field, msgParser *logstorage.JSONParser, msg string) ([]logstorage.Field, bool) {
-	if msgParser == nil || len(msg) < 2 || msg[0] != '{' || msg[len(msg)-1] != '}' {
-		return append(dst, logstorage.Field{
-			Name:  "_msg",
-			Value: msg,
-		}), false
-	}
-	if msgParser != nil && len(msg) >= 2 && msg[0] == '{' && msg[len(msg)-1] == '}' {
-		if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msg)); err == nil {
-			return append(dst, msgParser.Fields...), true
-		}
-	}
-	return append(dst, logstorage.Field{
-		Name:  "_msg",
-		Value: msg,
-	}), false
-}
-
-func parseLokiTimestamp(s string) (int64, error) {
-	if s == "" {
-		// Special case - an empty timestamp must be substituted with the current time by the caller.
-		return 0, nil
-	}
-	return insertutil.ParseUnixTimestamp(s)
-}

app/vlinsert/loki/loki_json_test.go

@@ -1,169 +0,0 @@
-package loki
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestParseJSONRequest_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-	f(``)
-
-	// Invalid json
-	f(`{}`)
-	f(`[]`)
-	f(`"foo"`)
-	f(`123`)
-
-	// invalid type for `streams` item
-	f(`{"streams":123}`)
-
-	// Missing `values` item
-	f(`{"streams":[{}]}`)
-
-	// Invalid type for `values` item
-	f(`{"streams":[{"values":"foobar"}]}`)
-
-	// Invalid type for `stream` item
-	f(`{"streams":[{"stream":[],"values":[]}]}`)
-
-	// Invalid type for `values` individual item
-	f(`{"streams":[{"values":[123]}]}`)
-
-	// Invalid length of `values` individual item
-	f(`{"streams":[{"values":[[]]}]}`)
-	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789","8123"]]}]}`)
-
-	// Invalid type for timestamp inside `values` individual item
-	f(`{"streams":[{"values":[[123,"456"]}]}`)
-
-	// Invalid type for log message
-	f(`{"streams":[{"values":[["123",1234]]}]}`)
-
-	// invalid structured metadata type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", ["metadata_1", "md_value"]]]}]}`)
-
-	// structured metadata with unexpected value type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": 1}]] }]}`)
-}
-
-func TestParseJSONRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1686026123.62", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1686026123620000000, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
-
-	// values with metadata
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"metadata_1":"md_value","_msg":"foo bar"}`)
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-}
-
-func TestParseJSONRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		if err := parseJSONRequest([]byte(s), tlp, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"111\",\"parent_id\":\"abc\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"111","parent_id":"abc"}`)
-}

app/vlinsert/loki/loki_json_timing_test.go

@@ -1,78 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func BenchmarkParseJSONRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseJSONRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		data := getJSONBody(streams, rows, labels)
-		for pb.Next() {
-			if err := parseJSONRequest(data, blp, nil, false, true); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getJSONBody(streams, rows, labels int) []byte {
-	body := append([]byte{}, `{"streams":[`...)
-	now := time.Now().UnixNano()
-	valuePrefix := fmt.Sprintf(`["%d","value_`, now)
-
-	for i := 0; i < streams; i++ {
-		body = append(body, `{"stream":{`...)
-
-		for j := 0; j < labels; j++ {
-			body = append(body, `"label_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `":"value_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, '"')
-			if j < labels-1 {
-				body = append(body, ',')
-			}
-
-		}
-		body = append(body, `}, "values":[`...)
-
-		for j := 0; j < rows; j++ {
-			body = append(body, valuePrefix...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `"]`...)
-			if j < rows-1 {
-				body = append(body, ',')
-			}
-		}
-
-		body = append(body, `]}`...)
-		if i < streams-1 {
-			body = append(body, ',')
-		}
-
-	}
-
-	body = append(body, `]}`...)
-
-	return body
-}

app/vlinsert/loki/loki_protobuf.go

@@ -1,218 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	pushReqsPool sync.Pool
-)
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	if encoding == "" {
-		// Loki protocol uses snappy compression by default.
-		// See https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-		encoding = "snappy"
-	}
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_protobuf", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseProtobufRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki protobuf data: %s", err)
-		return
-	}
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
-}
-
-var (
-	requestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
-)
-
-func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
-	req := getPushRequest()
-	defer putPushRequest(req)
-
-	err := req.UnmarshalProtobuf(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse request body: %w", err)
-	}
-
-	fields := getFields()
-	defer putFields(fields)
-
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
-	}
-
-	streams := req.Streams
-	currentTimestamp := time.Now().UnixNano()
-
-	for i := range streams {
-		stream := &streams[i]
-		// st.Labels contains labels for the stream.
-		// Labels are same for all entries in the stream.
-		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
-		if err != nil {
-			return fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
-		}
-		commonFieldsLen := len(fields.fields)
-
-		entries := stream.Entries
-		for j := range entries {
-			e := &entries[j]
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			for _, lp := range e.StructuredMetadata {
-				fields.fields = append(fields.fields, logstorage.Field{
-					Name:  lp.Name,
-					Value: lp.Value,
-				})
-			}
-
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, e.Line)
-
-			ts := e.Timestamp.UnixNano()
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
-			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
-		}
-	}
-	return nil
-}
-
-func getFields() *fields {
-	v := fieldsPool.Get()
-	if v == nil {
-		return &fields{}
-	}
-	return v.(*fields)
-}
-
-func putFields(f *fields) {
-	f.fields = f.fields[:0]
-	fieldsPool.Put(f)
-}
-
-var fieldsPool sync.Pool
-
-type fields struct {
-	fields []logstorage.Field
-}
-
-// parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
-//
-// See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
-func parsePromLabels(dst []logstorage.Field, s string) ([]logstorage.Field, error) {
-	// Make sure s is wrapped into `{...}`
-	s = strings.TrimSpace(s)
-	if len(s) < 2 {
-		return nil, fmt.Errorf("too short string to parse: %q", s)
-	}
-	if s[0] != '{' {
-		return nil, fmt.Errorf("missing `{` at the beginning of %q", s)
-	}
-	if s[len(s)-1] != '}' {
-		return nil, fmt.Errorf("missing `}` at the end of %q", s)
-	}
-	s = s[1 : len(s)-1]
-
-	for len(s) > 0 {
-		// Parse label name
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("cannot find `=` char for label value at %s", s)
-		}
-		name := s[:n]
-		s = s[n+1:]
-
-		// Parse label value
-		qs, err := strconv.QuotedPrefix(s)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse value for label %q at %s: %w", name, s, err)
-		}
-		s = s[len(qs):]
-		value, err := strconv.Unquote(qs)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unquote value %q for label %q: %w", qs, name, err)
-		}
-
-		// Append the found field to dst.
-		dst = append(dst, logstorage.Field{
-			Name:  name,
-			Value: value,
-		})
-
-		// Check whether there are other labels remaining
-		if len(s) == 0 {
-			break
-		}
-		if !strings.HasPrefix(s, ",") {
-			return nil, fmt.Errorf("missing `,` char at %s", s)
-		}
-		s = s[1:]
-		s = strings.TrimPrefix(s, " ")
-	}
-	return dst, nil
-}
-
-func getPushRequest() *PushRequest {
-	v := pushReqsPool.Get()
-	if v == nil {
-		return &PushRequest{}
-	}
-	return v.(*PushRequest)
-}
-
-func putPushRequest(req *PushRequest) {
-	req.reset()
-	pushReqsPool.Put(req)
-}

app/vlinsert/loki/loki_protobuf_test.go

@@ -1,218 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-type testLogMessageProcessor struct {
-	pr PushRequest
-}
-
-func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("unexpected non-nil streamFields: %v", streamFields))
-	}
-	msg := ""
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			msg = f.Value
-		}
-	}
-	var a []string
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			continue
-		}
-		item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-		a = append(a, item)
-	}
-	labels := "{" + strings.Join(a, ", ") + "}"
-	tlp.pr.Streams = append(tlp.pr.Streams, Stream{
-		Labels: labels,
-		Entries: []Entry{
-			{
-				Timestamp: time.Unix(0, timestamp),
-				Line:      strings.Clone(msg),
-			},
-		},
-	})
-}
-
-func (tlp *testLogMessageProcessor) MustClose() {
-}
-
-func TestParseProtobufRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
-		}
-
-		data := tlp.pr.MarshalProtobuf(nil)
-
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
-}
-
-func TestParseProtobufRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
-		}
-
-		data := tlp.pr.MarshalProtobuf(nil)
-
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"432\",\"parent_id\":\"qwerty\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"432","parent_id":"qwerty"}`)
-}
-
-func TestParsePromLabels_Success(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		var a []string
-		for _, f := range fields {
-			a = append(a, fmt.Sprintf("%s=%q", f.Name, f.Value))
-		}
-		result := "{" + strings.Join(a, ", ") + "}"
-		if result != s {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, s)
-		}
-	}
-
-	f("{}")
-	f(`{foo="bar"}`)
-	f(`{foo="bar", baz="x", y="z"}`)
-	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
-}
-
-func TestParsePromLabels_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(fields) > 0 {
-			t.Fatalf("unexpected non-empty fields: %s", fields)
-		}
-	}
-
-	f("")
-	f("{")
-	f(`{foo}`)
-	f(`{foo=bar}`)
-	f(`{foo="bar}`)
-	f(`{foo="ba\",r}`)
-	f(`{foo="bar" baz="aa"}`)
-	f(`foobar`)
-	f(`foo{bar="baz"}`)
-}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -1,80 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			if err := parseProtobufRequest(body, blp, nil, false, true); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
-	var b []byte
-	var entries []Entry
-	streams := make([]Stream, streamsCount)
-	for i := range streams {
-		b = b[:0]
-		b = append(b, '{')
-		for j := 0; j < labelsCount; j++ {
-			b = append(b, "label_"...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, `="value_`...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, '"')
-			if j < labelsCount-1 {
-				b = append(b, ',')
-			}
-		}
-		b = append(b, '}')
-		labels := string(b)
-
-		var rowsBuf []byte
-		entriesLen := len(entries)
-		for j := 0; j < rowsCount; j++ {
-			rowsBufLen := len(rowsBuf)
-			rowsBuf = append(rowsBuf, "value_"...)
-			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
-			entries = append(entries, Entry{
-				Timestamp: time.Now(),
-				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
-			})
-		}
-
-		st := &streams[i]
-		st.Labels = labels
-		st.Entries = entries[entriesLen:]
-	}
-	pr := PushRequest{
-		Streams: streams,
-	}
-
-	return pr.MarshalProtobuf(nil)
-}

app/vlinsert/loki/pb.go

@@ -1,302 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: push_request.proto
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package loki
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// PushRequest represents Loki PushRequest
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L14
-type PushRequest struct {
-	Streams []Stream
-
-	entriesBuf   []Entry
-	labelPairBuf []LabelPair
-}
-
-func (pr *PushRequest) reset() {
-	pr.Streams = pr.Streams[:0]
-
-	pr.entriesBuf = pr.entriesBuf[:0]
-	pr.labelPairBuf = pr.labelPairBuf[:0]
-}
-
-// UnmarshalProtobuf unmarshals pr from protobuf message at src.
-//
-// pr remains valid until src is modified.
-func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
-	pr.reset()
-	var err error
-	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
-	return err
-}
-
-// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
-func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	pr.marshalProtobuf(m.MessageMarshaler())
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, s := range pr.Streams {
-		s.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message PushRequest {
-	//   repeated Stream streams = 1;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
-			}
-			pr.Streams = append(pr.Streams, Stream{})
-			s := &pr.Streams[len(pr.Streams)-1]
-			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
-			}
-		}
-	}
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Stream represents Loki stream.
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L23
-type Stream struct {
-	Labels  string
-	Entries []Entry
-}
-
-func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, s.Labels)
-	for _, e := range s.Entries {
-		e.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message Stream {
-	//   string labels = 1;
-	//   repeated Entry entries = 2;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	entriesBufLen := len(entriesBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			labels, ok := fc.String()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
-			}
-			s.Labels = labels
-		case 2:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
-			}
-			entriesBuf = append(entriesBuf, Entry{})
-			e := &entriesBuf[len(entriesBuf)-1]
-			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
-			}
-		}
-	}
-	s.Entries = entriesBuf[entriesBufLen:]
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Entry represents Loki entry.
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L38
-type Entry struct {
-	Timestamp          time.Time
-	Line               string
-	StructuredMetadata []LabelPair
-}
-
-func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	marshalTime(mm, 1, e.Timestamp)
-	mm.AppendString(2, e.Line)
-	for _, lp := range e.StructuredMetadata {
-		lp.marshalProtobuf(mm.AppendMessage(3))
-	}
-}
-
-func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
-	// message Entry {
-	//   Timestamp timestamp = 1;
-	//   string line = 2;
-	//   repeated LabelPair structuredMetadata = 3;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	labelPairBufLen := len(labelPairBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
-			}
-			timestamp, err := unmarshalTime(data)
-			if err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
-			}
-			e.Timestamp = timestamp
-		case 2:
-			line, ok := fc.String()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Line")
-			}
-			e.Line = line
-		case 3:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
-			}
-			labelPairBuf = append(labelPairBuf, LabelPair{})
-			lp := &labelPairBuf[len(labelPairBuf)-1]
-			if err := lp.unmarshalProtobuf(data); err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
-			}
-		}
-	}
-	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
-	return labelPairBuf, nil
-}
-
-// LabelPair represents Loki label pair.
-//
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L33
-type LabelPair struct {
-	Name  string
-	Value string
-}
-
-func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, lp.Name)
-	mm.AppendString(2, lp.Value)
-}
-
-func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
-	// message LabelPair {
-	//   string name = 1;
-	//   string value = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			name, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot read name")
-			}
-			lp.Name = name
-		case 2:
-			value, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot unmarshal value")
-			}
-			lp.Value = value
-		}
-	}
-	return nil
-}
-
-func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
-	nsecs := timestamp.UnixNano()
-	ts := Timestamp{
-		Seconds: nsecs / 1e9,
-		Nanos:   int32(nsecs % 1e9),
-	}
-	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
-}
-
-func unmarshalTime(src []byte) (time.Time, error) {
-	var ts Timestamp
-	if err := ts.unmarshalProtobuf(src); err != nil {
-		return time.Time{}, err
-	}
-	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	return timestamp, nil
-}
-
-// Timestamp is protobuf well-known timestamp type.
-type Timestamp struct {
-	Seconds int64
-	Nanos   int32
-}
-
-func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendInt64(1, ts.Seconds)
-	mm.AppendInt32(2, ts.Nanos)
-}
-
-func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
-	// message Timestamp {
-	//   int64 seconds = 1;
-	//   int32 nanos = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			seconds, ok := fc.Int64()
-			if !ok {
-				return fmt.Errorf("cannot read Seconds")
-			}
-			ts.Seconds = seconds
-		case 2:
-			nanos, ok := fc.Int32()
-			if !ok {
-				return fmt.Errorf("cannot read Nanos")
-			}
-			ts.Nanos = nanos
-		}
-	}
-	return nil
-}

app/vlinsert/main.go

@@ -1,88 +0,0 @@
-package vlinsert
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-)
-
-var (
-	disableInsert   = flag.Bool("insert.disable", false, "Whether to disable /insert/* HTTP endpoints")
-	disableInternal = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
-)
-
-// Init initializes vlinsert
-func Init() {
-	syslog.MustInit()
-}
-
-// Stop stops vlinsert
-func Stop() {
-	syslog.MustStop()
-}
-
-// RequestHandler handles insert requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
-
-	if strings.HasPrefix(path, "/insert/") {
-		if *disableInsert {
-			httpserver.Errorf(w, r, "requests to /insert/* are disabled with -insert.disable command-line flag")
-			return true
-		}
-
-		return insertHandler(w, r, path)
-	}
-
-	if path == "/internal/insert" {
-		if *disableInternal || *disableInsert {
-			httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable or -insert.disable command-line flag")
-			return true
-		}
-		internalinsert.RequestHandler(w, r)
-		return true
-	}
-
-	return false
-}
-
-func insertHandler(w http.ResponseWriter, r *http.Request, path string) bool {
-	switch path {
-	case "/insert/jsonline":
-		jsonline.RequestHandler(w, r)
-		return true
-	case "/insert/ready":
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(200)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	}
-	switch {
-	// some clients may omit trailing slash at elasticsearch protocol.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
-	case strings.HasPrefix(path, "/insert/elasticsearch"):
-		return elasticsearch.RequestHandler(path, w, r)
-
-	case strings.HasPrefix(path, "/insert/loki/"):
-		return loki.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/opentelemetry/"):
-		return opentelemetry.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/journald/"):
-		return journald.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/insert/datadog/"):
-		return datadog.RequestHandler(path, w, r)
-	}
-
-	return false
-}

app/vlinsert/opentelemetry/opentelemetry.go

@@ -1,153 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var maxRequestSize = flagutil.NewBytes("opentelemetry.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single OpenTelemetry request")
-
-// RequestHandler processes Opentelemetry insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	// use the same path as opentelemetry collector
-	// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
-	case "/insert/opentelemetry/v1/logs":
-		if r.Header.Get("Content-Type") == "application/json" {
-			httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-			return true
-		}
-		handleProtobuf(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := insertutil.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf", false)
-		useDefaultStreamFields := len(cp.StreamFields) == 0
-		err := pushProtobufRequest(data, lmp, cp.MsgFields, useDefaultStreamFields)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read OpenTelemetry protocol data: %s", err)
-		return
-	}
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-
-	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-)
-
-func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {
-	var req pb.ExportLogsServiceRequest
-	if err := req.UnmarshalProtobuf(data); err != nil {
-		errorsTotal.Inc()
-		return fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
-	}
-
-	var commonFields []logstorage.Field
-	for _, rl := range req.ResourceLogs {
-		commonFields = commonFields[:0]
-		commonFields = appendKeyValues(commonFields, rl.Resource.Attributes, "")
-		commonFieldsLen := len(commonFields)
-		for _, sc := range rl.ScopeLogs {
-			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, msgFields, useDefaultStreamFields)
-		}
-	}
-
-	return nil
-}
-
-func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) []logstorage.Field {
-	fields := commonFields
-	for _, lr := range sc.LogRecords {
-		fields = fields[:len(commonFields)]
-		if lr.Body.KeyValueList != nil {
-			fields = appendKeyValues(fields, lr.Body.KeyValueList.Values, "")
-			logstorage.RenameField(fields[len(commonFields):], msgFields, "_msg")
-		} else {
-			fields = append(fields, logstorage.Field{
-				Name:  "_msg",
-				Value: lr.Body.FormatString(true),
-			})
-		}
-		fields = appendKeyValues(fields, lr.Attributes, "")
-		if len(lr.TraceID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "trace_id",
-				Value: lr.TraceID,
-			})
-		}
-		if len(lr.SpanID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "span_id",
-				Value: lr.SpanID,
-			})
-		}
-		fields = append(fields, logstorage.Field{
-			Name:  "severity",
-			Value: lr.FormatSeverity(),
-		})
-
-		var streamFields []logstorage.Field
-		if useDefaultStreamFields {
-			streamFields = commonFields
-		}
-		lmp.AddRow(lr.ExtractTimestampNano(), fields, streamFields)
-	}
-	return fields
-}
-
-func appendKeyValues(fields []logstorage.Field, kvs []*pb.KeyValue, parentField string) []logstorage.Field {
-	for _, attr := range kvs {
-		fieldName := attr.Key
-		if parentField != "" {
-			fieldName = parentField + "." + fieldName
-		}
-
-		if attr.Value.KeyValueList != nil {
-			fields = appendKeyValues(fields, attr.Value.KeyValueList.Values, fieldName)
-		} else {
-			fields = append(fields, logstorage.Field{
-				Name:  fieldName,
-				Value: attr.Value.FormatString(true),
-			})
-		}
-	}
-	return fields
-}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -1,208 +0,0 @@
-package opentelemetry
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func TestPushProtoOk(t *testing.T) {
-	f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-		lr := pb.ExportLogsServiceRequest{
-			ResourceLogs: src,
-		}
-
-		pData := lr.MarshalProtobuf(nil)
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := pushProtobufRequest(pData, tlp, nil, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// single line without resource attributes
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234},
-		`{"_msg":"log-line-message","severity":"Trace"}`,
-	)
-
-	// severities mapping
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 13, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 24, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1234, 1234},
-		`{"_msg":"log-line-message","severity":"Trace"}
-{"_msg":"log-line-message","severity":"Warn"}
-{"_msg":"log-line-message","severity":"Fatal4"}`,
-	)
-
-	// multi-line with resource attributes
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 25, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
-	)
-
-	// multi-scope with resource attributes and multi-line
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
-						{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
-					},
-				},
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
-						{TraceID: "1234", SpanID: "45", ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
-						{TraceID: "4bf92f3577b34da6a3ce929d0e0e4736", SpanID: "00f067aa0ba902b7", ObservedTimeUnixNano: 3333, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 2345, 2346, 2347, 2348, 3333},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Debug"}
-{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-1","trace_id":"1234","span_id":"45","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-2","trace_id":"4bf92f3577b34da6a3ce929d0e0e4736","span_id":"00f067aa0ba902b7","severity":"Unspecified"}`,
-	)
-
-	// nested fields
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{
-							TimeUnixNano: 1234,
-							Body:         pb.AnyValue{StringValue: ptrTo("nested fields")},
-							Attributes: []*pb.KeyValue{
-								{Key: "error", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-									{
-										Key:   "type",
-										Value: &pb.AnyValue{StringValue: ptrTo("document_parsing_exception")},
-									},
-									{
-										Key:   "reason",
-										Value: &pb.AnyValue{StringValue: ptrTo("failed to parse field [_msg] of type [text]")},
-									},
-									{
-										Key: "caused_by",
-										Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-											{
-												Key:   "type",
-												Value: &pb.AnyValue{StringValue: ptrTo("x_content_parse_exception")},
-											},
-											{
-												Key:   "reason",
-												Value: &pb.AnyValue{StringValue: ptrTo("unexpected end-of-input in VALUE_STRING")},
-											},
-											{
-												Key: "caused_by",
-												Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-													{
-														Key:   "type",
-														Value: &pb.AnyValue{StringValue: ptrTo("json_e_o_f_exception")},
-													},
-													{
-														Key:   "reason",
-														Value: &pb.AnyValue{StringValue: ptrTo("eof")},
-													},
-												}}},
-											},
-										}}},
-									},
-								}}}},
-							},
-						},
-					},
-				},
-			},
-		},
-	}, []int64{1234},
-		`{"_msg":"nested fields","error.type":"document_parsing_exception","error.reason":"failed to parse field [_msg] of type [text]",`+
-			`"error.caused_by.type":"x_content_parse_exception","error.caused_by.reason":"unexpected end-of-input in VALUE_STRING",`+
-			`"error.caused_by.caused_by.type":"json_e_o_f_exception","error.caused_by.caused_by.reason":"eof","severity":"Unspecified"}`)
-}
-
-func ptrTo[T any](s T) *T {
-	return &s
-}

app/vlinsert/opentelemetry/opentelemetry_timing_test.go

@@ -1,78 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, scopes := range []int{1, 2} {
-		for _, rows := range []int{100, 1000} {
-			for _, attributes := range []int{5, 10} {
-				b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, scopes, rows, attributes)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			if err := pushProtobufRequest(body, blp, nil, false); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
-	msg := "12345678910"
-
-	attrValues := []*pb.AnyValue{
-		{StringValue: ptrTo("string-attribute")},
-		{IntValue: ptrTo[int64](12345)},
-		{DoubleValue: ptrTo(3.14)},
-	}
-	attrs := make([]*pb.KeyValue, attributesCount)
-	for j := 0; j < attributesCount; j++ {
-		attrs[j] = &pb.KeyValue{
-			Key:   fmt.Sprintf("key-%d", j),
-			Value: attrValues[j%3],
-		}
-	}
-	entries := make([]pb.LogRecord, rowsCount)
-	for j := 0; j < rowsCount; j++ {
-		entries[j] = pb.LogRecord{
-			TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
-		}
-	}
-	scopes := make([]pb.ScopeLogs, scopesCount)
-
-	for j := 0; j < scopesCount; j++ {
-		scopes[j] = pb.ScopeLogs{
-			LogRecords: entries,
-		}
-	}
-
-	pr := pb.ExportLogsServiceRequest{
-		ResourceLogs: []pb.ResourceLogs{
-			{
-				Resource: pb.Resource{
-					Attributes: attrs,
-				},
-				ScopeLogs: scopes,
-			},
-		},
-	}
-
-	return pr.MarshalProtobuf(nil)
-}

app/vlinsert/syslog/syslog.go

@@ -1,607 +0,0 @@
-package syslog
-
-import (
-	"bufio"
-	"crypto/tls"
-	"encoding/json"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"net"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
-		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	streamFieldsTCP = flagutil.NewArrayString("syslog.streamFields.tcp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-	streamFieldsUDP = flagutil.NewArrayString("syslog.streamFields.udp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-
-	ignoreFieldsTCP = flagutil.NewArrayString("syslog.ignoreFields.tcp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-	ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-
-	decolorizeFieldsTCP = flagutil.NewArrayString("syslog.decolorizeFields.tcp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-	decolorizeFieldsUDP = flagutil.NewArrayString("syslog.decolorizeFields.udp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-
-	extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-	extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-
-	tenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
-	tenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
-
-	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-	listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+
-		"The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
-		"Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
-		"The provided key file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+
-		"See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+
-		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+
-		"Supported values: TLS10, TLS11, TLS12, TLS13. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-
-	compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-	compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-
-	useLocalTimestampTCP = flagutil.NewArrayBool("syslog.useLocalTimestamp.tcp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
-	useLocalTimestampUDP = flagutil.NewArrayBool("syslog.useLocalTimestamp.udp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
-)
-
-// MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports
-//
-// This function must be called after flag.Parse().
-//
-// MustStop() must be called in order to free up resources occupied by the initialized syslog parser.
-func MustInit() {
-	if workersStopCh != nil {
-		logger.Panicf("BUG: MustInit() called twice without MustStop() call")
-	}
-	workersStopCh = make(chan struct{})
-
-	for argIdx, addr := range *listenAddrTCP {
-		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runTCPListener(addr, argIdx)
-			workersWG.Done()
-		}(addr, argIdx)
-	}
-
-	for argIdx, addr := range *listenAddrUDP {
-		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runUDPListener(addr, argIdx)
-			workersWG.Done()
-		}(addr, argIdx)
-	}
-
-	currentYear := time.Now().Year()
-	globalCurrentYear.Store(int64(currentYear))
-	workersWG.Add(1)
-	go func() {
-		ticker := time.NewTicker(time.Minute)
-		for {
-			select {
-			case <-workersStopCh:
-				ticker.Stop()
-				workersWG.Done()
-				return
-			case <-ticker.C:
-				currentYear := time.Now().Year()
-				globalCurrentYear.Store(int64(currentYear))
-			}
-		}
-	}()
-
-	if *syslogTimezone != "" {
-		tz, err := time.LoadLocation(*syslogTimezone)
-		if err != nil {
-			logger.Fatalf("cannot parse -syslog.timezone=%q: %s", *syslogTimezone, err)
-		}
-		globalTimezone = tz
-	} else {
-		globalTimezone = time.Local
-	}
-}
-
-var (
-	globalCurrentYear atomic.Int64
-	globalTimezone    *time.Location
-)
-
-var (
-	workersWG     sync.WaitGroup
-	workersStopCh chan struct{}
-)
-
-// MustStop stops syslog parser initialized via MustInit()
-func MustStop() {
-	close(workersStopCh)
-	workersWG.Wait()
-	workersStopCh = nil
-}
-
-func runUDPListener(addr string, argIdx int) {
-	ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr)
-	if err != nil {
-		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
-	}
-
-	tenantIDStr := tenantIDUDP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodUDP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "udp")
-
-	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
-
-	streamFieldsStr := streamFieldsUDP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.udp=%q for -syslog.listenAddr.udp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsUDP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	decolorizeFieldsStr := decolorizeFieldsUDP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.udp=%q for -syslog.listenAddr.udp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.udp=%q for -syslog.listenAddr.udp=%q: %s", extraFieldsStr, addr, err)
-	}
-
-	doneCh := make(chan struct{})
-	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
-		close(doneCh)
-	}()
-
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
-	<-workersStopCh
-	if err := ln.Close(); err != nil {
-		logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err)
-	}
-	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
-}
-
-func runTCPListener(addr string, argIdx int) {
-	var tlsConfig *tls.Config
-	if tlsEnable.GetOptionalArg(argIdx) {
-		certFile := tlsCertFile.GetOptionalArg(argIdx)
-		keyFile := tlsKeyFile.GetOptionalArg(argIdx)
-		tc, err := netutil.GetServerTLSConfig(certFile, keyFile, *tlsMinVersion, *tlsCipherSuites)
-		if err != nil {
-			logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s",
-				certFile, keyFile, *tlsMinVersion, *tlsCipherSuites, err)
-		}
-		tlsConfig = tc
-	}
-	ln, err := netutil.NewTCPListener("syslog", addr, false, tlsConfig)
-	if err != nil {
-		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
-	}
-
-	tenantIDStr := tenantIDTCP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodTCP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "tcp")
-
-	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
-
-	streamFieldsStr := streamFieldsTCP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsTCP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	decolorizeFieldsStr := decolorizeFieldsTCP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", extraFieldsStr, addr, err)
-	}
-
-	doneCh := make(chan struct{})
-	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
-		close(doneCh)
-	}()
-
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
-	<-workersStopCh
-	if err := ln.Close(); err != nil {
-		logger.Fatalf("syslog: cannot close TCP listener at %s: %s", addr, err)
-	}
-	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
-}
-
-func checkCompressMethod(compressMethod, addr, protocol string) {
-	switch compressMethod {
-	case "", "none", "zstd", "gzip", "deflate":
-		return
-	default:
-		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
-	}
-}
-
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
-	gomaxprocs := cgroup.AvailableCPUs()
-	var wg sync.WaitGroup
-	localAddr := ln.LocalAddr()
-	for i := 0; i < gomaxprocs; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
-			var bb bytesutil.ByteBuffer
-			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
-			for {
-				bb.Reset()
-				bb.B = bb.B[:cap(bb.B)]
-				n, remoteAddr, err := ln.ReadFrom(bb.B)
-				if err != nil {
-					udpErrorsTotal.Inc()
-					var ne net.Error
-					if errors.As(err, &ne) {
-						if ne.Temporary() {
-							logger.Errorf("syslog: temporary error when listening for UDP at %q: %s", localAddr, err)
-							time.Sleep(time.Second)
-							continue
-						}
-						if strings.Contains(err.Error(), "use of closed network connection") {
-							break
-						}
-					}
-					logger.Errorf("syslog: cannot read UDP data from %s at %s: %s", remoteAddr, localAddr, err)
-					continue
-				}
-				bb.B = bb.B[:n]
-				udpRequestsTotal.Inc()
-				if err := processStream("udp", bb.NewReader(), encoding, useLocalTimestamp, cp); err != nil {
-					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
-				}
-			}
-		}()
-	}
-	wg.Wait()
-}
-
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
-	var cm ingestserver.ConnsMap
-	cm.Init("syslog")
-
-	var wg sync.WaitGroup
-	addr := ln.Addr()
-	for {
-		c, err := ln.Accept()
-		if err != nil {
-			var ne net.Error
-			if errors.As(err, &ne) {
-				if ne.Temporary() {
-					logger.Errorf("syslog: temporary error when listening for TCP addr %q: %s", addr, err)
-					time.Sleep(time.Second)
-					continue
-				}
-				if strings.Contains(err.Error(), "use of closed network connection") {
-					break
-				}
-				logger.Fatalf("syslog: unrecoverable error when accepting TCP connections at %q: %s", addr, err)
-			}
-			logger.Fatalf("syslog: unexpected error when accepting TCP connections at %q: %s", addr, err)
-		}
-		if !cm.Add(c) {
-			_ = c.Close()
-			break
-		}
-
-		wg.Add(1)
-		go func() {
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
-			if err := processStream("tcp", c, encoding, useLocalTimestamp, cp); err != nil {
-				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
-			}
-
-			cm.Delete(c)
-			_ = c.Close()
-			wg.Done()
-		}()
-	}
-
-	cm.CloseAll(0)
-	wg.Wait()
-}
-
-// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutil.CommonParams) error {
-	if err := insertutil.CanWriteData(); err != nil {
-		return err
-	}
-
-	lmp := cp.NewLogMessageProcessor("syslog_"+protocol, true)
-	err := processStreamInternal(r, encoding, useLocalTimestamp, lmp)
-	lmp.MustClose()
-
-	return err
-}
-
-func processStreamInternal(r io.Reader, encoding string, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return fmt.Errorf("cannot decode syslog data: %w", err)
-	}
-	defer protoparserutil.PutUncompressedReader(reader)
-
-	return processUncompressedStream(reader, useLocalTimestamp, lmp)
-}
-
-func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	slr := getSyslogLineReader(wcr)
-	defer putSyslogLineReader(slr)
-
-	n := 0
-	for {
-		ok := slr.nextLine()
-		wcr.DecConcurrency()
-		if !ok {
-			break
-		}
-
-		currentYear := int(globalCurrentYear.Load())
-		err := processLine(slr.line, currentYear, globalTimezone, useLocalTimestamp, lmp)
-		if err != nil {
-			errorsTotal.Inc()
-			return fmt.Errorf("cannot read line #%d: %s", n, err)
-		}
-		n++
-	}
-	return slr.Error()
-}
-
-type syslogLineReader struct {
-	line []byte
-
-	br  *bufio.Reader
-	err error
-}
-
-func (slr *syslogLineReader) reset(r io.Reader) {
-	slr.line = slr.line[:0]
-	slr.br.Reset(r)
-	slr.err = nil
-}
-
-// Error returns the last error occurred in slr.
-func (slr *syslogLineReader) Error() error {
-	if slr.err == nil || slr.err == io.EOF {
-		return nil
-	}
-	return slr.err
-}
-
-// nextLine reads the next syslog line from slr and stores it at slr.line.
-//
-// false is returned if the next line cannot be read. Error() must be called in this case
-// in order to verify whether there is an error or just slr stream has been finished.
-func (slr *syslogLineReader) nextLine() bool {
-	if slr.err != nil {
-		return false
-	}
-
-again:
-	prefix, err := slr.br.ReadSlice(' ')
-	if err != nil {
-		if err != io.EOF {
-			slr.err = fmt.Errorf("cannot read message frame prefix: %w", err)
-			return false
-		}
-		if len(prefix) == 0 {
-			slr.err = err
-			return false
-		}
-	}
-	// skip empty lines
-	for len(prefix) > 0 && prefix[0] == '\n' {
-		prefix = prefix[1:]
-	}
-	if len(prefix) == 0 {
-		// An empty prefix or a prefix with empty lines - try reading yet another prefix.
-		goto again
-	}
-
-	if prefix[0] >= '0' && prefix[0] <= '9' {
-		// This is octet-counting method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#msgxfer
-		msgLenStr := bytesutil.ToUnsafeString(prefix[:len(prefix)-1])
-		msgLen, err := strconv.ParseUint(msgLenStr, 10, 64)
-		if err != nil {
-			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
-			return false
-		}
-		if maxMsgLen := insertutil.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
-			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
-			return false
-		}
-		slr.line = slicesutil.SetLength(slr.line, int(msgLen))
-		if _, err := io.ReadFull(slr.br, slr.line); err != nil {
-			slr.err = fmt.Errorf("cannot read message with size %d bytes: %w", msgLen, err)
-			return false
-		}
-		return true
-	}
-
-	// This is octet-stuffing method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#octet-stuffing-legacy
-	slr.line = append(slr.line[:0], prefix...)
-	for {
-		line, err := slr.br.ReadSlice('\n')
-		if err == nil {
-			slr.line = append(slr.line, line[:len(line)-1]...)
-			return true
-		}
-		if err == io.EOF {
-			slr.line = append(slr.line, line...)
-			return true
-		}
-		if err == bufio.ErrBufferFull {
-			slr.line = append(slr.line, line...)
-			continue
-		}
-		slr.err = fmt.Errorf("cannot read message in octet-stuffing method: %w", err)
-		return false
-	}
-}
-
-func getSyslogLineReader(r io.Reader) *syslogLineReader {
-	v := syslogLineReaderPool.Get()
-	if v == nil {
-		br := bufio.NewReaderSize(r, 64*1024)
-		return &syslogLineReader{
-			br: br,
-		}
-	}
-	slr := v.(*syslogLineReader)
-	slr.reset(r)
-	return slr
-}
-
-func putSyslogLineReader(slr *syslogLineReader) {
-	syslogLineReaderPool.Put(slr)
-}
-
-var syslogLineReaderPool sync.Pool
-
-func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	p := logstorage.GetSyslogParser(currentYear, timezone)
-	lineStr := bytesutil.ToUnsafeString(line)
-	p.Parse(lineStr)
-
-	var ts int64
-	if useLocalTimestamp {
-		ts = time.Now().UnixNano()
-	} else {
-		nsecs, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
-		if err != nil {
-			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
-		}
-		ts = nsecs
-	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
-	logstorage.PutSyslogParser(p)
-
-	return nil
-}
-
-var timeFields = []string{"timestamp"}
-var msgFields = []string{"message"}
-
-var (
-	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
-
-	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
-	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
-)
-
-func parseFieldsList(s string) ([]string, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var a []string
-	err := json.Unmarshal([]byte(s), &a)
-	return a, err
-}
-
-func parseExtraFields(s string) ([]logstorage.Field, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var m map[string]string
-	if err := json.Unmarshal([]byte(s), &m); err != nil {
-		return nil, err
-	}
-	fields := make([]logstorage.Field, 0, len(m))
-	for k, v := range m {
-		fields = append(fields, logstorage.Field{
-			Name:  k,
-			Value: v,
-		})
-	}
-	sort.Slice(fields, func(i, j int) bool {
-		return fields[i].Name < fields[j].Name
-	})
-	return fields, nil
-}

app/vlinsert/syslog/syslog_test.go

@@ -1,129 +0,0 @@
-package syslog
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-)
-
-func TestSyslogLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		var lines []string
-		for slr.nextLine() {
-			lines = append(lines, string(slr.line))
-		}
-		if err := slr.Error(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines read;\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", nil)
-	f("\n\n\n", nil)
-
-	f("foobar", []string{"foobar"})
-	f("foobar\n", []string{"foobar\n"})
-	f("\n\nfoo\n\nbar\n\n", []string{"foo\n\nbar\n\n"})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...`, []string{"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches..."})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
-
-`, []string{
-		"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...",
-		"<165>Jun  4 12:08:33 abcd systemd[345]: abc defg",
-		`<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`,
-	})
-}
-
-func TestSyslogLineReader_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		if slr.nextLine() {
-			t.Fatalf("expecting failure to read the first line")
-		}
-		if err := slr.Error(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 aa")
-	f("1233423432 abc")
-}
-
-func TestProcessStreamInternal_Success(t *testing.T) {
-	f := func(data string, currentYear int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		MustInit()
-		defer MustStop()
-
-		globalTimezone = time.UTC
-		globalCurrentYear.Store(int64(currentYear))
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	data := `Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
-`
-	currentYear := 2023
-	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
-	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
-	f(data, currentYear, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		MustInit()
-		defer MustStop()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 foo")
-	f("123456789 bar")
-}

app/vlogscli/Makefile

@@ -1,109 +0,0 @@
-# All these commands must run from repository root.
-
-vlogscli:
-	APP_NAME=vlogscli $(MAKE) app-local
-
-vlogscli-race:
-	APP_NAME=vlogscli RACE=-race $(MAKE) app-local
-
-vlogscli-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker
-
-vlogscli-pure-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-pure
-
-vlogscli-linux-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-amd64
-
-vlogscli-linux-arm-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm
-
-vlogscli-linux-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm64
-
-vlogscli-linux-ppc64le-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-ppc64le
-
-vlogscli-linux-386-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-386
-
-vlogscli-darwin-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-amd64
-
-vlogscli-darwin-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-arm64
-
-vlogscli-freebsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-freebsd-amd64
-
-vlogscli-openbsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-openbsd-amd64
-
-vlogscli-windows-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-windows-amd64
-
-package-vlogscli:
-	APP_NAME=vlogscli $(MAKE) package-via-docker
-
-package-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-pure
-
-package-vlogscli-amd64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-amd64
-
-package-vlogscli-arm:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm
-
-package-vlogscli-arm64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm64
-
-package-vlogscli-ppc64le:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-ppc64le
-
-package-vlogscli-386:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-386
-
-publish-vlogscli:
-	APP_NAME=vlogscli $(MAKE) publish-via-docker
-
-vlogscli-linux-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-ppc64le:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-s390x:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-loong64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-386:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-freebsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-openbsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-windows-amd64:
-	GOARCH=amd64 APP_NAME=vlogscli $(MAKE) app-local-windows-goarch
-
-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) app-local-pure
-
-run-vlogscli:
-	APP_NAME=vlogscli $(MAKE) run-via-docker

app/vlogscli/README.md

@@ -1,5 +1 @@
-# vlogscli
-
-Command-line utility for querying [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlogscli/deployment/Dockerfile

@@ -1,6 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-ENTRYPOINT ["/vlogscli-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vlogscli-prod

app/vlogscli/json_prettifier.go

@@ -1,245 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-type outputMode int
-
-const (
-	outputModeJSONMultiline  = outputMode(0)
-	outputModeJSONSingleline = outputMode(1)
-	outputModeLogfmt         = outputMode(2)
-	outputModeCompact        = outputMode(3)
-)
-
-func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
-	switch outputMode {
-	case outputModeJSONMultiline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, true)
-		}
-	case outputModeJSONSingleline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, false)
-		}
-	case outputModeLogfmt:
-		return writeLogfmtObject
-	case outputModeCompact:
-		return writeCompactObject
-	default:
-		panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
-	}
-}
-
-type jsonPrettifier struct {
-	r         io.ReadCloser
-	formatter func(w io.Writer, fields []logstorage.Field) error
-
-	d *json.Decoder
-
-	pr *io.PipeReader
-	pw *io.PipeWriter
-	bw *bufio.Writer
-
-	wg sync.WaitGroup
-}
-
-func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
-	d := json.NewDecoder(r)
-	pr, pw := io.Pipe()
-	bw := bufio.NewWriter(pw)
-
-	formatter := getOutputFormatter(outputMode)
-
-	jp := &jsonPrettifier{
-		r:         r,
-		formatter: formatter,
-
-		d: d,
-
-		pr: pr,
-		pw: pw,
-		bw: bw,
-	}
-
-	jp.wg.Add(1)
-	go func() {
-		defer jp.wg.Done()
-		err := jp.prettifyJSONLines()
-		jp.closePipesWithError(err)
-	}()
-
-	return jp
-}
-
-func (jp *jsonPrettifier) closePipesWithError(err error) {
-	_ = jp.pr.CloseWithError(err)
-	_ = jp.pw.CloseWithError(err)
-}
-
-func (jp *jsonPrettifier) prettifyJSONLines() error {
-	for jp.d.More() {
-		fields, err := readNextJSONObject(jp.d)
-		if err != nil {
-			return err
-		}
-		sort.Slice(fields, func(i, j int) bool {
-			return fields[i].Name < fields[j].Name
-		})
-		if err := jp.formatter(jp.bw, fields); err != nil {
-			return err
-		}
-
-		// Flush bw after every output line in order to show results as soon as they appear.
-		if err := jp.bw.Flush(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (jp *jsonPrettifier) Close() error {
-	jp.closePipesWithError(io.ErrUnexpectedEOF)
-	err := jp.r.Close()
-	jp.wg.Wait()
-	return err
-}
-
-func (jp *jsonPrettifier) Read(p []byte) (int, error) {
-	return jp.pr.Read(p)
-}
-
-func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
-	t, err := d.Token()
-	if err != nil {
-		return nil, fmt.Errorf("cannot read '{': %w", err)
-	}
-	delim, ok := t.(json.Delim)
-	if !ok || delim.String() != "{" {
-		return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
-	}
-
-	var fields []logstorage.Field
-	for {
-		// Read object key
-		t, err := d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object key or closing brace: %w", err)
-		}
-		delim, ok := t.(json.Delim)
-		if ok {
-			if delim.String() == "}" {
-				return fields, nil
-			}
-			return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
-		}
-		key, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object key: %v; want string or '}'", t)
-		}
-
-		// read object value
-		t, err = d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object value: %w", err)
-		}
-		value, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object value: %v; want string", t)
-		}
-
-		fields = append(fields, logstorage.Field{
-			Name:  key,
-			Value: value,
-		})
-	}
-}
-
-func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
-	data := logstorage.MarshalFieldsToLogfmt(nil, fields)
-	_, err := fmt.Fprintf(w, "%s\n", data)
-	return err
-}
-
-func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
-	if len(fields) == 1 {
-		// Just write field value as is without name
-		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
-		return err
-	}
-	if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
-		// Write _time\tfieldValue as is
-		if fields[0].Name == "_time" {
-			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)
-			return err
-		}
-		_, err := fmt.Fprintf(w, "%s\t%s\n", fields[1].Value, fields[0].Value)
-		return err
-	}
-
-	// Fall back to logfmt
-	return writeLogfmtObject(w, fields)
-}
-
-func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
-	if len(fields) == 0 {
-		fmt.Fprintf(w, "{}\n")
-		return nil
-	}
-
-	fmt.Fprintf(w, "{")
-	writeNewlineIfNeeded(w, isMultiline)
-	if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
-		return err
-	}
-	for _, f := range fields[1:] {
-		fmt.Fprintf(w, ",")
-		writeNewlineIfNeeded(w, isMultiline)
-		if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
-			return err
-		}
-	}
-	writeNewlineIfNeeded(w, isMultiline)
-	fmt.Fprintf(w, "}\n")
-	return nil
-}
-
-func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
-	if isMultiline {
-		fmt.Fprintf(w, "\n")
-	}
-}
-
-func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
-	key := getJSONString(f.Name)
-	value := getJSONString(f.Value)
-	if isMultiline {
-		_, err := fmt.Fprintf(w, "  %s: %s", key, value)
-		return err
-	}
-	_, err := fmt.Fprintf(w, "%s:%s", key, value)
-	return err
-}
-
-func getJSONString(s string) string {
-	data, err := json.Marshal(s)
-	if err != nil {
-		panic(fmt.Errorf("unexpected error when marshaling string to JSON: %w", err))
-	}
-	return jsonHTMLReplacer.Replace(string(data))
-}
-
-var jsonHTMLReplacer = strings.NewReplacer(
-	`\u003c`, "\u003c",
-	`\u003e`, "\u003e",
-	`\u0026`, "\u0026",
-)

app/vlogscli/less_wrapper.go

@@ -1,123 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-
-	"github.com/mattn/go-isatty"
-)
-
-func isTerminal() bool {
-	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
-}
-
-func readWithLess(r io.Reader, disableColors, wrapLongLines bool) error {
-	if !isTerminal() {
-		// Just write everything to stdout if no terminal is available.
-		_, err := io.Copy(os.Stdout, r)
-		if err != nil && !isErrPipe(err) {
-			return fmt.Errorf("error when forwarding data to stdout: %w", err)
-		}
-		if err := os.Stdout.Sync(); err != nil {
-			return fmt.Errorf("cannot sync data to stdout: %w", err)
-		}
-		return nil
-	}
-
-	pr, pw, err := os.Pipe()
-	if err != nil {
-		return fmt.Errorf("cannot create pipe: %w", err)
-	}
-	defer func() {
-		_ = pr.Close()
-		_ = pw.Close()
-	}()
-
-	// Ignore Ctrl+C in the current process, so 'less' could handle it properly
-	cancel := ignoreSignals(os.Interrupt)
-	defer cancel()
-
-	// Start 'less' process
-	path, err := exec.LookPath("less")
-	if err != nil {
-		return fmt.Errorf("cannot find 'less' command: %w", err)
-	}
-	opts := []string{"less", "-F", "-X"}
-	if !disableColors {
-		opts = append(opts, "-R")
-	}
-	if !wrapLongLines {
-		opts = append(opts, "-S")
-	}
-	p, err := os.StartProcess(path, opts, &os.ProcAttr{
-		Env:   append(os.Environ(), "LESSCHARSET=utf-8"),
-		Files: []*os.File{pr, os.Stdout, os.Stderr},
-	})
-	if err != nil {
-		return fmt.Errorf("cannot start 'less' process: %w", err)
-	}
-
-	// Close pr after 'less' finishes in a parallel goroutine
-	// in order to unblock forwarding data to stopped 'less' below.
-	waitch := make(chan *os.ProcessState)
-	go func() {
-		// Wait for 'less' process to finish.
-		ps, err := p.Wait()
-		if err != nil {
-			fatalf("unexpected error when waiting for 'less' process: %w", err)
-		}
-		_ = pr.Close()
-		waitch <- ps
-	}()
-
-	// Forward data from r to 'less'
-	_, err = io.Copy(pw, r)
-	_ = pw.Sync()
-	_ = pw.Close()
-
-	// Wait until 'less' finished
-	ps := <-waitch
-
-	// Verify 'less' status.
-	if !ps.Success() {
-		return fmt.Errorf("'less' finished with unexpected code %d", ps.ExitCode())
-	}
-
-	if err != nil && !isErrPipe(err) {
-		return fmt.Errorf("error when forwarding data to 'less': %w", err)
-	}
-
-	return nil
-}
-
-func isErrPipe(err error) bool {
-	return errors.Is(err, syscall.EPIPE) || errors.Is(err, io.ErrClosedPipe)
-}
-
-func ignoreSignals(sigs ...os.Signal) func() {
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, sigs...)
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for {
-			_, ok := <-ch
-			if !ok {
-				return
-			}
-		}
-	}()
-	return func() {
-		signal.Stop(ch)
-		close(ch)
-		wg.Wait()
-	}
-}

app/vlogscli/main.go

@@ -1,457 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strconv"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/ergochat/readline"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
-		"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs . See also -tail.url")
-	tailURL = flag.String("tail.url", "", "URL for live tailing queries to VictoriaLogs; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing ."+
-		"The url is automatically detected from -datasource.url by replacing /query with /tail at the end if -tail.url is empty")
-	historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
-	header      = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
-	accountID   = flag.Int("accountID", 0, "Account ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-	projectID   = flag.Int("projectID", 0, "Project ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-)
-
-const (
-	firstLinePrompt = ";> "
-	nextLinePrompt  = ""
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.InitNoLogFlags()
-
-	hes, err := parseHeaders(*header)
-	if err != nil {
-		fatalf("cannot parse -header command-line flag: %s", err)
-	}
-	headers = hes
-
-	incompleteLine := ""
-	cfg := &readline.Config{
-		Prompt:                 firstLinePrompt,
-		DisableAutoSaveHistory: true,
-		Listener: func(line []rune, pos int, _ rune) ([]rune, int, bool) {
-			incompleteLine = string(line)
-			return line, pos, false
-		},
-	}
-	rl, err := readline.NewFromConfig(cfg)
-	if err != nil {
-		fatalf("cannot initialize readline: %s", err)
-	}
-
-	fmt.Fprintf(rl, "sending queries to -datasource.url=%s\n", *datasourceURL)
-	fmt.Fprintf(rl, `type ? and press enter to see available commands`+"\n")
-	runReadlineLoop(rl, &incompleteLine)
-
-	if err := rl.Close(); err != nil {
-		fatalf("cannot close readline: %s", err)
-	}
-
-}
-
-func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
-	historyLines, err := loadFromHistory(*historyFile)
-	if err != nil {
-		fatalf("cannot load query history: %s", err)
-	}
-	for _, line := range historyLines {
-		if err := rl.SaveToHistory(line); err != nil {
-			fatalf("cannot initialize query history: %s", err)
-		}
-	}
-
-	outputMode := outputModeJSONMultiline
-	disableColors := true
-	wrapLongLines := false
-	s := ""
-	for {
-		line, err := rl.ReadLine()
-		if err != nil {
-			switch err {
-			case io.EOF:
-				if s != "" {
-					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode, disableColors, wrapLongLines)
-				}
-				return
-			case readline.ErrInterrupt:
-				if s == "" && *incompleteLine == "" {
-					fmt.Fprintf(rl, "interrupted\n")
-					os.Exit(128 + int(syscall.SIGINT))
-				}
-				// Default value for Ctrl+C - clear the prompt and store the incompletely entered line into history
-				s += *incompleteLine
-				historyLines = pushToHistory(rl, historyLines, s)
-				s = ""
-				rl.SetPrompt(firstLinePrompt)
-				continue
-			default:
-				fatalf("unexpected error in readline: %s", err)
-			}
-		}
-
-		s += line
-		if s == "" {
-			// Skip empty lines
-			continue
-		}
-
-		if isQuitCommand(s) {
-			fmt.Fprintf(rl, "bye!\n")
-			_ = pushToHistory(rl, historyLines, s)
-			return
-		}
-		if isHelpCommand(s) {
-			printCommandsHelp(rl)
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\s` {
-			fmt.Fprintf(rl, "singleline json output mode\n")
-			outputMode = outputModeJSONSingleline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\m` {
-			fmt.Fprintf(rl, "multiline json output mode\n")
-			outputMode = outputModeJSONMultiline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\c` {
-			fmt.Fprintf(rl, "compact output mode\n")
-			outputMode = outputModeCompact
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\logfmt` {
-			fmt.Fprintf(rl, "logfmt output mode\n")
-			outputMode = outputModeLogfmt
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\wrap_long_lines` {
-			if wrapLongLines {
-				wrapLongLines = false
-				fmt.Fprintf(rl, "wrapping of long lines is disabled\n")
-			} else {
-				wrapLongLines = true
-				fmt.Fprintf(rl, "wrapping of long lines is enabled\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\disable_colors` {
-			if !disableColors {
-				disableColors = true
-				fmt.Fprintf(rl, `disabled colors in compact output mode; enter \enable_colors for enabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\enable_colors` {
-			if disableColors {
-				disableColors = false
-				fmt.Fprintf(rl, `enabled colors in compact output mode; type \disable_colors for disabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if line != "" && !strings.HasSuffix(line, ";") {
-			// Assume the query is incomplete and allow the user finishing the query on the next line
-			s += "\n"
-			rl.SetPrompt(nextLinePrompt)
-			continue
-		}
-
-		// Execute the query
-		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode, disableColors, wrapLongLines)
-		cancel()
-
-		historyLines = pushToHistory(rl, historyLines, s)
-		s = ""
-		rl.SetPrompt(firstLinePrompt)
-	}
-}
-
-func pushToHistory(rl *readline.Instance, historyLines []string, s string) []string {
-	s = strings.TrimSpace(s)
-	if len(historyLines) == 0 || historyLines[len(historyLines)-1] != s {
-		historyLines = append(historyLines, s)
-		if len(historyLines) > 500 {
-			historyLines = historyLines[len(historyLines)-500:]
-		}
-		if err := saveToHistory(*historyFile, historyLines); err != nil {
-			fatalf("cannot save query history: %s", err)
-		}
-	}
-	if err := rl.SaveToHistory(s); err != nil {
-		fatalf("cannot update query history: %s", err)
-	}
-	return historyLines
-}
-
-func loadFromHistory(filePath string) ([]string, error) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	linesQuoted := strings.Split(string(data), "\n")
-	lines := make([]string, 0, len(linesQuoted))
-	i := 0
-	for _, lineQuoted := range linesQuoted {
-		i++
-		if lineQuoted == "" {
-			continue
-		}
-		line, err := strconv.Unquote(lineQuoted)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse line #%d at %s: %w; line: [%s]", i, filePath, err, line)
-		}
-		lines = append(lines, line)
-	}
-	return lines, nil
-}
-
-func saveToHistory(filePath string, lines []string) error {
-	linesQuoted := make([]string, len(lines))
-	for i, line := range lines {
-		lineQuoted := strconv.Quote(line)
-		linesQuoted[i] = lineQuoted
-	}
-	data := strings.Join(linesQuoted, "\n")
-	return os.WriteFile(filePath, []byte(data), 0600)
-}
-
-func isQuitCommand(s string) bool {
-	switch s {
-	case `\q`, "q", "quit", "exit":
-		return true
-	default:
-		return false
-	}
-}
-
-func isHelpCommand(s string) bool {
-	switch s {
-	case `\h`, "h", "help", "?":
-		return true
-	default:
-		return false
-	}
-}
-
-func printCommandsHelp(w io.Writer) {
-	fmt.Fprintf(w, "%s", `Available commands:
-\q - quit
-\h - show this help
-\s - singleline json output mode
-\m - multiline json output mode
-\c - compact output mode
-\logfmt - logfmt output mode
-\wrap_long_lines - toggles wrapping long lines
-\enable_colors - enable ANSI colors in compact output mode
-\disable_colors - disable ANSI colors in compact output mode
-\tail <query> - live tail <query> results
-
-See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
-`)
-}
-
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, disableColors, wrapLongLines bool) {
-	if strings.HasPrefix(qStr, `\tail `) {
-		tailQuery(ctx, output, qStr, outputMode)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, *datasourceURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if err := readWithLess(respBody, disableColors, wrapLongLines); err != nil {
-		fmt.Fprintf(output, "error when reading query response: %s\n", err)
-		return
-	}
-}
-
-func tailQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
-	qStr = strings.TrimPrefix(qStr, `\tail `)
-	qURL, err := getTailURL()
-	if err != nil {
-		fmt.Fprintf(output, "%s\n", err)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, qURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if _, err := io.Copy(output, respBody); err != nil {
-		if !errors.Is(err, context.Canceled) && !isErrPipe(err) {
-			fmt.Fprintf(output, "error when live tailing query response: %s\n", err)
-		}
-		fmt.Fprintf(output, "\n")
-		return
-	}
-}
-
-func getTailURL() (string, error) {
-	if *tailURL != "" {
-		return *tailURL, nil
-	}
-
-	u, err := url.Parse(*datasourceURL)
-	if err != nil {
-		return "", fmt.Errorf("cannot parse -datasource.url=%q: %w", *datasourceURL, err)
-	}
-	if !strings.HasSuffix(u.Path, "/query") {
-		return "", fmt.Errorf("cannot find /query suffix in -datasource.url=%q", *datasourceURL)
-	}
-	u.Path = u.Path[:len(u.Path)-len("/query")] + "/tail"
-	return u.String(), nil
-}
-
-func getQueryResponse(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, qURL string) io.ReadCloser {
-	// Parse the query and convert it to canonical view.
-	qStr = strings.TrimSuffix(qStr, ";")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		fmt.Fprintf(output, "cannot parse query: %s\n", err)
-		return nil
-	}
-	qStr = q.String()
-	fmt.Fprintf(output, "executing [%s]...", qStr)
-
-	// Prepare HTTP request for qURL
-	args := make(url.Values)
-	args.Set("query", qStr)
-	data := strings.NewReader(args.Encode())
-
-	req, err := http.NewRequestWithContext(ctx, "POST", qURL, data)
-	if err != nil {
-		panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	for _, h := range headers {
-		req.Header.Set(h.Name, h.Value)
-	}
-	req.Header.Set("AccountID", strconv.Itoa(*accountID))
-	req.Header.Set("ProjectID", strconv.Itoa(*projectID))
-
-	// Execute HTTP request at qURL
-	startTime := time.Now()
-	resp, err := httpClient.Do(req)
-	queryDuration := time.Since(startTime)
-	fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
-	if err != nil {
-		if errors.Is(err, context.Canceled) {
-			fmt.Fprintf(output, "\n")
-		} else {
-			fmt.Fprintf(output, "cannot execute query: %s\n", err)
-		}
-		return nil
-	}
-
-	// Verify response code
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			body = []byte(fmt.Sprintf("cannot read response body: %s", err))
-		}
-		fmt.Fprintf(output, "unexpected status code: %d; response body:\n%s\n", resp.StatusCode, body)
-		return nil
-	}
-
-	// Prettify the response body
-	jp := newJSONPrettifier(resp.Body, outputMode)
-
-	return jp
-}
-
-var httpClient = &http.Client{}
-
-var headers []headerEntry
-
-type headerEntry struct {
-	Name  string
-	Value string
-}
-
-func parseHeaders(a []string) ([]headerEntry, error) {
-	hes := make([]headerEntry, len(a))
-	for i, s := range a {
-		a := strings.SplitN(s, ":", 2)
-		if len(a) != 2 {
-			return nil, fmt.Errorf("cannot parse header=%q; it must contain at least one ':'; for example, 'Cookie: foo'", s)
-		}
-		hes[i] = headerEntry{
-			Name:  strings.TrimSpace(a[0]),
-			Value: strings.TrimSpace(a[1]),
-		}
-	}
-	return hes, nil
-}
-
-func fatalf(format string, args ...any) {
-	fmt.Fprintf(os.Stderr, format+"\n", args...)
-	os.Exit(1)
-}
-
-func usage() {
-	const s = `
-vlogscli is a command-line tool for querying VictoriaLogs.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/querying/vlogscli/
-`
-	flagutil.Usage(s)
-}

app/vlogscli/multiarch/Dockerfile

@@ -1,11 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-ENTRYPOINT ["/vlogscli-prod"]
-ARG TARGETARCH
-COPY vlogscli-linux-${TARGETARCH}-prod ./vlogscli-prod

app/vlogsgenerator/Makefile

@@ -1,7 +0,0 @@
-# All these commands must run from repository root.
-
-vlogsgenerator:
-	APP_NAME=vlogsgenerator $(MAKE) app-local
-
-vlogsgenerator-race:
-	APP_NAME=vlogsgenerator RACE=-race $(MAKE) app-local

app/vlogsgenerator/README.md

@@ -1,158 +1 @@
-# vlogsgenerator
-
-Logs generator for [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-## How to build vlogsgenerator?
-
-Run `make vlogsgenerator` from the repository root. This builds `bin/vlogsgenerator` binary.
-
-## How run vlogsgenerator?
-
-`vlogsgenerator` generates logs in [JSON line format](https://jsonlines.org/) suitable for the ingestion
-via [`/insert/jsonline` endpoint at VictoriaLogs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api).
-
-By default it writes the generated logs into `stdout`. For example, the following command writes generated logs to `stdout`:
-
-```
-bin/vlogsgenerator
-```
-
-It is possible to redirect the generated logs to file. For example, the following command writes the generated logs to `logs.json` file:
-
-```
-bin/vlogsgenerator > logs.json
-```
-
-The generated logs at `logs.json` file can be inspected with the following command:
-
-```
-head logs.json | jq .
-```
-
-Below is an example output:
-
-```json
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 8 and worker 0; ip=185.69.136.129; uuid=b4fe8f1a-c93c-dea3-ba11-5b9f0509291e; u64=8996587920687045253",
-  "host": "host_8",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 8",
-  "const_1": "some value 1 8",
-  "const_2": "some value 2 8",
-  "var_0": "some value 0 12752539384823438260",
-  "dict_0": "warn",
-  "dict_1": "info",
-  "u8_0": "6",
-  "u16_0": "35202",
-  "u32_0": "1964973739",
-  "u64_0": "4810489083243239145",
-  "float_0": "1.868",
-  "ip_0": "250.34.75.125",
-  "timestamp_0": "1799-03-16T01:34:18.311Z",
-  "json_0": "{\"foo\":\"bar_3\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":32}"
-}
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 9 and worker 0; ip=164.244.254.194; uuid=7e8373b1-ce0d-1ce7-8e96-4bcab8955598; u64=13949903463741076522",
-  "host": "host_9",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 9",
-  "const_1": "some value 1 9",
-  "const_2": "some value 2 9",
-  "var_0": "some value 0 5371555382075206134",
-  "dict_0": "INFO",
-  "dict_1": "FATAL",
-  "u8_0": "219",
-  "u16_0": "31459",
-  "u32_0": "3918836777",
-  "u64_0": "6593354256620219850",
-  "float_0": "1.085",
-  "ip_0": "253.151.88.158",
-  "timestamp_0": "2042-10-05T16:42:57.082Z",
-  "json_0": "{\"foo\":\"bar_5\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":27}"
-}
-```
-
-The `run_id` field uniquely identifies every `vlogsgenerator` invocation.
-
-### How to write logs to VictoriaLogs?
-
-The generated logs can be written directly to VictoriaLogs by passing the address of [`/insert/jsonline` endpoint](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api)
-to `-addr` command-line flag. For example, the following command writes the generated logs to VictoriaLogs running at `localhost`:
-
-```
-bin/vlogsgenerator -addr=http://localhost:9428/insert/jsonline
-```
-
-### Configuration
-
-`vlogsgenerator` accepts various command-line flags, which can be used for configuring the number and the shape of the generated logs.
-These flags can be inspected by running `vlogsgenerator -help`. Below are the most interesting flags:
-
-* `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to generate.
-* `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
-
-The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
-
-For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
-[log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields), where every logs stream contains `10_000` log entries,
-and writes them to `http://localhost:9428/insert/jsonline`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-### Churn rate
-
-It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
-by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
-logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
-but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -totalStreams=1_000 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-In this case the total number of generated logs equals to `-totalStreams` * `-logsPerStream` = `10_000_000`.
-
-### Benchmark tuning
-
-By default `vlogsgenerator` generates and writes logs by a single worker. This may limit the maximum data ingestion rate during benchmarks.
-The number of workers can be changed via `-workers` command-line flag. For example, the following command generates and writes logs with `16` workers:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -workers=16
-```
-
-### Output statistics
-
-Every 10 seconds `vlogsgenerator` writes statistics about the generated logs into `stderr`. The frequency of the generated statistics can be adjusted via `-statInterval` command-line flag.
-For example, the following command writes statistics every 2 seconds:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -statInterval=2s
-```
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlogsgenerator/main.go

@@ -1,354 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"math"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"os"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
-	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
-
-	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
-	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
-	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
-	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
-		"during data generation")
-	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
-	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constant values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	i64FieldsPerLog = flag.Int("i64FieldsPerLog", 1, "The number of fields with int64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-
-	statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	var remoteWriteURL *url.URL
-	if *addr != "stdout" {
-		urlParsed, err := url.Parse(*addr)
-		if err != nil {
-			logger.Fatalf("cannot parse -addr=%q: %s", *addr, err)
-		}
-		qs, err := url.ParseQuery(urlParsed.RawQuery)
-		if err != nil {
-			logger.Fatalf("cannot parse query string in -addr=%q: %w", *addr, err)
-		}
-		qs.Set("_stream_fields", "host,worker_id")
-		urlParsed.RawQuery = qs.Encode()
-		remoteWriteURL = urlParsed
-	}
-
-	if start.nsec >= end.nsec {
-		logger.Fatalf("-start=%s must be smaller than -end=%s", start, end)
-	}
-	if *activeStreams <= 0 {
-		logger.Fatalf("-activeStreams must be bigger than 0; got %d", *activeStreams)
-	}
-	if *logsPerStream <= 0 {
-		logger.Fatalf("-logsPerStream must be bigger than 0; got %d", *logsPerStream)
-	}
-	if *totalStreams < *activeStreams {
-		*totalStreams = *activeStreams
-	}
-
-	cfg := &workerConfig{
-		url:           remoteWriteURL,
-		activeStreams: *activeStreams,
-		totalStreams:  *totalStreams,
-	}
-
-	// divide total and active streams among workers
-	if *workers <= 0 {
-		logger.Fatalf("-workers must be bigger than 0; got %d", *workers)
-	}
-	if *workers > *activeStreams {
-		logger.Fatalf("-workers=%d cannot exceed -activeStreams=%d", *workers, *activeStreams)
-	}
-	cfg.activeStreams /= *workers
-	cfg.totalStreams /= *workers
-
-	logger.Infof("start -workers=%d workers for ingesting -logsPerStream=%d log entries per each -totalStreams=%d (-activeStreams=%d) on a time range -start=%s, -end=%s to -addr=%s",
-		*workers, *logsPerStream, *totalStreams, *activeStreams, toRFC3339(start.nsec), toRFC3339(end.nsec), *addr)
-
-	startTime := time.Now()
-	var wg sync.WaitGroup
-	for i := 0; i < *workers; i++ {
-		wg.Add(1)
-		go func(workerID int) {
-			defer wg.Done()
-			generateAndPushLogs(cfg, workerID)
-		}(i)
-	}
-
-	go func() {
-		prevEntries := uint64(0)
-		prevBytes := uint64(0)
-		ticker := time.NewTicker(*statInterval)
-		for range ticker.C {
-			currEntries := logEntriesCount.Load()
-			deltaEntries := currEntries - prevEntries
-			rateEntries := float64(deltaEntries) / statInterval.Seconds()
-
-			currBytes := bytesGenerated.Load()
-			deltaBytes := currBytes - prevBytes
-			rateBytes := float64(deltaBytes) / statInterval.Seconds()
-			logger.Infof("generated %dK log entries (%dK total) at %.0fK entries/sec, %dMB (%dMB total) at %.0fMB/sec",
-				deltaEntries/1e3, currEntries/1e3, rateEntries/1e3, deltaBytes/1e6, currBytes/1e6, rateBytes/1e6)
-
-			prevEntries = currEntries
-			prevBytes = currBytes
-		}
-	}()
-
-	wg.Wait()
-
-	dSecs := time.Since(startTime).Seconds()
-	currEntries := logEntriesCount.Load()
-	currBytes := bytesGenerated.Load()
-	rateEntries := float64(currEntries) / dSecs
-	rateBytes := float64(currBytes) / dSecs
-	logger.Infof("ingested %dK log entries (%dMB) in %.3f seconds; avg ingestion rate: %.0fK entries/sec, %.0fMB/sec", currEntries/1e3, currBytes/1e6, dSecs, rateEntries/1e3, rateBytes/1e6)
-}
-
-var logEntriesCount atomic.Uint64
-
-var bytesGenerated atomic.Uint64
-
-type workerConfig struct {
-	url           *url.URL
-	activeStreams int
-	totalStreams  int
-}
-
-type statWriter struct {
-	w io.Writer
-}
-
-func (sw *statWriter) Write(p []byte) (int, error) {
-	bytesGenerated.Add(uint64(len(p)))
-	return sw.w.Write(p)
-}
-
-func generateAndPushLogs(cfg *workerConfig, workerID int) {
-	pr, pw := io.Pipe()
-	sw := &statWriter{
-		w: pw,
-	}
-
-	// The 1MB write buffer increases data ingestion performance by reducing the number of send() syscalls
-	bw := bufio.NewWriterSize(sw, 1024*1024)
-
-	doneCh := make(chan struct{})
-	go func() {
-		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
-		_ = bw.Flush()
-		_ = pw.Close()
-		close(doneCh)
-	}()
-
-	if cfg.url == nil {
-		_, err := io.Copy(os.Stdout, pr)
-		if err != nil {
-			logger.Fatalf("unexpected error when writing logs to stdout: %s", err)
-		}
-		return
-	}
-
-	req, err := http.NewRequest("POST", cfg.url.String(), pr)
-	if err != nil {
-		logger.Fatalf("cannot create request to %q: %s", cfg.url, err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 != 2 {
-		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
-	}
-
-	// Wait until all the generateLogs goroutine is finished.
-	<-doneCh
-}
-
-func generateLogs(bw *bufio.Writer, workerID, activeStreams, totalStreams int) {
-	streamLifetime := int64(float64(end.nsec-start.nsec) * (float64(activeStreams) / float64(totalStreams)))
-	streamStep := int64(float64(end.nsec-start.nsec) / float64(totalStreams-activeStreams+1))
-	step := streamLifetime / (*logsPerStream - 1)
-
-	currNsec := start.nsec
-	for currNsec < end.nsec {
-		firstStreamID := int((currNsec - start.nsec) / streamStep)
-		generateLogsAtTimestamp(bw, workerID, currNsec, firstStreamID, activeStreams)
-		currNsec += step
-	}
-}
-
-var runID = toUUID(rand.Uint64(), rand.Uint64())
-
-func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStreamID, activeStreams int) {
-	streamID := firstStreamID
-	timeStr := toRFC3339(ts)
-	for i := 0; i < activeStreams; i++ {
-		ip := toIPv4(rand.Uint32())
-		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
-			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
-		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
-		for j := 0; j < *constFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"const_%d":"some value %d %d"`, j, j, streamID)
-		}
-		for j := 0; j < *varFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"var_%d":"some value %d %d"`, j, j, rand.Uint64())
-		}
-		for j := 0; j < *dictFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"dict_%d":"%s"`, j, dictValues[rand.Intn(len(dictValues))])
-		}
-		for j := 0; j < *u8FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u8_%d":"%d"`, j, uint8(rand.Uint32()))
-		}
-		for j := 0; j < *u16FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u16_%d":"%d"`, j, uint16(rand.Uint32()))
-		}
-		for j := 0; j < *u32FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u32_%d":"%d"`, j, rand.Uint32())
-		}
-		for j := 0; j < *u64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
-		}
-		for j := 0; j < *i64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"i64_%d":"%d"`, j, int64(rand.Uint64()))
-		}
-		for j := 0; j < *floatFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
-		}
-		for j := 0; j < *ipFieldsPerLog; j++ {
-			ip := toIPv4(rand.Uint32())
-			fmt.Fprintf(bw, `,"ip_%d":"%s"`, j, ip)
-		}
-		for j := 0; j < *timestampFieldsPerLog; j++ {
-			timestamp := toISO8601(int64(rand.Uint64()))
-			fmt.Fprintf(bw, `,"timestamp_%d":"%s"`, j, timestamp)
-		}
-		for j := 0; j < *jsonFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"json_%d":"{\"foo\":\"bar_%d\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":%d}"`, j, rand.Intn(10), rand.Intn(100))
-		}
-		fmt.Fprintf(bw, "}\n")
-
-		logEntriesCount.Add(1)
-		streamID++
-	}
-}
-
-var dictValues = []string{
-	"debug",
-	"info",
-	"warn",
-	"error",
-	"fatal",
-	"ERROR",
-	"FATAL",
-	"INFO",
-}
-
-func newTimeFlag(name, defaultValue, description string) *timeFlag {
-	var tf timeFlag
-	if err := tf.Set(defaultValue); err != nil {
-		logger.Panicf("invalid defaultValue=%q for flag %q: %w", defaultValue, name, err)
-	}
-	flag.Var(&tf, name, description)
-	return &tf
-}
-
-type timeFlag struct {
-	s    string
-	nsec int64
-}
-
-func (tf *timeFlag) Set(s string) error {
-	msec, err := timeutil.ParseTimeMsec(s)
-	if err != nil {
-		return fmt.Errorf("cannot parse time from %q: %w", s, err)
-	}
-	tf.s = s
-	tf.nsec = msec * 1e6
-	return nil
-}
-
-func (tf *timeFlag) String() string {
-	return tf.s
-}
-
-func toRFC3339(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format(time.RFC3339Nano)
-}
-
-func toISO8601(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format("2006-01-02T15:04:05.000Z")
-}
-
-func toIPv4(n uint32) string {
-	dst := make([]byte, 0, len("255.255.255.255"))
-	dst = marshalUint64(dst, uint64(n>>24))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>16)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>8)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64(n&0xff))
-	return string(dst)
-}
-
-func toUUID(a, b uint64) string {
-	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", a&(1<<32-1), (a>>32)&(1<<16-1), (a >> 48), b&(1<<16-1), b>>16)
-}
-
-// marshalUint64 appends string representation of n to dst and returns the result.
-func marshalUint64(dst []byte, n uint64) []byte {
-	return strconv.AppendUint(dst, n, 10)
-}

app/vlselect/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlselect/internalselect/internalselect.go

@@ -1,316 +0,0 @@
-package internalselect
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-)
-
-// RequestHandler processes requests to /internal/select/*
-func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-
-	path := r.URL.Path
-	rh := requestHandlers[path]
-	if rh == nil {
-		httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
-		return
-	}
-
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
-	if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		// The return is skipped intentionally in order to track the duration of failed queries.
-	}
-	metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
-}
-
-var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
-	"/internal/select/query":               processQueryRequest,
-	"/internal/select/field_names":         processFieldNamesRequest,
-	"/internal/select/field_values":        processFieldValuesRequest,
-	"/internal/select/stream_field_names":  processStreamFieldNamesRequest,
-	"/internal/select/stream_field_values": processStreamFieldValuesRequest,
-	"/internal/select/streams":             processStreamsRequest,
-	"/internal/select/stream_ids":          processStreamIDsRequest,
-}
-
-func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	var wLock sync.Mutex
-	var dataLenBuf []byte
-
-	sendBuf := func(bb *bytesutil.ByteBuffer) error {
-		if len(bb.B) == 0 {
-			return nil
-		}
-
-		data := bb.B
-		if !cp.DisableCompression {
-			bufLen := len(bb.B)
-			bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
-			data = bb.B[bufLen:]
-		}
-
-		wLock.Lock()
-		dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
-		_, err := w.Write(dataLenBuf)
-		if err == nil {
-			_, err = w.Write(data)
-		}
-		wLock.Unlock()
-
-		// Reset the sent buf
-		bb.Reset()
-
-		return err
-	}
-
-	var bufs atomicutil.Slice[bytesutil.ByteBuffer]
-
-	var errGlobalLock sync.Mutex
-	var errGlobal error
-
-	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
-		if errGlobal != nil {
-			return
-		}
-
-		bb := bufs.Get(workerID)
-
-		bb.B = db.Marshal(bb.B)
-
-		if len(bb.B) < 1024*1024 {
-			// Fast path - the bb is too small to be sent to the client yet.
-			return
-		}
-
-		// Slow path - the bb must be sent to the client.
-		if err := sendBuf(bb); err != nil {
-			errGlobalLock.Lock()
-			if errGlobal != nil {
-				errGlobal = err
-			}
-			errGlobalLock.Unlock()
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
-		return err
-	}
-	if errGlobal != nil {
-		return errGlobal
-	}
-
-	// Send the remaining data
-	for _, bb := range bufs.All() {
-		if err := sendBuf(bb); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streams, cp.DisableCompression)
-}
-
-func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
-}
-
-type commonParams struct {
-	TenantIDs []logstorage.TenantID
-	Query     *logstorage.Query
-
-	DisableCompression bool
-}
-
-func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
-	version := r.FormValue("version")
-	if version != expectedProtocolVersion {
-		return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
-	}
-
-	tenantIDsStr := r.FormValue("tenant_ids")
-	tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
-	}
-
-	timestamp, err := getInt64FromRequest(r, "timestamp")
-	if err != nil {
-		return nil, err
-	}
-
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
-	}
-
-	s := r.FormValue("disable_compression")
-	disableCompression, err := strconv.ParseBool(s)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
-	}
-
-	cp := &commonParams{
-		TenantIDs: tenantIDs,
-		Query:     q,
-
-		DisableCompression: disableCompression,
-	}
-	return cp, nil
-}
-
-func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
-	var b []byte
-	for i := range vhs {
-		b = vhs[i].Marshal(b)
-	}
-
-	if !disableCompression {
-		b = zstd.CompressLevel(nil, b, 1)
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	if _, err := w.Write(b); err != nil {
-		return fmt.Errorf("cannot send response to the client: %w", err)
-	}
-
-	return nil
-}
-
-func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
-	s := r.FormValue(argName)
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
-	}
-	return n, nil
-}

app/vlselect/logsql/facets_response.qtpl

@@ -1,49 +0,0 @@
-{% import (
-	"slices"
-) %}
-
-{% stripspace %}
-
-{% func FacetsResponse(m map[string][]facetEntry) %}
-{
-	{% code
-		sortedKeys := make([]string, 0, len(m))
-		for k := range m {
-			sortedKeys = append(sortedKeys, k)
-		}
-		slices.Sort(sortedKeys)
-	%}
-	"facets":[
-		{% if len(sortedKeys) > 0 %}
-			{%= facetsLine(m, sortedKeys[0]) %}
-			{% for _, k := range sortedKeys[1:] %}
-				,{%= facetsLine(m, k) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetsLine(m map[string][]facetEntry, k string) %}
-{
-	"field_name":{%q= k %},
-	"values":[
-		{% code	fes := m[k] %}
-		{% if len(fes) > 0 %}
-			{%= facetLine(fes[0]) %}
-			{% for _, fe := range fes[1:] %}
-				,{%= facetLine(fe) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetLine(fe facetEntry) %}
-{
-	"field_value":{%q= fe.value %},
-	"hits":{%s= fe.hits %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/facets_response.qtpl.go

@@ -1,178 +0,0 @@
-// Code generated by qtc from "facets_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-import (
-	"slices"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-func StreamFacetsResponse(qw422016 *qt422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:7
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/facets_response.qtpl:10
-	sortedKeys := make([]string, 0, len(m))
-	for k := range m {
-		sortedKeys = append(sortedKeys, k)
-	}
-	slices.Sort(sortedKeys)
-
-//line app/vlselect/logsql/facets_response.qtpl:15
-	qw422016.N().S(`"facets":[`)
-//line app/vlselect/logsql/facets_response.qtpl:17
-	if len(sortedKeys) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:18
-		streamfacetsLine(qw422016, m, sortedKeys[0])
-//line app/vlselect/logsql/facets_response.qtpl:19
-		for _, k := range sortedKeys[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:19
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:20
-			streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:21
-		}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func WriteFacetsResponse(qq422016 qtio422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	StreamFacetsResponse(qw422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func FacetsResponse(m map[string][]facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:25
-	WriteFacetsResponse(qb422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:27
-func streamfacetsLine(qw422016 *qt422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:27
-	qw422016.N().S(`{"field_name":`)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().Q(k)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().S(`,"values":[`)
-//line app/vlselect/logsql/facets_response.qtpl:31
-	fes := m[k]
-
-//line app/vlselect/logsql/facets_response.qtpl:32
-	if len(fes) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:33
-		streamfacetLine(qw422016, fes[0])
-//line app/vlselect/logsql/facets_response.qtpl:34
-		for _, fe := range fes[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:34
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:35
-			streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:36
-		}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func writefacetsLine(qq422016 qtio422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func facetsLine(m map[string][]facetEntry, k string) string {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:40
-	writefacetsLine(qb422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:42
-func streamfacetLine(qw422016 *qt422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:42
-	qw422016.N().S(`{"field_value":`)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().Q(fe.value)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().S(`,"hits":`)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(fe.hits)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func writefacetLine(qq422016 qtio422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func facetLine(fe facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:47
-	writefacetLine(qb422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:47
-}

app/vlselect/logsql/hits_response.qtpl

@@ -1,70 +0,0 @@
-{% import (
-	"slices"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// FieldsForHits formats labels for /select/logsql/hits response
-{% func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) %}
-{
-	{% if len(columns) > 0 %}
-		{%q= columns[0].Name %}:{%q= columns[0].Values[rowIdx] %}
-		{% for _, c := range columns[1:] %}
-			,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-		{% endfor %}
-	{% endif %}
-}
-{% endfunc %}
-
-{% func HitsSeries(m map[string]*hitsSeries) %}
-{
-	{% code
-		sortedKeys := make([]string, 0, len(m))
-		for k := range m {
-			sortedKeys = append(sortedKeys, k)
-		}
-		slices.Sort(sortedKeys)
-	%}
-	"hits":[
-		{% if len(sortedKeys) > 0 %}
-			{%= hitsSeriesLine(m, sortedKeys[0]) %}
-			{% for _, k := range sortedKeys[1:] %}
-				,{%= hitsSeriesLine(m, k) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func hitsSeriesLine(m map[string]*hitsSeries, k string) %}
-{
-	{% code
-		hs := m[k]
-		hs.sort()
-		timestamps := hs.timestamps
-		hits := hs.hits
-	%}
-	"fields":{%s= k %},
-	"timestamps":[
-		{% if len(timestamps) > 0 %}
-			{%q= timestamps[0] %}
-			{% for _, ts := range timestamps[1:] %}
-				,{%q= ts %}
-			{% endfor %}
-		{% endif %}
-	],
-	"values":[
-		{% if len(hits) > 0 %}
-			{%dul= hits[0] %}
-			{% for _, v := range hits[1:] %}
-				,{%dul= v %}
-			{% endfor %}
-		{% endif %}
-	],
-	"total":{%dul= hs.hitsTotal %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/hits_response.qtpl.go

@@ -1,223 +0,0 @@
-// Code generated by qtc from "hits_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/hits_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/hits_response.qtpl:1
-import (
-	"slices"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// FieldsForHits formats labels for /select/logsql/hits response
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-func StreamFieldsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/hits_response.qtpl:10
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:12
-	if len(columns) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().Q(columns[0].Name)
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().Q(columns[0].Values[rowIdx])
-//line app/vlselect/logsql/hits_response.qtpl:14
-		for _, c := range columns[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/hits_response.qtpl:16
-		}
-//line app/vlselect/logsql/hits_response.qtpl:17
-	}
-//line app/vlselect/logsql/hits_response.qtpl:17
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:19
-func WriteFieldsForHits(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	StreamFieldsForHits(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:19
-func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:19
-	WriteFieldsForHits(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:21
-func StreamHitsSeries(qw422016 *qt422016.Writer, m map[string]*hitsSeries) {
-//line app/vlselect/logsql/hits_response.qtpl:21
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:24
-	sortedKeys := make([]string, 0, len(m))
-	for k := range m {
-		sortedKeys = append(sortedKeys, k)
-	}
-	slices.Sort(sortedKeys)
-
-//line app/vlselect/logsql/hits_response.qtpl:29
-	qw422016.N().S(`"hits":[`)
-//line app/vlselect/logsql/hits_response.qtpl:31
-	if len(sortedKeys) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:32
-		streamhitsSeriesLine(qw422016, m, sortedKeys[0])
-//line app/vlselect/logsql/hits_response.qtpl:33
-		for _, k := range sortedKeys[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:33
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:34
-			streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:35
-		}
-//line app/vlselect/logsql/hits_response.qtpl:36
-	}
-//line app/vlselect/logsql/hits_response.qtpl:36
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:39
-func WriteHitsSeries(qq422016 qtio422016.Writer, m map[string]*hitsSeries) {
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	StreamHitsSeries(qw422016, m)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:39
-func HitsSeries(m map[string]*hitsSeries) string {
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:39
-	WriteHitsSeries(qb422016, m)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:41
-func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:41
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:44
-	hs := m[k]
-	hs.sort()
-	timestamps := hs.timestamps
-	hits := hs.hits
-
-//line app/vlselect/logsql/hits_response.qtpl:48
-	qw422016.N().S(`"fields":`)
-//line app/vlselect/logsql/hits_response.qtpl:49
-	qw422016.N().S(k)
-//line app/vlselect/logsql/hits_response.qtpl:49
-	qw422016.N().S(`,"timestamps":[`)
-//line app/vlselect/logsql/hits_response.qtpl:51
-	if len(timestamps) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:52
-		qw422016.N().Q(timestamps[0])
-//line app/vlselect/logsql/hits_response.qtpl:53
-		for _, ts := range timestamps[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:53
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:54
-			qw422016.N().Q(ts)
-//line app/vlselect/logsql/hits_response.qtpl:55
-		}
-//line app/vlselect/logsql/hits_response.qtpl:56
-	}
-//line app/vlselect/logsql/hits_response.qtpl:56
-	qw422016.N().S(`],"values":[`)
-//line app/vlselect/logsql/hits_response.qtpl:59
-	if len(hits) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:60
-		qw422016.N().DUL(hits[0])
-//line app/vlselect/logsql/hits_response.qtpl:61
-		for _, v := range hits[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:61
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:62
-			qw422016.N().DUL(v)
-//line app/vlselect/logsql/hits_response.qtpl:63
-		}
-//line app/vlselect/logsql/hits_response.qtpl:64
-	}
-//line app/vlselect/logsql/hits_response.qtpl:64
-	qw422016.N().S(`],"total":`)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().DUL(hs.hitsTotal)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/hits_response.qtpl:68
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:68
-func writehitsSeriesLine(qq422016 qtio422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:68
-func hitsSeriesLine(m map[string]*hitsSeries, k string) string {
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:68
-	writehitsSeriesLine(qb422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:68
-}

app/vlselect/logsql/logsql.qtpl

@@ -1,32 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// ValuesWithHitsJSON generates JSON from the given values.
-{% func ValuesWithHitsJSON(values []logstorage.ValueWithHits) %}
-{
-	"values":{%= valuesWithHitsJSONArray(values) %}
-}
-{% endfunc %}
-
-{% func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) %}
-[
-	{% if len(values) > 0 %}
-		{%= valueWithHitsJSON(values[0]) %}
-		{% for _, v := range values[1:] %}
-			,{%= valueWithHitsJSON(v) %}
-		{% endfor %}
-	{% endif %}
-]
-{% endfunc %}
-
-{% func valueWithHitsJSON(v logstorage.ValueWithHits) %}
-{
-	"value":{%q= v.Value %},
-	"hits":{%dul= v.Hits %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/logsql.qtpl.go

@@ -1,152 +0,0 @@
-// Code generated by qtc from "logsql.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/logsql.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/logsql.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ValuesWithHitsJSON generates JSON from the given values.
-
-//line app/vlselect/logsql/logsql.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/logsql.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/logsql.qtpl:8
-func StreamValuesWithHitsJSON(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:8
-	qw422016.N().S(`{"values":`)
-//line app/vlselect/logsql/logsql.qtpl:10
-	streamvaluesWithHitsJSONArray(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:10
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:12
-func WriteValuesWithHitsJSON(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:12
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-	StreamValuesWithHitsJSON(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:12
-func ValuesWithHitsJSON(values []logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:12
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:12
-	WriteValuesWithHitsJSON(qb422016, values)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:14
-func streamvaluesWithHitsJSONArray(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:14
-	qw422016.N().S(`[`)
-//line app/vlselect/logsql/logsql.qtpl:16
-	if len(values) > 0 {
-//line app/vlselect/logsql/logsql.qtpl:17
-		streamvalueWithHitsJSON(qw422016, values[0])
-//line app/vlselect/logsql/logsql.qtpl:18
-		for _, v := range values[1:] {
-//line app/vlselect/logsql/logsql.qtpl:18
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/logsql.qtpl:19
-			streamvalueWithHitsJSON(qw422016, v)
-//line app/vlselect/logsql/logsql.qtpl:20
-		}
-//line app/vlselect/logsql/logsql.qtpl:21
-	}
-//line app/vlselect/logsql/logsql.qtpl:21
-	qw422016.N().S(`]`)
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:23
-func writevaluesWithHitsJSONArray(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:23
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-	streamvaluesWithHitsJSONArray(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:23
-func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:23
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:23
-	writevaluesWithHitsJSONArray(qb422016, values)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:25
-func streamvalueWithHitsJSON(qw422016 *qt422016.Writer, v logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:25
-	qw422016.N().S(`{"value":`)
-//line app/vlselect/logsql/logsql.qtpl:27
-	qw422016.N().Q(v.Value)
-//line app/vlselect/logsql/logsql.qtpl:27
-	qw422016.N().S(`,"hits":`)
-//line app/vlselect/logsql/logsql.qtpl:28
-	qw422016.N().DUL(v.Hits)
-//line app/vlselect/logsql/logsql.qtpl:28
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/logsql.qtpl:30
-}
-
-//line app/vlselect/logsql/logsql.qtpl:30
-func writevalueWithHitsJSON(qq422016 qtio422016.Writer, v logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:30
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-	streamvalueWithHitsJSON(qw422016, v)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-}
-
-//line app/vlselect/logsql/logsql.qtpl:30
-func valueWithHitsJSON(v logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:30
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:30
-	writevalueWithHitsJSON(qb422016, v)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:30
-}

app/vlselect/logsql/logsql_test.go

@@ -1,103 +0,0 @@
-package logsql
-
-import (
-	"testing"
-)
-
-func TestParseExtraFilters_Success(t *testing.T) {
-	f := func(s, resultExpected string) {
-		t.Helper()
-
-		f, err := parseExtraFilters(s)
-		if err != nil {
-			t.Fatalf("unexpected error in parseExtraFilters: %s", err)
-		}
-		result := f.String()
-		if result != resultExpected {
-			t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	f("", "")
-
-	// JSON string
-	f(`{"foo":"bar"}`, `foo:=bar`)
-	f(`{"foo":["bar","baz"]}`, `foo:in(bar,baz)`)
-	f(`{"z":"=b ","c":["d","e,"],"a":[],"_msg":"x"}`, `z:="=b " c:in(d,"e,") =x`)
-
-	// LogsQL filter
-	f(`foobar`, `foobar`)
-	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
-}
-
-func TestParseExtraFilters_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := parseExtraFilters(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// Invalid JSON
-	f(`{"foo"}`)
-	f(`[1,2]`)
-	f(`{"foo":[1]}`)
-
-	// Invliad LogsQL filter
-	f(`foo:(bar`)
-
-	// excess pipe
-	f(`foo | count()`)
-}
-
-func TestParseExtraStreamFilters_Success(t *testing.T) {
-	f := func(s, resultExpected string) {
-		t.Helper()
-
-		f, err := parseExtraStreamFilters(s)
-		if err != nil {
-			t.Fatalf("unexpected error in parseExtraStreamFilters: %s", err)
-		}
-		result := f.String()
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	f("", "")
-
-	// JSON string
-	f(`{"foo":"bar"}`, `{foo="bar"}`)
-	f(`{"foo":["bar","baz"]}`, `{foo=~"bar|baz"}`)
-	f(`{"z":"b","c":["d","e|\""],"a":[],"_msg":"x"}`, `{z="b",c=~"d|e\\|\"",_msg="x"}`)
-
-	// LogsQL filter
-	f(`foobar`, `foobar`)
-	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
-}
-
-func TestParseExtraStreamFilters_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := parseExtraStreamFilters(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// Invalid JSON
-	f(`{"foo"}`)
-	f(`[1,2]`)
-	f(`{"foo":[1]}`)
-
-	// Invliad LogsQL filter
-	f(`foo:(bar`)
-
-	// excess pipe
-	f(`foo | count()`)
-}

app/vlselect/logsql/query_response.qtpl

@@ -1,64 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// JSONRow creates JSON row from the given fields.
-{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-	{% code
-		i := 0
-		for i < len(columns) && columns[i].Values[rowIdx] == "" {
-			i++
-		}
-		columns = columns[i:]
-	%}
-	{% if len(columns) == 0 %}
-		{% return %}
-	{% endif %}
-	{
-	{% code	c := &columns[0] %}
-	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% code columns = columns[1:] %}
-	{% for colIdx := range columns %}
-		{% code
-			c := &columns[colIdx]
-			v := c.Values[rowIdx]
-		%}
-		{% if v == "" %}
-			{% continue %}
-		{% endif %}
-		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% endfor %}
-	}{% newline %}
-{% endfunc %}
-
-// JSONRows prints formatted rows
-{% func JSONRows(rows [][]logstorage.Field) %}
-	{% if len(rows) == 0 %}
-		{% return %}
-	{% endif %}
-	{% for _, fields := range rows %}
-		{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
-		{% if len(fields) == 0 %}
-			{% continue %}
-		{% endif %}
-		{
-		{% if len(fields) > 0 %}
-			{% code
-				f := fields[0]
-				fields = fields[1:]
-			%}
-			{%q= f.Name %}:{%q= f.Value %}
-			{% for _, f := range fields %}
-				{% if f.Value == "" %}
-					{% continue %}
-				{% endif %}
-				,{%q= f.Name %}:{%q= f.Value %}
-			{% endfor %}
-		{% endif %}
-		}{% newline %}
-	{% endfor %}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/query_response.qtpl.go

@@ -1,201 +0,0 @@
-// Code generated by qtc from "query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/query_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/query_response.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// JSONRow creates JSON row from the given fields.
-
-//line app/vlselect/logsql/query_response.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:10
-	i := 0
-	for i < len(columns) && columns[i].Values[rowIdx] == "" {
-		i++
-	}
-	columns = columns[i:]
-
-//line app/vlselect/logsql/query_response.qtpl:16
-	if len(columns) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:17
-		return
-//line app/vlselect/logsql/query_response.qtpl:18
-	}
-//line app/vlselect/logsql/query_response.qtpl:18
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:20
-	c := &columns[0]
-
-//line app/vlselect/logsql/query_response.qtpl:21
-	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:21
-	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:21
-	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:22
-	columns = columns[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:23
-	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:25
-		c := &columns[colIdx]
-		v := c.Values[rowIdx]
-
-//line app/vlselect/logsql/query_response.qtpl:28
-		if v == "" {
-//line app/vlselect/logsql/query_response.qtpl:29
-			continue
-//line app/vlselect/logsql/query_response.qtpl:30
-		}
-//line app/vlselect/logsql/query_response.qtpl:30
-		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:31
-		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:31
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:31
-		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:32
-	}
-//line app/vlselect/logsql/query_response.qtpl:32
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:33
-	qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/query_response.qtpl:34
-func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:34
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:34
-	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/query_response.qtpl:34
-func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:34
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:34
-	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:34
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:34
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:34
-}
-
-// JSONRows prints formatted rows
-
-//line app/vlselect/logsql/query_response.qtpl:37
-func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:38
-	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:39
-		return
-//line app/vlselect/logsql/query_response.qtpl:40
-	}
-//line app/vlselect/logsql/query_response.qtpl:41
-	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:42
-		fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
-
-//line app/vlselect/logsql/query_response.qtpl:43
-		if len(fields) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:44
-			continue
-//line app/vlselect/logsql/query_response.qtpl:45
-		}
-//line app/vlselect/logsql/query_response.qtpl:45
-		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:47
-		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:49
-			f := fields[0]
-			fields = fields[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:52
-			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:52
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:52
-			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:53
-			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:54
-				if f.Value == "" {
-//line app/vlselect/logsql/query_response.qtpl:55
-					continue
-//line app/vlselect/logsql/query_response.qtpl:56
-				}
-//line app/vlselect/logsql/query_response.qtpl:56
-				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:57
-				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:57
-				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:57
-				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:58
-			}
-//line app/vlselect/logsql/query_response.qtpl:59
-		}
-//line app/vlselect/logsql/query_response.qtpl:59
-		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:60
-		qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:61
-	}
-//line app/vlselect/logsql/query_response.qtpl:62
-}
-
-//line app/vlselect/logsql/query_response.qtpl:62
-func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:62
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:62
-	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:62
-}
-
-//line app/vlselect/logsql/query_response.qtpl:62
-func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:62
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:62
-	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:62
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:62
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:62
-}

app/vlselect/logsql/stats_query_range_response.qtpl

@@ -1,52 +0,0 @@
-{% stripspace %}
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-{% func StatsQueryRangeResponse(rows []*statsSeries) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"matrix",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsSeries(rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsSeries(rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsSeries(ss *statsSeries) %}
-{
-	"metric":{
-		"__name__":{%q= ss.Name %}
-		{% if len(ss.Labels) > 0 %}
-			{% for _, label := range ss.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"values":[
-		{% code points := ss.Points %}
-		{% if len(points) > 0 %}
-			{%= formatStatsPoint(&points[0]) %}
-			{% code points = points[1:] %}
-			{% for i := range points %}
-				,{%= formatStatsPoint(&points[i]) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func formatStatsPoint(p *statsPoint) %}
-[
-	{%f= float64(p.Timestamp)/1e9 %},
-	{%q= p.Value %}
-]
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_range_response.qtpl.go

@@ -1,188 +0,0 @@
-// Code generated by qtc from "stats_query_range_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-func StreamStatsQueryRangeResponse(qw422016 *qt422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:11
-		streamformatStatsSeries(qw422016, rows[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:14
-			streamformatStatsSeries(qw422016, rows[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func WriteStatsQueryRangeResponse(qq422016 qtio422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	StreamStatsQueryRangeResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func StatsQueryRangeResponse(rows []*statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	WriteStatsQueryRangeResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-func streamformatStatsSeries(qw422016 *qt422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:25
-	qw422016.N().Q(ss.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:26
-	if len(ss.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-		for _, label := range ss.Labels {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	qw422016.N().S(`},"values":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:33
-	points := ss.Points
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:34
-	if len(points) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:35
-		streamformatStatsPoint(qw422016, &points[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:36
-		points = points[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-		for i := range points {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:38
-			streamformatStatsPoint(qw422016, &points[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:39
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func writeformatStatsSeries(qq422016 qtio422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	streamformatStatsSeries(qw422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func formatStatsSeries(ss *statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	writeformatStatsSeries(qb422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-func streamformatStatsPoint(qw422016 *qt422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-	qw422016.N().S(`[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().F(float64(p.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().Q(p.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().S(`]`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func writeformatStatsPoint(qq422016 qtio422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	streamformatStatsPoint(qw422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func formatStatsPoint(p *statsPoint) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	writeformatStatsPoint(qb422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}

app/vlselect/logsql/stats_query_response.qtpl

@@ -1,36 +0,0 @@
-{% stripspace %}
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-{% func StatsQueryResponse(rows []statsRow) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"vector",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsRow(&rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsRow(&rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsRow(r *statsRow) %}
-{
-	"metric":{
-		"__name__":{%q= r.Name %}
-		{% if len(r.Labels) > 0 %}
-			{% for _, label := range r.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"value":[{%f= float64(r.Timestamp)/1e9 %},{%q= r.Value %}]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_response.qtpl.go

@@ -1,133 +0,0 @@
-// Code generated by qtc from "stats_query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-func StreamStatsQueryResponse(qw422016 *qt422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:11
-		streamformatStatsRow(qw422016, &rows[0])
-//line app/vlselect/logsql/stats_query_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:14
-			streamformatStatsRow(qw422016, &rows[i])
-//line app/vlselect/logsql/stats_query_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func WriteStatsQueryResponse(qq422016 qtio422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	StreamStatsQueryResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func StatsQueryResponse(rows []statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	WriteStatsQueryResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-func streamformatStatsRow(qw422016 *qt422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_response.qtpl:25
-	qw422016.N().Q(r.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:26
-	if len(r.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-		for _, label := range r.Labels {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	qw422016.N().S(`},"value":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().F(float64(r.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().Q(r.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func writeformatStatsRow(qq422016 qtio422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	streamformatStatsRow(qw422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func formatStatsRow(r *statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	writeformatStatsRow(qb422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}

app/vlselect/main.go

@@ -1,324 +0,0 @@
-package vlselect
-
-import (
-	"context"
-	"embed"
-	"flag"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/internalselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration")
-	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
-		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden to a smaller value on a per-query basis via 'timeout' query arg")
-
-	disableSelect   = flag.Bool("select.disable", false, "Whether to disable /select/* HTTP endpoints")
-	disableInternal = flag.Bool("internalselect.disable", false, "Whether to disable /internal/select/* HTTP endpoints")
-)
-
-func getDefaultMaxConcurrentRequests() int {
-	n := cgroup.AvailableCPUs()
-	if n <= 4 {
-		n *= 2
-	}
-	if n > 16 {
-		// A single request can saturate all the CPU cores, so there is no sense
-		// in allowing higher number of concurrent requests - they will just contend
-		// for unavailable CPU time.
-		n = 16
-	}
-	return n
-}
-
-// Init initializes vlselect
-func Init() {
-	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
-}
-
-// Stop stops vlselect
-func Stop() {
-}
-
-var concurrencyLimitCh chan struct{}
-
-var (
-	concurrencyLimitReached = metrics.NewCounter(`vl_concurrent_select_limit_reached_total`)
-	concurrencyLimitTimeout = metrics.NewCounter(`vl_concurrent_select_limit_timeout_total`)
-
-	_ = metrics.NewGauge(`vl_concurrent_select_capacity`, func() float64 {
-		return float64(cap(concurrencyLimitCh))
-	})
-	_ = metrics.NewGauge(`vl_concurrent_select_current`, func() float64 {
-		return float64(len(concurrencyLimitCh))
-	})
-)
-
-//go:embed vmui
-var vmuiFiles embed.FS
-
-var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
-
-// RequestHandler handles select requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
-
-	if strings.HasPrefix(path, "/select/") {
-		if *disableSelect {
-			httpserver.Errorf(w, r, "requests to /select/* are disabled with -select.disable command-line flag")
-			return true
-		}
-
-		return selectHandler(w, r, path)
-	}
-
-	if strings.HasPrefix(path, "/internal/select/") {
-		if *disableInternal || *disableSelect {
-			httpserver.Errorf(w, r, "requests to /internal/select/* are disabled with -internalselect.disable or -select.disable command-line flag")
-			return true
-		}
-		internalselect.RequestHandler(r.Context(), w, r)
-		return true
-	}
-
-	return false
-}
-
-func selectHandler(w http.ResponseWriter, r *http.Request, path string) bool {
-	ctx := r.Context()
-
-	if path == "/select/vmui" {
-		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
-		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
-		// is hidden behind vmauth or similar proxy.
-		_ = r.ParseForm()
-		newURL := "vmui/?" + r.Form.Encode()
-		httpserver.Redirect(w, newURL)
-		return true
-	}
-	if strings.HasPrefix(path, "/select/vmui/") {
-		if strings.HasPrefix(path, "/select/vmui/static/") {
-			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
-			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
-			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
-			w.Header().Set("Cache-Control", "max-age=31536000")
-		}
-		r.URL.Path = strings.TrimPrefix(path, "/select")
-		vmuiFileServer.ServeHTTP(w, r)
-		return true
-	}
-
-	if path == "/select/logsql/tail" {
-		logsqlTailRequests.Inc()
-		// Process live tailing request without timeout, since it is OK to run live tailing requests for very long time.
-		// Also do not apply concurrency limit to tail requests, since these limits are intended for non-tail requests.
-		logsql.ProcessLiveTailRequest(ctx, w, r)
-		return true
-	}
-
-	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
-	startTime := time.Now()
-	d := getMaxQueryDuration(r)
-	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
-	defer cancel()
-
-	if !incRequestConcurrency(ctxWithTimeout, w, r) {
-		return true
-	}
-	defer decRequestConcurrency()
-
-	ok := processSelectRequest(ctxWithTimeout, w, r, path)
-	if !ok {
-		return false
-	}
-
-	logRequestErrorIfNeeded(ctxWithTimeout, w, r, startTime)
-	return true
-}
-
-func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http.Request, startTime time.Time) {
-	err := ctx.Err()
-	switch err {
-	case nil:
-		// nothing to do
-	case context.Canceled:
-		// do not log canceled requests, since they are expected and legal.
-	case context.DeadlineExceeded:
-		err = &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
-				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", time.Since(startTime).Seconds(), maxQueryDuration),
-			StatusCode: http.StatusServiceUnavailable,
-		}
-		httpserver.Errorf(w, r, "%s", err)
-	default:
-		httpserver.Errorf(w, r, "unexpected error: %s", err)
-	}
-}
-
-func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {
-	startTime := time.Now()
-	stopCh := ctx.Done()
-	select {
-	case concurrencyLimitCh <- struct{}{}:
-		return true
-	default:
-		// Sleep for a while until giving up. This should resolve short bursts in requests.
-		concurrencyLimitReached.Inc()
-		select {
-		case concurrencyLimitCh <- struct{}{}:
-			return true
-		case <-stopCh:
-			switch ctx.Err() {
-			case context.Canceled:
-				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-				requestURI := httpserver.GetRequestURI(r)
-				logger.Infof("client has canceled the pending request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-					time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			case context.DeadlineExceeded:
-				concurrencyLimitTimeout.Inc()
-				err := &httpserver.ErrorWithStatusCode{
-					Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-						"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-						"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
-						"to pass bigger value to 'timeout' query arg",
-						time.Since(startTime).Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
-					StatusCode: http.StatusServiceUnavailable,
-				}
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return false
-		}
-	}
-}
-
-func decRequestConcurrency() {
-	<-concurrencyLimitCh
-}
-
-func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
-	httpserver.EnableCORS(w, r)
-	startTime := time.Now()
-	switch path {
-	case "/select/logsql/facets":
-		logsqlFacetsRequests.Inc()
-		logsql.ProcessFacetsRequest(ctx, w, r)
-		logsqlFacetsDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/field_names":
-		logsqlFieldNamesRequests.Inc()
-		logsql.ProcessFieldNamesRequest(ctx, w, r)
-		logsqlFieldNamesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/field_values":
-		logsqlFieldValuesRequests.Inc()
-		logsql.ProcessFieldValuesRequest(ctx, w, r)
-		logsqlFieldValuesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/hits":
-		logsqlHitsRequests.Inc()
-		logsql.ProcessHitsRequest(ctx, w, r)
-		logsqlHitsDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/query":
-		logsqlQueryRequests.Inc()
-		logsql.ProcessQueryRequest(ctx, w, r)
-		logsqlQueryDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stats_query":
-		logsqlStatsQueryRequests.Inc()
-		logsql.ProcessStatsQueryRequest(ctx, w, r)
-		logsqlStatsQueryDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stats_query_range":
-		logsqlStatsQueryRangeRequests.Inc()
-		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
-		logsqlStatsQueryRangeDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stream_field_names":
-		logsqlStreamFieldNamesRequests.Inc()
-		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
-		logsqlStreamFieldNamesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stream_field_values":
-		logsqlStreamFieldValuesRequests.Inc()
-		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
-		logsqlStreamFieldValuesDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/stream_ids":
-		logsqlStreamIDsRequests.Inc()
-		logsql.ProcessStreamIDsRequest(ctx, w, r)
-		logsqlStreamIDsDuration.UpdateDuration(startTime)
-		return true
-	case "/select/logsql/streams":
-		logsqlStreamsRequests.Inc()
-		logsql.ProcessStreamsRequest(ctx, w, r)
-		logsqlStreamsDuration.UpdateDuration(startTime)
-		return true
-	default:
-		return false
-	}
-}
-
-// getMaxQueryDuration returns the maximum duration for query from r.
-func getMaxQueryDuration(r *http.Request) time.Duration {
-	dms, err := httputil.GetDuration(r, "timeout", 0)
-	if err != nil {
-		dms = 0
-	}
-	d := time.Duration(dms) * time.Millisecond
-	if d <= 0 || d > *maxQueryDuration {
-		d = *maxQueryDuration
-	}
-	return d
-}
-
-var (
-	logsqlFacetsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
-	logsqlFacetsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/facets"}`)
-
-	logsqlFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
-	logsqlFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_names"}`)
-
-	logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
-	logsqlFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_values"}`)
-
-	logsqlHitsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
-	logsqlHitsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/hits"}`)
-
-	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/query"}`)
-
-	logsqlStatsQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
-	logsqlStatsQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query"}`)
-
-	logsqlStatsQueryRangeRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
-	logsqlStatsQueryRangeDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query_range"}`)
-
-	logsqlStreamFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
-	logsqlStreamFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_names"}`)
-
-	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
-	logsqlStreamFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_values"}`)
-
-	logsqlStreamIDsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
-	logsqlStreamIDsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_ids"}`)
-
-	logsqlStreamsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
-	logsqlStreamsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/streams"}`)
-
-	// no need to track duration for tail requests, as they usually take long time
-	logsqlTailRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
-)

app/vlselect/vmui/assets/vendor-D1GxaB_c.css

@@ -1 +0,0 @@
-.uplot,.uplot *,.uplot *:before,.uplot *:after{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol,"Noto Color Emoji";line-height:1.5;width:min-content}.u-title{text-align:center;font-size:18px;font-weight:700}.u-wrap{position:relative;-webkit-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;position:relative;width:100%;height:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{vertical-align:middle;display:inline-block}.u-legend .u-marker{width:1em;height:1em;margin-right:4px;background-clip:padding-box!important}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:#00000012;position:absolute;pointer-events:none}.u-cursor-x,.u-cursor-y{position:absolute;left:0;top:0;pointer-events:none;will-change:transform}.u-hz .u-cursor-x,.u-vt .u-cursor-y{height:100%;border-right:1px dashed #607D8B}.u-hz .u-cursor-y,.u-vt .u-cursor-x{width:100%;border-bottom:1px dashed #607D8B}.u-cursor-pt{position:absolute;top:0;left:0;border-radius:50%;border:0 solid;pointer-events:none;will-change:transform;background-clip:padding-box!important}.u-axis.u-off,.u-select.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-cursor-pt.u-off{display:none}

app/vlselect/vmui/config.json

@@ -1,5 +0,0 @@
-{
-  "license": {
-    "type": "opensource"
-  }
-}

app/vlselect/vmui/favicon.svg

@@ -1 +0,0 @@
-<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>

app/vlselect/vmui/index.html

@@ -1,57 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="./favicon.svg" />
-  <link rel="apple-touch-icon" href="./favicon.svg" />
-  <link rel="mask-icon" href="./favicon.svg" color="#000000">
-
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta name="description" content="Explore your log data with VictoriaLogs UI"/>
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="./manifest.json"/>
-  <!--
-    Notice the use of in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>UI for VictoriaLogs</title>
-
-  <meta name="twitter:card" content="summary">
-  <meta name="twitter:title" content="UI for VictoriaLogs">
-  <meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/">
-  <meta name="twitter:description" content="Explore your log data with VictoriaLogs UI">
-  <meta name="twitter:image" content="./preview.jpg">
-
-  <meta property="og:type" content="website">
-  <meta property="og:title" content="UI for VictoriaLogs">
-  <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
-  <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-721xTF8u.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-V4vnRsM-.js">
-  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C36SC0pJ.css">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-</body>
-</html>

app/vlselect/vmui/manifest.json

@@ -1,15 +0,0 @@
-{
-  "short_name": "vmui",
-  "name": "vmui",
-  "icons": [
-    {
-      "src": "favicon.svg",
-      "sizes": "any",
-      "type": "image/svg+xml"
-    }
-  ],
-  "start_url": ".",
-  "display": "standalone",
-  "theme_color": "#000000",
-  "background_color": "#ffffff"
-}

app/vlselect/vmui/robots.txt

@@ -1,3 +0,0 @@
-# https://www.robotstxt.org/robotstxt.html
-User-agent: *
-Disallow:

app/vlstorage/README.md

@@ -0,0 +1 @@
+VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlstorage/main.go

@@ -1,411 +0,0 @@
-package vlstorage
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-)
-
-var (
-	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
-		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
-	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
-		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
-	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
-		"see https://docs.victoriametrics.com/victorialogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
-		"see https://docs.victoriametrics.com/victorialogs/#storage")
-	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
-		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
-	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows")
-	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
-		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
-	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
-		"the storage stops accepting new data")
-
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge . It overrides -httpAuth.* . "+
-		"See https://docs.victoriametrics.com/victorialogs/#forced-merge")
-	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush . It overrides -httpAuth.* . "+
-		"See https://docs.victoriametrics.com/victorialogs/#forced-flush")
-
-	storageNodeAddrs = flagutil.NewArrayString("storageNode", "Comma-separated list of TCP addresses for storage nodes to route the ingested logs to and to send select queries to. "+
-		"If the list is empty, then the ingested logs are stored and queried locally from -storageDataPath")
-	insertConcurrency        = flag.Int("insert.concurrency", 2, "The average number of concurrent data ingestion requests, which can be sent to every -storageNode")
-	insertDisableCompression = flag.Bool("insert.disableCompression", false, "Whether to disable compression when sending the ingested data to -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-	selectDisableCompression = flag.Bool("select.disableCompression", false, "Whether to disable compression for select query responses received from -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-
-	storageNodeUsername     = flagutil.NewArrayString("storageNode.username", "Optional basic auth username to use for the corresponding -storageNode")
-	storageNodePassword     = flagutil.NewArrayString("storageNode.password", "Optional basic auth password to use for the corresponding -storageNode")
-	storageNodePasswordFile = flagutil.NewArrayString("storageNode.passwordFile", "Optional path to basic auth password to use for the corresponding -storageNode. "+
-		"The file is re-read every second")
-	storageNodeBearerToken     = flagutil.NewArrayString("storageNode.bearerToken", "Optional bearer auth token to use for the corresponding -storageNode")
-	storageNodeBearerTokenFile = flagutil.NewArrayString("storageNode.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -storageNode. "+
-		"The token is re-read from the file every second")
-
-	storageNodeTLS = flagutil.NewArrayBool("storageNode.tls", "Whether to use TLS (HTTPS) protocol for communicating with the corresponding -storageNode. "+
-		"By default communication is performed via HTTP")
-	storageNodeTLSCAFile = flagutil.NewArrayString("storageNode.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -storageNode. "+
-		"By default, system CA is used")
-	storageNodeTLSCertFile = flagutil.NewArrayString("storageNode.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
-		"to the corresponding -storageNode")
-	storageNodeTLSKeyFile    = flagutil.NewArrayString("storageNode.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -storageNode")
-	storageNodeTLSServerName = flagutil.NewArrayString("storageNode.tlsServerName", "Optional TLS server name to use for connections to the corresponding -storageNode. "+
-		"By default, the server name from -storageNode is used")
-	storageNodeTLSInsecureSkipVerify = flagutil.NewArrayBool("storageNode.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -storageNode")
-)
-
-var localStorage *logstorage.Storage
-var localStorageMetrics *metrics.Set
-
-var netstorageInsert *netinsert.Storage
-var netstorageSelect *netselect.Storage
-
-// Init initializes vlstorage.
-//
-// Stop must be called when vlstorage is no longer needed
-func Init() {
-	if len(*storageNodeAddrs) == 0 {
-		initLocalStorage()
-	} else {
-		initNetworkStorage()
-	}
-}
-
-func initLocalStorage() {
-	if localStorage != nil {
-		logger.Panicf("BUG: initLocalStorage() has been already called")
-	}
-
-	if retentionPeriod.Duration() < 24*time.Hour {
-		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
-	}
-	cfg := &logstorage.StorageConfig{
-		Retention:              retentionPeriod.Duration(),
-		MaxDiskSpaceUsageBytes: maxDiskSpaceUsageBytes.N,
-		FlushInterval:          *inmemoryDataFlushInterval,
-		FutureRetention:        futureRetention.Duration(),
-		LogNewStreams:          *logNewStreams,
-		LogIngestedRows:        *logIngestedRows,
-		MinFreeDiskSpaceBytes:  minFreeDiskSpaceBytes.N,
-	}
-	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
-	startTime := time.Now()
-	localStorage = logstorage.MustOpenStorage(*storageDataPath, cfg)
-
-	var ss logstorage.StorageStats
-	localStorage.UpdateStats(&ss)
-	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
-		"smallPartSize: %d bytes; bigPartSize: %d bytes",
-		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
-		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
-
-	// register local storage metrics
-	localStorageMetrics = metrics.NewSet()
-	localStorageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, localStorage)
-	})
-	metrics.RegisterSet(localStorageMetrics)
-}
-
-func initNetworkStorage() {
-	if netstorageInsert != nil || netstorageSelect != nil {
-		logger.Panicf("BUG: initNetworkStorage() has been already called")
-	}
-
-	authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
-	isTLSs := make([]bool, len(*storageNodeAddrs))
-	for i := range authCfgs {
-		authCfgs[i] = newAuthConfigForStorageNode(i)
-		isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
-	}
-
-	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
-	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
-
-	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
-	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
-
-	logger.Infof("initialized all the network services")
-}
-
-func newAuthConfigForStorageNode(argIdx int) *promauth.Config {
-	username := storageNodeUsername.GetOptionalArg(argIdx)
-	password := storageNodePassword.GetOptionalArg(argIdx)
-	passwordFile := storageNodePasswordFile.GetOptionalArg(argIdx)
-	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
-		basicAuthCfg = &promauth.BasicAuthConfig{
-			Username:     username,
-			Password:     promauth.NewSecret(password),
-			PasswordFile: passwordFile,
-		}
-	}
-
-	token := storageNodeBearerToken.GetOptionalArg(argIdx)
-	tokenFile := storageNodeBearerTokenFile.GetOptionalArg(argIdx)
-
-	tlsCfg := &promauth.TLSConfig{
-		CAFile:             storageNodeTLSCAFile.GetOptionalArg(argIdx),
-		CertFile:           storageNodeTLSCertFile.GetOptionalArg(argIdx),
-		KeyFile:            storageNodeTLSKeyFile.GetOptionalArg(argIdx),
-		ServerName:         storageNodeTLSServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: storageNodeTLSInsecureSkipVerify.GetOptionalArg(argIdx),
-	}
-
-	opts := &promauth.Options{
-		BasicAuth:       basicAuthCfg,
-		BearerToken:     token,
-		BearerTokenFile: tokenFile,
-		TLSConfig:       tlsCfg,
-	}
-	ac, err := opts.NewConfig()
-	if err != nil {
-		logger.Panicf("FATAL: cannot populate auth config for storage node #%d: %s", argIdx, err)
-	}
-
-	return ac
-}
-
-// Stop stops vlstorage.
-func Stop() {
-	if localStorage != nil {
-		metrics.UnregisterSet(localStorageMetrics, true)
-		localStorageMetrics = nil
-
-		localStorage.MustClose()
-		localStorage = nil
-	} else {
-		netstorageInsert.MustStop()
-		netstorageInsert = nil
-
-		netstorageSelect.MustStop()
-		netstorageSelect = nil
-	}
-}
-
-// RequestHandler is a storage request handler.
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	switch path {
-	case "/internal/force_merge":
-		return processForceMerge(w, r)
-	case "/internal/force_flush":
-		return processForceFlush(w, r)
-	}
-	return false
-}
-
-func processForceMerge(w http.ResponseWriter, r *http.Request) bool {
-	if localStorage == nil {
-		// Force merge isn't supported by non-local storage
-		return false
-	}
-
-	if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-		return true
-	}
-
-	// Run force merge in background
-	partitionNamePrefix := r.FormValue("partition_prefix")
-	go func() {
-		activeForceMerges.Inc()
-		defer activeForceMerges.Dec()
-		logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-		startTime := time.Now()
-		localStorage.MustForceMerge(partitionNamePrefix)
-		logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-	}()
-	return true
-}
-
-func processForceFlush(w http.ResponseWriter, r *http.Request) bool {
-	if localStorage == nil {
-		// Force merge isn't supported by non-local storage
-		return false
-	}
-
-	if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey) {
-		return true
-	}
-
-	logger.Infof("flushing storage to make pending data available for reading")
-	localStorage.DebugFlush()
-	return true
-}
-
-// Storage implements insertutil.LogRowsStorage interface
-type Storage struct{}
-
-// CanWriteData returns non-nil error if it cannot write data to vlstorage
-func (*Storage) CanWriteData() error {
-	if localStorage == nil {
-		// The data can be always written in non-local mode.
-		return nil
-	}
-
-	if localStorage.IsReadOnly() {
-		return &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
-				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
-			StatusCode: http.StatusTooManyRequests,
-		}
-	}
-	return nil
-}
-
-// MustAddRows adds lr to vlstorage
-//
-// It is advised to call CanWriteData() before calling MustAddRows()
-func (*Storage) MustAddRows(lr *logstorage.LogRows) {
-	if localStorage != nil {
-		// Store lr in the local storage.
-		localStorage.MustAddRows(lr)
-	} else {
-		// Store lr across the remote storage nodes.
-		lr.ForEachRow(netstorageInsert.AddRow)
-	}
-}
-
-// RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	if localStorage != nil {
-		return localStorage.RunQuery(ctx, tenantIDs, q, writeBlock)
-	}
-	return netstorageSelect.RunQuery(ctx, tenantIDs, q, writeBlock)
-}
-
-// GetFieldNames executes q and returns field names seen in results.
-func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetFieldNames(ctx, tenantIDs, q)
-}
-
-// GetFieldValues executes q and returns unique values for the fieldName seen in results.
-//
-// If limit > 0, then up to limit unique values are returned.
-func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
-}
-
-// GetStreamFieldNames executes q and returns stream field names seen in results.
-func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetStreamFieldNames(ctx, tenantIDs, q)
-}
-
-// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
-//
-// If limit > 0, then up to limit unique stream field values are returned.
-func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-}
-
-// GetStreams executes q and returns streams seen in query results.
-//
-// If limit > 0, then up to limit unique streams are returned.
-func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreams(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreams(ctx, tenantIDs, q, limit)
-}
-
-// GetStreamIDs executes q and returns streamIDs seen in query results.
-//
-// If limit > 0, then up to limit unique streamIDs are returned.
-func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamIDs(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreamIDs(ctx, tenantIDs, q, limit)
-}
-
-func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
-	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
-
-	if maxDiskSpaceUsageBytes.N > 0 {
-		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_max_disk_space_usage_bytes{path=%q}`, *storageDataPath), uint64(maxDiskSpaceUsageBytes.N))
-	}
-
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
-
-	isReadOnly := uint64(0)
-	if ss.IsReadOnly {
-		isReadOnly = 1
-	}
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), isReadOnly)
-
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/inmemory"}`, ss.InmemoryActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/small"}`, ss.SmallPartActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/big"}`, ss.BigPartActiveMerges)
-
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/inmemory"}`, ss.InmemoryMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/small"}`, ss.SmallPartMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/big"}`, ss.BigPartMergesTotal)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/inmemory"}`, ss.InmemoryRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/small"}`, ss.SmallPartRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/big"}`, ss.BigPartRowsCount)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/inmemory"}`, ss.InmemoryParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/small"}`, ss.SmallParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/big"}`, ss.BigParts)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/inmemory"}`, ss.InmemoryBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/small"}`, ss.SmallPartBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/big"}`, ss.BigPartBlocks)
-
-	metrics.WriteGaugeUint64(w, `vl_partitions`, ss.PartitionsCount)
-	metrics.WriteCounterUint64(w, `vl_streams_created_total`, ss.StreamsCreatedTotal)
-
-	metrics.WriteGaugeUint64(w, `vl_indexdb_rows`, ss.IndexdbItemsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_parts`, ss.IndexdbPartsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_blocks`, ss.IndexdbBlocksCount)
-
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="indexdb"}`, ss.IndexdbSizeBytes)
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="storage"}`, ss.CompressedInmemorySize+ss.CompressedSmallPartSize+ss.CompressedBigPartSize)
-
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/inmemory"}`, ss.CompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/small"}`, ss.CompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/big"}`, ss.CompressedBigPartSize)
-
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/inmemory"}`, ss.UncompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/small"}`, ss.UncompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/big"}`, ss.UncompressedBigPartSize)
-
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
-}
-
-var activeForceMerges = metrics.NewCounter("vl_active_force_merges")

app/vlstorage/netinsert/netinsert.go

@@ -1,369 +0,0 @@
-package netinsert
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/valyala/fastrand"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-)
-
-// the maximum size of a single data block sent to storage node.
-const maxInsertBlockSize = 2 * 1024 * 1024
-
-// ProtocolVersion is the version of the data ingestion protocol.
-//
-// It must be changed every time the data encoding at /internal/insert HTTP endpoint is changed.
-const ProtocolVersion = "v1"
-
-// Storage is a network storage for sending data to remote storage nodes in the cluster.
-type Storage struct {
-	sns []*storageNode
-
-	disableCompression bool
-
-	srt *streamRowsTracker
-
-	pendingDataBuffers chan *bytesutil.ByteBuffer
-
-	stopCh chan struct{}
-	wg     sync.WaitGroup
-}
-
-type storageNode struct {
-	// scheme is http or https scheme to communicate with addr
-	scheme string
-
-	// addr is TCP address of storage node to send the ingested data to
-	addr string
-
-	// s is a storage, which holds the given storageNode
-	s *Storage
-
-	// c is an http client used for sending data blocks to addr.
-	c *http.Client
-
-	// ac is auth config used for setting request headers such as Authorization and Host.
-	ac *promauth.Config
-
-	// pendingData contains pending data, which must be sent to the storage node at the addr.
-	pendingDataMu        sync.Mutex
-	pendingData          *bytesutil.ByteBuffer
-	pendingDataLastFlush time.Time
-
-	// the unix timestamp until the storageNode is disabled for data writing.
-	disabledUntil atomic.Uint64
-}
-
-func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
-	tr := httputil.NewTransport(false, "vlinsert_backend")
-	tr.TLSHandshakeTimeout = 20 * time.Second
-	tr.DisableCompression = true
-
-	scheme := "http"
-	if isTLS {
-		scheme = "https"
-	}
-
-	sn := &storageNode{
-		scheme: scheme,
-		addr:   addr,
-		s:      s,
-		c: &http.Client{
-			Transport: ac.NewRoundTripper(tr),
-		},
-		ac: ac,
-
-		pendingData: &bytesutil.ByteBuffer{},
-	}
-
-	s.wg.Add(1)
-	go func() {
-		defer s.wg.Done()
-		sn.backgroundFlusher()
-	}()
-
-	return sn
-}
-
-func (sn *storageNode) backgroundFlusher() {
-	t := time.NewTicker(time.Second)
-	defer t.Stop()
-
-	for {
-		select {
-		case <-sn.s.stopCh:
-			return
-		case <-t.C:
-			sn.flushPendingData()
-		}
-	}
-}
-
-func (sn *storageNode) flushPendingData() {
-	sn.pendingDataMu.Lock()
-	if time.Since(sn.pendingDataLastFlush) < time.Second {
-		// nothing to flush
-		sn.pendingDataMu.Unlock()
-		return
-	}
-
-	pendingData := sn.grabPendingDataForFlushLocked()
-	sn.pendingDataMu.Unlock()
-
-	sn.mustSendInsertRequest(pendingData)
-}
-
-func (sn *storageNode) addRow(r *logstorage.InsertRow) {
-	bb := bbPool.Get()
-	b := bb.B
-
-	b = r.Marshal(b)
-
-	if len(b) > maxInsertBlockSize {
-		logger.Warnf("skipping too long log entry, since its length exceeds %d bytes; the actual log entry length is %d bytes; log entry contents: %s", maxInsertBlockSize, len(b), b)
-		return
-	}
-
-	var pendingData *bytesutil.ByteBuffer
-	sn.pendingDataMu.Lock()
-	if sn.pendingData.Len()+len(b) > maxInsertBlockSize {
-		pendingData = sn.grabPendingDataForFlushLocked()
-	}
-	sn.pendingData.MustWrite(b)
-	sn.pendingDataMu.Unlock()
-
-	bb.B = b
-	bbPool.Put(bb)
-
-	if pendingData != nil {
-		sn.mustSendInsertRequest(pendingData)
-	}
-}
-
-var bbPool bytesutil.ByteBufferPool
-
-func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
-	sn.pendingDataLastFlush = time.Now()
-	pendingData := sn.pendingData
-	sn.pendingData = <-sn.s.pendingDataBuffers
-
-	return pendingData
-}
-
-func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
-	defer func() {
-		pendingData.Reset()
-		sn.s.pendingDataBuffers <- pendingData
-	}()
-
-	err := sn.sendInsertRequest(pendingData)
-	if err == nil {
-		return
-	}
-
-	if !errors.Is(err, errTemporarilyDisabled) {
-		logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
-	}
-	for !sn.s.sendInsertRequestToAnyNode(pendingData) {
-		logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
-
-		t := timerpool.Get(time.Second)
-		select {
-		case <-sn.s.stopCh:
-			timerpool.Put(t)
-			logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
-			return
-		case <-t.C:
-			timerpool.Put(t)
-		}
-	}
-}
-
-func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
-	dataLen := pendingData.Len()
-	if dataLen == 0 {
-		// Nothing to send.
-		return nil
-	}
-
-	if sn.disabledUntil.Load() > fasttime.UnixTimestamp() {
-		return errTemporarilyDisabled
-	}
-
-	ctx, cancel := contextutil.NewStopChanContext(sn.s.stopCh)
-	defer cancel()
-
-	var body io.Reader
-	if !sn.s.disableCompression {
-		bb := zstdBufPool.Get()
-		defer zstdBufPool.Put(bb)
-
-		bb.B = zstd.CompressLevel(bb.B[:0], pendingData.B, 1)
-		body = bb.NewReader()
-	} else {
-		body = pendingData.NewReader()
-	}
-
-	reqURL := sn.getRequestURL("/internal/insert")
-	req, err := http.NewRequestWithContext(ctx, "POST", reqURL, body)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating an http request: %s", err)
-	}
-	req.Header.Set("Content-Type", "application/octet-stream")
-	if !sn.s.disableCompression {
-		req.Header.Set("Content-Encoding", "zstd")
-	}
-	if err := sn.ac.SetHeaders(req, true); err != nil {
-		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
-	}
-
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		// Disable sn for data writing for 10 seconds.
-		sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-		return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 == 2 {
-		return nil
-	}
-
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		respBody = []byte(fmt.Sprintf("%s", err))
-	}
-
-	// Disable sn for data writing for 10 seconds.
-	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-	return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
-}
-
-func (sn *storageNode) getRequestURL(path string) string {
-	return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
-}
-
-var zstdBufPool bytesutil.ByteBufferPool
-
-// NewStorage returns new Storage for the given addrs with the given authCfgs.
-//
-// The concurrency is the average number of concurrent connections per every addr.
-//
-// If disableCompression is set, then the data is sent uncompressed to the remote storage.
-//
-// Call MustStop on the returned storage when it is no longer needed.
-func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
-	pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
-	for i := 0; i < cap(pendingDataBuffers); i++ {
-		pendingDataBuffers <- &bytesutil.ByteBuffer{}
-	}
-
-	s := &Storage{
-		disableCompression: disableCompression,
-		pendingDataBuffers: pendingDataBuffers,
-		stopCh:             make(chan struct{}),
-	}
-
-	sns := make([]*storageNode, len(addrs))
-	for i, addr := range addrs {
-		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
-	}
-	s.sns = sns
-
-	s.srt = newStreamRowsTracker(len(sns))
-
-	return s
-}
-
-// MustStop stops the s.
-func (s *Storage) MustStop() {
-	close(s.stopCh)
-	s.wg.Wait()
-	s.sns = nil
-}
-
-// AddRow adds the given log row into s.
-func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
-	idx := s.srt.getNodeIdx(streamHash)
-	sn := s.sns[idx]
-	sn.addRow(r)
-}
-
-func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
-	startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
-	for i := range s.sns {
-		idx := (startIdx + i) % len(s.sns)
-		sn := s.sns[idx]
-		err := sn.sendInsertRequest(pendingData)
-		if err == nil {
-			return true
-		}
-		if !errors.Is(err, errTemporarilyDisabled) {
-			logger.Warnf("cannot send pending data to the storage node %q: %s; trying to send it to another storage node", sn.addr, err)
-		}
-	}
-	return false
-}
-
-var errTemporarilyDisabled = fmt.Errorf("writing to the node is temporarily disabled")
-
-type streamRowsTracker struct {
-	mu sync.Mutex
-
-	nodesCount    int64
-	rowsPerStream map[uint64]uint64
-}
-
-func newStreamRowsTracker(nodesCount int) *streamRowsTracker {
-	return &streamRowsTracker{
-		nodesCount:    int64(nodesCount),
-		rowsPerStream: make(map[uint64]uint64),
-	}
-}
-
-func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
-	if srt.nodesCount == 1 {
-		// Fast path for a single node.
-		return 0
-	}
-
-	srt.mu.Lock()
-	defer srt.mu.Unlock()
-
-	streamRows := srt.rowsPerStream[streamHash] + 1
-	srt.rowsPerStream[streamHash] = streamRows
-
-	if streamRows <= 1000 {
-		// Write the initial rows for the stream to a single storage node for better locality.
-		// This should work great for log streams containing small number of logs, since will be distributed
-		// evenly among available storage nodes because they have different streamHash.
-		return streamHash % uint64(srt.nodesCount)
-	}
-
-	// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
-	// in order to improve query performance over this stream (the data for the log stream
-	// can be processed in parallel on all the storage nodes).
-	//
-	// The random distribution is preferred over round-robin distribution in order to avoid possible
-	// dependency between the order of the ingested logs and the number of storage nodes,
-	// which may lead to non-uniform distribution of logs among storage nodes.
-	return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
-}

app/vlstorage/netinsert/netinsert_test.go

@@ -1,57 +0,0 @@
-package netinsert
-
-import (
-	"fmt"
-	"math"
-	"math/rand"
-	"testing"
-
-	"github.com/cespare/xxhash/v2"
-)
-
-func TestStreamRowsTracker(t *testing.T) {
-	f := func(rowsCount, streamsCount, nodesCount int) {
-		t.Helper()
-
-		// generate stream hashes
-		streamHashes := make([]uint64, streamsCount)
-		for i := range streamHashes {
-			streamHashes[i] = xxhash.Sum64([]byte(fmt.Sprintf("stream %d.", i)))
-		}
-
-		srt := newStreamRowsTracker(nodesCount)
-
-		rng := rand.New(rand.NewSource(0))
-		rowsPerNode := make([]uint64, nodesCount)
-		for i := 0; i < rowsCount; i++ {
-			streamIdx := rng.Intn(streamsCount)
-			h := streamHashes[streamIdx]
-			nodeIdx := srt.getNodeIdx(h)
-			rowsPerNode[nodeIdx]++
-		}
-
-		// Verify that rows are uniformly distributed among nodes.
-		expectedRowsPerNode := float64(rowsCount) / float64(nodesCount)
-		for nodeIdx, nodeRows := range rowsPerNode {
-			if math.Abs(float64(nodeRows)-expectedRowsPerNode)/expectedRowsPerNode > 0.15 {
-				t.Fatalf("non-uniform distribution of rows among nodes; node %d has %d rows, while it must have %v rows; rowsPerNode=%d",
-					nodeIdx, nodeRows, expectedRowsPerNode, rowsPerNode)
-			}
-		}
-	}
-
-	rowsCount := 10000
-	streamsCount := 9
-	nodesCount := 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 10000
-	streamsCount = 100
-	nodesCount = 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 100000
-	streamsCount = 1000
-	nodesCount = 9
-	f(rowsCount, streamsCount, nodesCount)
-}