docs: add HA options

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -5,10 +5,10 @@ body:
   - type: markdown
     attributes:
       value: |
-        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade) 
+        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
         to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
         and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) first.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/troubleshooting/) first.
   - type: textarea
     id: describe-the-bug
     attributes:
@@ -60,12 +60,12 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
         
         See how to setup monitoring here:
-          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
+          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/#monitoring)
     validations:
       required: false
   - type: textarea

.github/ISSUE_TEMPLATE/question.yml

@@ -24,9 +24,9 @@ body:
       label: Troubleshooting docs
       description: I am familiar with the following troubleshooting docs
       options:
-        - label: General - https://docs.victoriametrics.com/victoriametrics/troubleshooting/
+        - label: General - https://docs.victoriametrics.com/troubleshooting/
           required: false
-        - label: vmagent - https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
+        - label: vmagent - https://docs.victoriametrics.com/vmagent/#troubleshooting
           required: false
-        - label: vmalert - https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting
+        - label: vmalert - https://docs.victoriametrics.com/vmalert/#troubleshooting
           required: false

.github/copilot-instructions.md

@@ -1,23 +0,0 @@
-# Project Overview
-
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
-
-## Folder Structure
-
-- `/app`: Contains the compilable binaries.
-- `/lib`: Contains the golang reusable libraries
-- `/docs/victoriametrics`: Contains documentation for the project.
-- `/apptest/tests`: Contains integration tests.
-
-## Libraries and Frameworks
-
-- Backend: Golang, no framework. Use third-party libraries sparingly.
-- Frontend: React.
-
-## Code review guidelines
-
-Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
-Verify the entry is under the ## tip section and matches the structure and style of existing entries.
-Chore-only changes may be omitted from the changelog.
-
-

.github/pull_request_template.md

@@ -6,5 +6,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 
 The following checks are **mandatory**:
 
-- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
-- [ ] My change adheres to [VictoriaMetrics development goals](https://docs.victoriametrics.com/victoriametrics/goals/).
+- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).

.github/workflows/build.yml

@@ -1,69 +0,0 @@
-name: build
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile'
-      - '**/Makefile'
-      - '!app/vmui/**'
-      - '.github/workflows/build.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile'
-      - '**/Makefile'
-      - '!app/vmui/**'
-      - '.github/workflows/build.yml'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  build:
-    name: ${{ matrix.os }}-${{ matrix.arch }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-          - os: linux
-            arch: arm
-          - os: linux
-            arch: ppc64le
-          - os: linux
-            arch: 386
-          - os: freebsd
-            arch: amd64
-          - os: openbsd
-            arch: amd64
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Build vmcluster for ${{ matrix.os }}-${{ matrix.arch }}
-        run: make vmcluster-${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/codeql-analysis-go.yml

@@ -1,62 +0,0 @@
-name: 'CodeQL Go'
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v5
-
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: go
-
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: 'language:go'

.github/workflows/codeql-analysis-js.yml

@@ -0,0 +1,46 @@
+name: "CodeQL - JS"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+  
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["javascript"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,103 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["go"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+        if: ${{ matrix.language == 'go' }}
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
+        if: ${{ matrix.language == 'go' }}
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/docs.yaml

@@ -1,57 +0,0 @@
-name: publish-docs
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'docs/**'
-      - '.github/workflows/docs.yaml'
-  workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-        with:
-          path: __vm
-
-      - name: Checkout private code
-        uses: actions/checkout@v5
-        with:
-          repository: VictoriaMetrics/vmdocs
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: __vm-docs
-
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v6
-        id: import-gpg
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-          git_config_global: true
-
-      - name: Copy docs
-        id: update
-        run: |
-          find docs -type d -maxdepth 1 -mindepth 1 -exec \
-            sh -c 'rsync -zarvh --delete {}/ ../__vm-docs/content/$(basename {})/' \;
-          echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
-        working-directory: __vm
-
-      - name: Push to vmdocs
-        run: |
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          if [[ -n $(git status --porcelain) ]]; then
-            git add .
-            git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.update.outputs.SHORT_SHA }}"
-            git push
-          fi
-        working-directory: __vm-docs

.github/workflows/main.yml

@@ -0,0 +1,120 @@
+name: main
+on:
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+  pull_request:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-check-all-
+
+      - name: Run check-all
+        run: |
+          make check-all
+          git diff --exit-code
+
+  build:
+    needs: lint
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
+
+      - name: Build
+        run: make crossbuild
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        scenario: ["test-full", "test-pure", "test-full-386"]
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Cache Go artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/go/bin
+          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
+          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
+
+      - name: run tests
+        run: make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v4
+        with:
+          file: ./coverage.txt

.github/workflows/sync-docs.yml

@@ -0,0 +1,51 @@
+name: publish-docs
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'docs/**'
+  workflow_dispatch: {}
+permissions:
+  contents: read  # This is required for actions/checkout and to commit back image update
+  deployments: write
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+        with:
+          path: main
+      - name: Checkout private code
+        uses: actions/checkout@v4
+        with:
+          repository: VictoriaMetrics/vmdocs
+          token: ${{ secrets.VM_BOT_GH_TOKEN }}
+          path: docs
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+          workdir: docs
+      - name: Set short git commit SHA
+        id: vars
+        run: |
+          calculatedSha=$(git rev-parse --short ${{ github.sha }})
+          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
+        working-directory: main
+      - name: update code and commit
+        run: |
+          rm -rf content
+          cp -r ../main/docs content
+          make clean-after-copy
+          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
+          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
+          git add .
+          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
+          git push
+        working-directory: docs

.github/workflows/test.yml

@@ -1,113 +0,0 @@
-name: test
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - 'go.*'
-      - '.github/workflows/main.yml'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-
-jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-
-      - name: Cache golangci-lint
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/golangci-lint
-            ~/go/bin
-          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
-
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
-
-  unit:
-    name: unit
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        scenario:
-          - 'test-full'
-          - 'test-full-386'
-          - 'test-pure'
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v5
-        with:
-          files: ./coverage.txt
-
-  integration:
-    name: integration
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache-dependency-path: |
-            go.sum
-            Makefile
-            app/**/Makefile
-          go-version: stable
-
-      - name: Run integration tests
-        run: make integration-test

.github/workflows/vmui.yml

@@ -1,82 +0,0 @@
-name: vmui
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - 'app/vmui/packages/vmui/**'
-      - '.github/workflows/vmui.yml'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - 'app/vmui/packages/vmui/**'
-      - '.github/workflows/vmui.yml'
-
-permissions:
-  contents: read
-  packages: read
-  pull-requests: read
-  checks: write
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  vmui-checks:
-    name: VMUI Checks (lint, test, typecheck)
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v5
-
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: '24.x'
-
-      - name: Cache node-modules
-        uses: actions/cache@v4
-        with:
-          path: |
-            app/vmui/packages/vmui/node_modules
-          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: vmui-artifacts-${{ runner.os }}-
-
-      - name: Run lint
-        id: lint
-        run: make vmui-lint
-        continue-on-error: true
-
-      - name: Run tests
-        id: test
-        run: make vmui-test
-        continue-on-error: true
-
-      - name: Run typecheck
-        id: typecheck
-        run: make vmui-typecheck
-        continue-on-error: true
-
-      - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@v3
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          report-json: app/vmui/packages/vmui/vmui-lint-report.json
-
-      - name: Check overall status
-        run: |
-          echo "Lint status: ${{ steps.lint.outcome }}"
-          echo "Test status: ${{ steps.test.outcome }}"
-          echo "Typecheck status: ${{ steps.typecheck.outcome }}"
-
-          if [[ "${{ steps.lint.outcome }}" == "failure" || "${{ steps.test.outcome }}" == "failure" || "${{ steps.typecheck.outcome }}" == "failure" ]]; then
-            echo "One or more checks failed"
-            exit 1
-          else
-            echo "All checks passed"
-          fi

.gitignore

@@ -7,12 +7,10 @@
 .vscode
 *.test
 *.swp
-/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
 /vmagent-remotewrite-data
-/vlagent-remotewritewrite
 /vmstorage-data
 /vmselect-cache
 /package/temp-deb-*
@@ -24,8 +22,4 @@ Gemfile.lock
 /_site
 _site
 *.tmp
-/docs/.jekyll-metadata
-coverage.txt
-cspell.json
-*~
-deployment/docker/provisioning/plugins/
+/docs/.jekyll-metadata

.golangci.yml

@@ -1,29 +1,22 @@
-version: "2"
+run:
+  timeout: 2m
+
 linters:
-  settings:
-    errcheck:
-      exclude-functions:
-        - fmt.Fprintf
-        - fmt.Fprint
-        - (net/http.ResponseWriter).Write
-  exclusions:
-    generated: lax
-    presets:
-      - common-false-positives
-      - legacy
-      - std-error-handling
-    rules:
-      - linters:
-          - staticcheck
-        text: 'SA(4003|1019|5011):'
-    paths:
-      - third_party$
-      - builtin$
-      - examples$
-formatters:
-  exclusions:
-    generated: lax
-    paths:
-      - third_party$
-      - builtin$
-      - examples$
+  enable:
+  - revive
+
+issues:
+  exclude-rules:
+  - linters:
+    - staticcheck
+    text: "SA(4003|1019|5011):"
+  include:
+  - EXC0012
+  - EXC0014
+
+linters-settings:
+  errcheck:
+    exclude-functions:
+      - "fmt.Fprintf"
+      - "fmt.Fprint"
+      - "(net/http.ResponseWriter).Write"

.wwhrd.yml

@@ -4,4 +4,3 @@ allowlist:
   - BSD-3-Clause
   - BSD-2-Clause
   - ISC
-  - MPL-2.0

CODE_OF_CONDUCT_RU.md

@@ -0,0 +1,120 @@
+
+# Кодекс Поведения участника
+
+## Наши обязательства
+
+Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
+свободным от притеснений для всех, независимо от возраста, телосложения,
+видимых или невидимых ограничений способности, этнической принадлежности,
+половых признаков, гендерной идентичности и выражения, уровня опыта,
+образования, социо-экономического статуса, национальности, внешности,
+расы, религии, или сексуальной идентичности и ориентации.
+
+Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
+дружелюбное, многообразное, инклюзивное и здоровое сообщество.
+
+## Наши стандарты
+
+Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
+
+* Проявление доброты и эмпатии к другим участникам проекта
+* Уважение к чужой точке зрения и опыту
+* Конструктивная критика и принятие конструктивной критики
+* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
+  и извлечение уроков из опыта
+* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
+
+Примеры неприемлемого поведения участников включают в себя:
+
+* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
+* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
+* Публичное или приватное домогательство
+* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
+* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
+
+## Обязанности
+
+Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
+поведения и будут предпринимать соответствующие и честные меры по исправлению положения
+в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
+
+Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
+комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
+с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
+
+## Область применения
+
+Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
+а также когда человек официально представляет сообщество в публичных местах.
+Примеры представления проекта или сообщества включают использование официальной электронной почты,
+публикации в официальном аккаунте в социальных сетях,
+или упоминания как представителя в онлайн или оффлайн мероприятии.
+
+## Приведение в исполнение
+
+О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
+поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
+Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
+
+Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
+неприкосновенность автора сообщения.
+
+## Руководство по исполнению
+
+Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
+чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
+
+### 1. Исправление
+
+**Общественное влияние**: Использование недопустимой лексики или другое поведение,
+считающиеся непрофессиональным или нежелательным в сообществе.
+
+**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
+объясняющее суть нарушения и почему такое поведение
+было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
+
+### 2. Предупреждение
+
+**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
+
+**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
+На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
+включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
+в публичных пространствах, а так же во внешних каналах,
+таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
+
+### 3. Временный бан
+
+**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
+включая продолжительное неуместное поведение.
+
+**Последствия**: Временный запрет (бан) на любое взаимодействие
+или публичное общение с сообществом на определенный период времени.
+На этот период не допускается публичное или личное взаимодействие с людьми,
+вовлеченными в инцидент, включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса.
+Нарушение этих правил влечет за собой вечный бан.
+
+### 4. Вечный бан
+
+**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
+включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
+или проявление агрессии либо пренебрежительного отношения к категориям лиц.
+
+**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
+
+## Атрибуция
+
+Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
+версии 2.0, доступной по адресу
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
+
+Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

CONTRIBUTING.md

@@ -1 +1 @@
-The document has been moved [here](https://docs.victoriametrics.com/victoriametrics/contributing/).
+The document has been moved [here](https://docs.victoriametrics.com/contributing/).

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2024 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -5,29 +5,22 @@ MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
+LATEST_TAG ?= cluster-latest
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
 PKG_TAG := $(BUILDINFO_TAG)
 endif
 
-EXTRA_DOCKER_TAG_SUFFIX ?=
-EXTRA_GO_BUILD_TAGS ?=
-
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
-TAR_OWNERSHIP ?= --owner=1000 --group=1000
-
-GOLANGCI_LINT_VERSION := 2.4.0
 
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include codespell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
 include package/release/Makefile
-include benchmarks/Makefile
 
 all: \
 	vminsert \
@@ -82,20 +75,8 @@ vmcluster-windows-amd64: \
 	vmselect-windows-amd64 \
 	vmstorage-windows-amd64
 
-vmcluster-darwin-amd64: \
-	vminsert-darwin-amd64 \
-	vmselect-darwin-amd64 \
-	vmstorage-darwin-amd64
-
-vmcluster-darwin-arm64: \
-	vminsert-darwin-arm64 \
-	vmselect-darwin-arm64 \
-	vmstorage-darwin-arm64
-
-# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 crossbuild: vmcluster-crossbuild
 
-# When adding a new crossbuild target, please also add it to the .github/workflows/build.yml
 vmcluster-crossbuild:
 	$(MAKE_PARALLEL) vmcluster-linux-amd64 \
 		vmcluster-linux-arm64 \
@@ -115,52 +96,12 @@ package: \
 	package-vmselect \
 	package-vmstorage
 
-publish-final-images:
-	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=victoria-metrics $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmagent $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmalert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmauth $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackup $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmrestore $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-from-rc && \
-	PKG_TAG=$(TAG) $(MAKE) publish-latest
-
-publish-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
-
 publish-release:
 	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && $(MAKE) publish
+	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
 
 release:
 	$(MAKE_PARALLEL) release-vmcluster
@@ -170,9 +111,7 @@ release-vmcluster: \
 	release-vmcluster-linux-arm64 \
 	release-vmcluster-freebsd-amd64 \
 	release-vmcluster-openbsd-amd64 \
-	release-vmcluster-windows-amd64 \
-	release-vmcluster-darwin-amd64 \
-	release-vmcluster-darwin-arm64
+	release-vmcluster-windows-amd64
 
 release-vmcluster-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
@@ -189,18 +128,12 @@ release-vmcluster-openbsd-amd64:
 release-vmcluster-windows-amd64:
 	GOARCH=amd64 $(MAKE) release-vmcluster-windows-goarch
 
-release-vmcluster-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-darwin-arm64:
-	 GOOS=darwin GOARCH=arm64 $(MAKE) release-vmcluster-goos-goarch
-
 release-vmcluster-goos-goarch: \
 	vminsert-$(GOOS)-$(GOARCH)-prod \
 	vmselect-$(GOOS)-$(GOARCH)-prod \
 	vmstorage-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vminsert-$(GOOS)-$(GOARCH)-prod \
 			vmselect-$(GOOS)-$(GOARCH)-prod \
 			vmstorage-$(GOOS)-$(GOARCH)-prod \
@@ -239,64 +172,53 @@ pprof-cpu:
 fmt:
 	gofmt -l -w -s ./lib
 	gofmt -l -w -s ./app
-	gofmt -l -w -s ./apptest
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
+	go vet ./lib/...
 	go vet ./app/...
-	go vet ./apptest/...
 
 check-all: fmt vet golangci-lint govulncheck
 
-clean-checkers: remove-golangci-lint remove-govulncheck
-
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
-
-integration-test:
-	$(MAKE) apptest
-
-apptest:
-	$(MAKE) all vmctl vmbackup vmrestore
-	go test ./apptest/... -skip="^TestSingle.*"
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
+	go test -bench=. ./lib/...
 	go test -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	go get -u ./lib/...
-	go get -u ./app/...
-	go mod tidy -compat=1.24
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.22
 	go mod vendor
 
 app-local:
-	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=1 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-pure:
-	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-goos-goarch:
-	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-$(GOOS)-$(GOARCH)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME)
 
 app-local-windows-goarch:
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
+	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
 	qtc
@@ -306,13 +228,10 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)
-
-remove-golangci-lint:
-	rm -rf `which golangci-lint`
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
 
 govulncheck: install-govulncheck
 	govulncheck ./...
@@ -320,11 +239,37 @@ govulncheck: install-govulncheck
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 
-remove-govulncheck:
-	rm -rf `which govulncheck`
-
 install-wwhrd:
 	which wwhrd || go install github.com/frapposelli/wwhrd@latest
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
+
+copy-docs:
+# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
+# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
+	echo "---" > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "sort: ${ORDER}" >> ${DST}; \
+		echo "weight: ${ORDER}" >> ${DST}; \
+		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
+	fi
+
+	echo "title: ${TITLE}" >> ${DST}
+	@if [ ${OLD_URL} ]; then \
+		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
+	fi
+	echo "---" >> ${DST}
+	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	rm -rf docs/*.tmp
+
+# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
+# For ORDER=0 it adds no order tag/weight tag.
+# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
+# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
+# Images starting with <img src="docs/ are replaced with <img src="
+# Cluster docs are supposed to be ordered as 2nd.
+# The rest of docs is ordered manually.
+docs-sync:
+	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md OLD_URL='/Cluster-VictoriaMetrics.html' ORDER=2 TITLE='Cluster version' $(MAKE) copy-docs

SECURITY.md

@@ -6,13 +6,13 @@ The following versions of VictoriaMetrics receive regular security fixes:
 
 | Version | Supported          |
 |---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
+| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to <security@victoriametrics.com>
+Please report any security issues to security@victoriametrics.com

app/victoria-logs/Makefile

@@ -0,0 +1,110 @@
+# All these commands must run from repository root.
+
+victoria-logs:
+	APP_NAME=victoria-logs $(MAKE) app-local
+
+victoria-logs-race:
+	APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
+
+victoria-logs-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker
+
+victoria-logs-pure-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
+
+victoria-logs-linux-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
+
+victoria-logs-linux-arm-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
+
+victoria-logs-linux-arm64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
+
+victoria-logs-linux-ppc64le-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
+
+victoria-logs-linux-386-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
+
+victoria-logs-darwin-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
+
+victoria-logs-darwin-arm64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
+
+victoria-logs-freebsd-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
+
+victoria-logs-openbsd-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
+
+victoria-logs-windows-amd64-prod:
+	APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
+
+package-victoria-logs:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker
+
+package-victoria-logs-pure:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
+
+package-victoria-logs-amd64:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
+
+package-victoria-logs-arm:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
+
+package-victoria-logs-arm64:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
+
+package-victoria-logs-ppc64le:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
+
+package-victoria-logs-386:
+	APP_NAME=victoria-logs $(MAKE) package-via-docker-386
+
+publish-victoria-logs:
+	APP_NAME=victoria-logs $(MAKE) publish-via-docker
+
+victoria-logs-linux-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-arm:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-arm64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-ppc64le:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-s390x:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
+
+victoria-logs-linux-386:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
+
+victoria-logs-darwin-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-darwin-arm64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-freebsd-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-openbsd-amd64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
+
+victoria-logs-windows-amd64:
+	GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
+
+victoria-logs-pure:
+	APP_NAME=victoria-logs $(MAKE) app-local-pure
+
+run-victoria-logs:
+	mkdir -p victoria-logs-data
+	DOCKER_OPTS='-v $(shell pwd)/victoria-logs-data:/victoria-logs-data' \
+	APP_NAME=victoria-logs \
+	ARGS='' \
+	$(MAKE) run-via-docker

app/victoria-logs/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/victoria-logs/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image
+FROM $base_image
+
+EXPOSE 9428
+
+ENTRYPOINT ["/victoria-logs-prod"]
+ARG src_binary
+COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+)
+
+var (
+	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	flag.Usage = usage
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+
+	listenAddrs := *httpListenAddrs
+	if len(listenAddrs) == 0 {
+		listenAddrs = []string{":9428"}
+	}
+	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
+	startTime := time.Now()
+
+	vlstorage.Init()
+	vlselect.Init()
+	vlinsert.Init()
+
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
+
+	pushmetrics.Init()
+	sig := procutil.WaitForSigterm()
+	logger.Infof("received signal %s", sig)
+	pushmetrics.Stop()
+
+	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	startTime = time.Now()
+	if err := httpserver.Stop(listenAddrs); err != nil {
+		logger.Fatalf("cannot stop the webservice: %s", err)
+	}
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
+
+	vlinsert.Stop()
+	vlselect.Stop()
+	vlstorage.Stop()
+
+	fs.MustStopDirRemover()
+
+	logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.URL.Path == "/" {
+		if r.Method != http.MethodGet {
+			return false
+		}
+		w.Header().Add("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/'>https://docs.victoriametrics.com/victorialogs/</a></br>")
+		fmt.Fprintf(w, "Useful endpoints:</br>")
+		httpserver.WriteAPIHelp(w, [][2]string{
+			{"select/vmui", "Web UI for VictoriaLogs"},
+			{"metrics", "available service metrics"},
+			{"flags", "command-line flags"},
+		})
+		return true
+	}
+	if vlinsert.RequestHandler(w, r) {
+		return true
+	}
+	if vlselect.RequestHandler(w, r) {
+		return true
+	}
+	return false
+}
+
+func usage() {
+	const s = `
+victoria-logs is a log management and analytics service.
+
+See the docs at https://docs.victoriametrics.com/victorialogs/
+`
+	flagutil.Usage(s)
+}

app/victoria-logs/multiarch/Dockerfile

@@ -0,0 +1,12 @@
+# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
+ARG certs_image
+ARG root_image
+FROM $certs_image as certs
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
+
+FROM $root_image
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+EXPOSE 9428
+ENTRYPOINT ["/victoria-logs-prod"]
+ARG TARGETARCH
+COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/vlagent/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlinsert/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlinsert/elasticsearch/bulk_response.qtpl

@@ -0,0 +1,20 @@
+{% stripspace %}
+
+{% func BulkResponse(n int, tookMs int64) %}
+{
+	"took":{%dl tookMs %},
+	"errors":false,
+	"items":[
+		{% for i := 0; i < n; i++ %}
+		{
+			"create":{
+				"status":201
+			}
+		}
+		{% if i+1 < n %},{% endif %}
+		{% endfor %}
+	]
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlinsert/elasticsearch/bulk_response.qtpl.go

@@ -0,0 +1,69 @@
+// Code generated by qtc from "bulk_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+package elasticsearch
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+func StreamBulkResponse(qw422016 *qt422016.Writer, n int, tookMs int64) {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
+	qw422016.N().S(`{"took":`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
+	qw422016.N().DL(tookMs)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
+	qw422016.N().S(`,"errors":false,"items":[`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
+	for i := 0; i < n; i++ {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
+		qw422016.N().S(`{"create":{"status":201}}`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
+		if i+1 < n {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
+			qw422016.N().S(`,`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
+		}
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
+	}
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
+	qw422016.N().S(`]}`)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+}
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+func WriteBulkResponse(qq422016 qtio422016.Writer, n int, tookMs int64) {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	StreamBulkResponse(qw422016, n, tookMs)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+}
+
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+func BulkResponse(n int, tookMs int64) string {
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	WriteBulkResponse(qb422016, n, tookMs)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qs422016 := string(qb422016.B)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+	return qs422016
+//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
+}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -0,0 +1,280 @@
+package elasticsearch
+
+import (
+	"bufio"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+)
+
+var (
+	elasticsearchVersion = flag.String("elasticsearch.version", "8.9.0", "Elasticsearch version to report to client")
+)
+
+// RequestHandler processes Elasticsearch insert requests
+func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
+	w.Header().Add("Content-Type", "application/json")
+	// This header is needed for Logstash
+	w.Header().Set("X-Elastic-Product", "Elasticsearch")
+
+	if strings.HasPrefix(path, "/_ilm/policy") {
+		// Return fake response for Elasticsearch ilm request.
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	if strings.HasPrefix(path, "/_index_template") {
+		// Return fake response for Elasticsearch index template request.
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	if strings.HasPrefix(path, "/_ingest") {
+		// Return fake response for Elasticsearch ingest pipeline request.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	if strings.HasPrefix(path, "/_nodes") {
+		// Return fake response for Elasticsearch nodes discovery request.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
+	switch path {
+	case "/":
+		switch r.Method {
+		case http.MethodGet:
+			// Return fake response for Elasticsearch ping request.
+			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
+			fmt.Fprintf(w, `{
+			"version": {
+				"number": %q
+			}
+		}`, *elasticsearchVersion)
+		case http.MethodHead:
+			// Return empty response for Logstash ping request.
+		}
+
+		return true
+	case "/_license":
+		// Return fake response for Elasticsearch license request.
+		fmt.Fprintf(w, `{
+			"license": {
+				"uid": "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
+				"type": "oss",
+				"status": "active",
+				"expiry_date_in_millis" : 4000000000000
+			}
+		}`)
+		return true
+	case "/_bulk":
+		startTime := time.Now()
+		bulkRequestsTotal.Inc()
+
+		cp, err := insertutils.GetCommonParams(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		if err := vlstorage.CanWriteData(); err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+		processLogMessage := cp.GetProcessLogMessageFunc(lr)
+		isGzip := r.Header.Get("Content-Encoding") == "gzip"
+		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
+		vlstorage.MustAddRows(lr)
+		logstorage.PutLogRows(lr)
+		if err != nil {
+			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
+			return true
+		}
+
+		tookMs := time.Since(startTime).Milliseconds()
+		bw := bufferedwriter.Get(w)
+		defer bufferedwriter.Put(bw)
+		WriteBulkResponse(bw, n, tookMs)
+		_ = bw.Flush()
+
+		// update bulkRequestDuration only for successfully parsed requests
+		// There is no need in updating bulkRequestDuration for request errors,
+		// since their timings are usually much smaller than the timing for successful request parsing.
+		bulkRequestDuration.UpdateDuration(startTime)
+
+		return true
+	default:
+		return false
+	}
+}
+
+var (
+	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
+	rowsIngestedTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
+	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
+)
+
+func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
+	processLogMessage func(timestamp int64, fields []logstorage.Field),
+) (int, error) {
+	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
+
+	if isGzip {
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
+		}
+		defer common.PutGzipReader(zr)
+		r = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(r)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
+
+	n := 0
+	nCheckpoint := 0
+	for {
+		ok, err := readBulkLine(sc, timeField, msgField, processLogMessage)
+		wcr.DecConcurrency()
+		if err != nil || !ok {
+			rowsIngestedTotal.Add(n - nCheckpoint)
+			return n, err
+		}
+		n++
+		if batchSize := n - nCheckpoint; n >= 1000 {
+			rowsIngestedTotal.Add(batchSize)
+			nCheckpoint = n
+		}
+	}
+}
+
+var lineBufferPool bytesutil.ByteBufferPool
+
+func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
+	processLogMessage func(timestamp int64, fields []logstorage.Field),
+) (bool, error) {
+	var line []byte
+
+	// Read the command, must be "create" or "index"
+	for len(line) == 0 {
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
+						insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
+		}
+		line = sc.Bytes()
+	}
+	lineStr := bytesutil.ToUnsafeString(line)
+	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
+		return false, fmt.Errorf(`unexpected command %q; expecting "create" or "index"`, line)
+	}
+
+	// Decode log message
+	if !sc.Scan() {
+		if err := sc.Err(); err != nil {
+			if errors.Is(err, bufio.ErrTooLong) {
+				return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
+			}
+			return false, err
+		}
+		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
+	}
+	line = sc.Bytes()
+	p := logstorage.GetJSONParser()
+	if err := p.ParseLogMessage(line); err != nil {
+		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
+	}
+
+	ts, err := extractTimestampFromFields(timeField, p.Fields)
+	if err != nil {
+		return false, fmt.Errorf("cannot parse timestamp: %w", err)
+	}
+	if ts == 0 {
+		ts = time.Now().UnixNano()
+	}
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	processLogMessage(ts, p.Fields)
+	logstorage.PutJSONParser(p)
+
+	return true, nil
+}
+
+func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		timestamp, err := parseElasticsearchTimestamp(f.Value)
+		if err != nil {
+			return 0, err
+		}
+		f.Value = ""
+		return timestamp, nil
+	}
+	return 0, nil
+}
+
+func parseElasticsearchTimestamp(s string) (int64, error) {
+	if s == "0" || s == "" {
+		// Special case - zero or empty timestamp must be substituted
+		// with the current time by the caller.
+		return 0, nil
+	}
+	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
+		// Try parsing timestamp in milliseconds
+		n, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
+		}
+		if n > int64(math.MaxInt64)/1e6 {
+			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+		}
+		if n < int64(math.MinInt64)/1e6 {
+			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+		}
+		n *= 1e6
+		return n, nil
+	}
+	if len(s) == len("YYYY-MM-DD") {
+		t, err := time.Parse("2006-01-02", s)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse date %q: %w", s, err)
+		}
+		return t.UnixNano(), nil
+	}
+	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
+	if !ok {
+		return 0, fmt.Errorf("cannot parse timestamp %q", s)
+	}
+	return nsecs, nil
+}

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -0,0 +1,122 @@
+package elasticsearch
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestReadBulkRequestFailure(t *testing.T) {
+	f := func(data string) {
+		t.Helper()
+
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
+		}
+
+		r := bytes.NewBufferString(data)
+		rows, err := readBulkRequest(r, false, "_time", "_msg", processLogMessage)
+		if err == nil {
+			t.Fatalf("expecting non-empty error")
+		}
+		if rows != 0 {
+			t.Fatalf("unexpected non-zero rows=%d", rows)
+		}
+	}
+	f("foobar")
+	f(`{}`)
+	f(`{"create":{}}`)
+	f(`{"creat":{}}
+{}`)
+	f(`{"create":{}}
+foobar`)
+}
+
+func TestReadBulkRequestSuccess(t *testing.T) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+			result += string(logstorage.MarshalFieldsToJSON(nil, fields)) + "\n"
+		}
+
+		// Read the request without compression
+		r := bytes.NewBufferString(data)
+		rows, err := readBulkRequest(r, false, timeField, msgField, processLogMessage)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+
+		// Read the request with compression
+		timestamps = nil
+		result = ""
+		compressedData := compressData(data)
+		r = bytes.NewBufferString(compressedData)
+		rows, err = readBulkRequest(r, true, timeField, msgField, processLogMessage)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Verify an empty data
+	f("", "_time", "_msg", 0, nil, "")
+	f("\n", "_time", "_msg", 0, nil, "")
+	f("\n\n", "_time", "_msg", 0, nil, "")
+
+	// Verify non-empty data
+	data := `{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"index":{"_index":"filebeat-8.8.0"}}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
+`
+	timeField := "@timestamp"
+	msgField := "message"
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}
+`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
+}
+
+func compressData(s string) string {
+	var bb bytes.Buffer
+	zw := gzip.NewWriter(&bb)
+	if _, err := zw.Write([]byte(s)); err != nil {
+		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
+	}
+	if err := zw.Close(); err != nil {
+		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
+	}
+	return bb.String()
+}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -0,0 +1,50 @@
+package elasticsearch
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func BenchmarkReadBulkRequest(b *testing.B) {
+	b.Run("gzip:off", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, false)
+	})
+	b.Run("gzip:on", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, true)
+	})
+}
+
+func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
+	data := `{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+{"create":{"_index":"filebeat-8.8.0"}}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"create":{"_index":"filebeat-8.8.0"}}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
+`
+	if isGzip {
+		data = compressData(data)
+	}
+	dataBytes := bytesutil.ToUnsafeBytes(data)
+
+	timeField := "@timestamp"
+	msgField := "message"
+	processLogMessage := func(_ int64, _ []logstorage.Field) {}
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(data)))
+	b.RunParallel(func(pb *testing.PB) {
+		r := &bytes.Reader{}
+		for pb.Next() {
+			r.Reset(dataBytes)
+			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
+			if err != nil {
+				panic(fmt.Errorf("unexpected error: %w", err))
+			}
+		}
+	})
+}

app/vlinsert/insertutils/common_params.go

@@ -0,0 +1,117 @@
+package insertutils
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// CommonParams contains common HTTP parameters used by log ingestion APIs.
+//
+// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
+type CommonParams struct {
+	TenantID     logstorage.TenantID
+	TimeField    string
+	MsgField     string
+	StreamFields []string
+	IgnoreFields []string
+
+	Debug           bool
+	DebugRequestURI string
+	DebugRemoteAddr string
+}
+
+// GetCommonParams returns CommonParams from r.
+func GetCommonParams(r *http.Request) (*CommonParams, error) {
+	// Extract tenantID
+	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// Extract time field name from _time_field query arg
+	var timeField = "_time"
+	if tf := r.FormValue("_time_field"); tf != "" {
+		timeField = tf
+	}
+
+	// Extract message field name from _msg_field query arg
+	var msgField = ""
+	if msgf := r.FormValue("_msg_field"); msgf != "" {
+		msgField = msgf
+	}
+
+	streamFields := httputils.GetArray(r, "_stream_fields")
+	ignoreFields := httputils.GetArray(r, "ignore_fields")
+
+	debug := httputils.GetBool(r, "debug")
+	debugRequestURI := ""
+	debugRemoteAddr := ""
+	if debug {
+		debugRequestURI = httpserver.GetRequestURI(r)
+		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
+	}
+
+	cp := &CommonParams{
+		TenantID:        tenantID,
+		TimeField:       timeField,
+		MsgField:        msgField,
+		StreamFields:    streamFields,
+		IgnoreFields:    ignoreFields,
+		Debug:           debug,
+		DebugRequestURI: debugRequestURI,
+		DebugRemoteAddr: debugRemoteAddr,
+	}
+	return cp, nil
+}
+
+// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
+	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
+	cp := &CommonParams{
+		TenantID:  tenantID,
+		TimeField: "timestamp",
+		MsgField:  "message",
+		StreamFields: []string{
+			"hostname",
+			"app_name",
+			"proc_id",
+		},
+	}
+
+	return cp
+}
+
+// GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
+func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
+	return func(timestamp int64, fields []logstorage.Field) {
+		if len(fields) > *MaxFieldsPerLine {
+			rf := logstorage.RowFormatter(fields)
+			logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+			rowsDroppedTotalTooManyFields.Inc()
+			return
+		}
+
+		lr.MustAdd(cp.TenantID, timestamp, fields)
+		if cp.Debug {
+			s := lr.GetRowString(0)
+			lr.ResetKeepSettings()
+			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
+			rowsDroppedTotalDebug.Inc()
+			return
+		}
+		if lr.NeedFlush() {
+			vlstorage.MustAddRows(lr)
+			lr.ResetKeepSettings()
+		}
+	}
+}
+
+var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/flags.go

@@ -0,0 +1,15 @@
+package insertutils
+
+import (
+	"flag"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+)
+
+var (
+	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+
+	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
+)

app/vlinsert/insertutils/timestamp.go

@@ -0,0 +1,33 @@
+package insertutils
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ExtractTimestampISO8601FromFields extracts ISO8601 timestamp in nanoseconds from the field with the name timeField at fields.
+//
+// The value for the timeField is set to empty string after returning from the function,
+// so it could be ignored during data ingestion.
+//
+// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
+func ExtractTimestampISO8601FromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		nsecs, ok := logstorage.TryParseTimestampISO8601(f.Value)
+		if !ok {
+			if f.Value == "0" || f.Value == "" {
+				return time.Now().UnixNano(), nil
+			}
+			return time.Now().UnixNano(), fmt.Errorf("cannot unmarshal iso8601 timestamp from %s=%q", timeField, f.Value)
+		}
+		f.Value = ""
+		return nsecs, nil
+	}
+	return time.Now().UnixNano(), nil
+}

app/vlinsert/jsonline/jsonline.go

@@ -0,0 +1,130 @@
+package jsonline
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// RequestHandler processes jsonline insert requests
+func RequestHandler(w http.ResponseWriter, r *http.Request) {
+	startTime := time.Now()
+	w.Header().Add("Content-Type", "application/json")
+
+	if r.Method != "POST" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	requestsTotal.Inc()
+
+	cp, err := insertutils.GetCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			logger.Errorf("cannot read gzipped jsonline request: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
+
+	n := 0
+	for {
+		ok, err := readLine(sc, cp.TimeField, cp.MsgField, processLogMessage)
+		wcr.DecConcurrency()
+		if err != nil {
+			errorsTotal.Inc()
+			logger.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
+			break
+		}
+		if !ok {
+			break
+		}
+		n++
+		rowsIngestedTotal.Inc()
+	}
+
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+
+	// update requestDuration only for successfully parsed requests.
+	// There is no need in updating requestDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	requestDuration.UpdateDuration(startTime)
+}
+
+func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
+	var line []byte
+	for len(line) == 0 {
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
+		}
+		line = sc.Bytes()
+	}
+
+	p := logstorage.GetJSONParser()
+	if err := p.ParseLogMessage(line); err != nil {
+		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
+	}
+	ts, err := insertutils.ExtractTimestampISO8601FromFields(timeField, p.Fields)
+	if err != nil {
+		return false, fmt.Errorf("cannot get timestamp: %w", err)
+	}
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	processLogMessage(ts, p.Fields)
+	logstorage.PutJSONParser(p)
+
+	return true, nil
+}
+
+var lineBufferPool bytesutil.ByteBufferPool
+
+var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+
+	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
+	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
+
+	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
+)

app/vlinsert/jsonline/jsonline_test.go

@@ -0,0 +1,61 @@
+package jsonline
+
+import (
+	"bufio"
+	"bytes"
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestReadLine_Success(t *testing.T) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+			result += string(logstorage.MarshalFieldsToJSON(nil, fields)) + "\n"
+		}
+
+		r := bytes.NewBufferString(data)
+		sc := bufio.NewScanner(r)
+		rows := 0
+		for {
+			ok, err := readLine(sc, timeField, msgField, processLogMessage)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if !ok {
+				break
+			}
+			rows++
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
+`
+	timeField := "@timestamp"
+	msgField := "message"
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}
+`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
+}

app/vlinsert/loki/loki.go

@@ -0,0 +1,59 @@
+package loki
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// RequestHandler processes Loki insert requests
+func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
+	switch path {
+	case "/api/v1/push":
+		handleInsert(r, w)
+		return true
+	case "/ready":
+		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ready"))
+		return true
+	default:
+		return false
+	}
+}
+
+// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
+func handleInsert(r *http.Request, w http.ResponseWriter) {
+	contentType := r.Header.Get("Content-Type")
+	switch contentType {
+	case "application/json":
+		handleJSON(r, w)
+	default:
+		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
+		handleProtobuf(r, w)
+	}
+}
+
+func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
+	cp, err := insertutils.GetCommonParams(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// If parsed tenant is (0,0) it is likely to be default tenant
+	// Try parsing tenant from Loki headers
+	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
+		org := r.Header.Get("X-Scope-OrgID")
+		if org != "" {
+			tenantID, err := logstorage.GetTenantIDFromString(org)
+			if err != nil {
+				return nil, err
+			}
+			cp.TenantID = tenantID
+		}
+
+	}
+
+	return cp, nil
+}

app/vlinsert/loki/loki_json.go

@@ -0,0 +1,203 @@
+package loki
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
+)
+
+var parserPool fastjson.ParserPool
+
+func handleJSON(r *http.Request, w http.ResponseWriter) {
+	startTime := time.Now()
+	requestsJSONTotal.Inc()
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
+
+	cp, err := getCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
+		return
+	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+	n, err := parseJSONRequest(data, processLogMessage)
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
+		return
+	}
+
+	rowsIngestedJSONTotal.Add(n)
+
+	// update requestJSONDuration only for successfully parsed requests
+	// There is no need in updating requestJSONDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	requestJSONDuration.UpdateDuration(startTime)
+}
+
+var (
+	requestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
+	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
+	requestJSONDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
+)
+
+func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+	p := parserPool.Get()
+	defer parserPool.Put(p)
+	v, err := p.ParseBytes(data)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
+	}
+
+	streamsV := v.Get("streams")
+	if streamsV == nil {
+		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
+	}
+	streams, err := streamsV.Array()
+	if err != nil {
+		return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
+	}
+
+	currentTimestamp := time.Now().UnixNano()
+	var commonFields []logstorage.Field
+	rowsIngested := 0
+	for _, stream := range streams {
+		// populate common labels from `stream` dict
+		commonFields = commonFields[:0]
+		labelsV := stream.Get("stream")
+		var labels *fastjson.Object
+		if labelsV != nil {
+			o, err := labelsV.Object()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
+			}
+			labels = o
+		}
+		labels.Visit(func(k []byte, v *fastjson.Value) {
+			if err != nil {
+				return
+			}
+			vStr, errLocal := v.StringBytes()
+			if errLocal != nil {
+				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
+				return
+			}
+			commonFields = append(commonFields, logstorage.Field{
+				Name:  bytesutil.ToUnsafeString(k),
+				Value: bytesutil.ToUnsafeString(vStr),
+			})
+		})
+		if err != nil {
+			return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
+		}
+
+		// populate messages from `values` array
+		linesV := stream.Get("values")
+		if linesV == nil {
+			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
+		}
+		lines, err := linesV.Array()
+		if err != nil {
+			return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
+		}
+
+		fields := commonFields
+		for _, line := range lines {
+			lineA, err := line.Array()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
+			}
+			if len(lineA) != 2 {
+				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
+			}
+
+			// parse timestamp
+			timestamp, err := lineA[0].StringBytes()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
+			}
+			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
+			if err != nil {
+				return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
+			}
+			if ts == 0 {
+				ts = currentTimestamp
+			}
+
+			// parse log message
+			msg, err := lineA[1].StringBytes()
+			if err != nil {
+				return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
+			}
+
+			fields = append(fields[:len(commonFields)], logstorage.Field{
+				Name:  "_msg",
+				Value: bytesutil.ToUnsafeString(msg),
+			})
+			processLogMessage(ts, fields)
+		}
+		rowsIngested += len(lines)
+	}
+
+	return rowsIngested, nil
+}
+
+func parseLokiTimestamp(s string) (int64, error) {
+	if s == "" {
+		// Special case - an empty timestamp must be substituted with the current time by the caller.
+		return 0, nil
+	}
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		// Fall back to parsing floating-point value
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return 0, err
+		}
+		if f > math.MaxInt64 {
+			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
+		}
+		if f < math.MinInt64 {
+			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
+		}
+		n = int64(f)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
+	}
+	return n, nil
+}

app/vlinsert/loki/loki_json_test.go

@@ -0,0 +1,130 @@
+package loki
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestParseJSONRequestFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
+			t.Fatalf("unexpected call to parseJSONRequest callback!")
+		})
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if n != 0 {
+			t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
+		}
+	}
+	f(``)
+
+	// Invalid json
+	f(`{}`)
+	f(`[]`)
+	f(`"foo"`)
+	f(`123`)
+
+	// invalid type for `streams` item
+	f(`{"streams":123}`)
+
+	// Missing `values` item
+	f(`{"streams":[{}]}`)
+
+	// Invalid type for `values` item
+	f(`{"streams":[{"values":"foobar"}]}`)
+
+	// Invalid type for `stream` item
+	f(`{"streams":[{"stream":[],"values":[]}]}`)
+
+	// Invalid type for `values` individual item
+	f(`{"streams":[{"values":[123]}]}`)
+
+	// Invalid length of `values` individual item
+	f(`{"streams":[{"values":[[]]}]}`)
+	f(`{"streams":[{"values":[["123"]]}]}`)
+	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
+
+	// Invalid type for timestamp inside `values` individual item
+	f(`{"streams":[{"values":[[123,"456"]}]}`)
+
+	// Invalid type for log message
+	f(`{"streams":[{"values":[["123",1234]]}]}`)
+}
+
+func TestParseJSONRequestSuccess(t *testing.T) {
+	f := func(s string, resultExpected string) {
+		t.Helper()
+		var lines []string
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+			var a []string
+			for _, f := range fields {
+				a = append(a, f.String())
+			}
+			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
+			lines = append(lines, line)
+		})
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if n != len(lines) {
+			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
+		}
+		result := strings.Join(lines, "\n")
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Empty streams
+	f(`{"streams":[]}`, ``)
+	f(`{"streams":[{"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+
+	// Empty stream labels
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+
+	// Non-empty stream labels
+	f(`{"streams":[{"stream":{
+	"label1": "value1",
+	"label2": "value2"
+},"values":[
+	["1577836800000000001", "foo bar"],
+	["1477836900005000002", "abc"],
+	["147.78369e9", "foobar"]
+]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
+_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
+_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+
+	// Multiple streams
+	f(`{
+	"streams": [
+		{
+			"stream": {
+				"foo": "bar",
+				"a": "b"
+			},
+			"values": [
+				["1577836800000000001", "foo bar"],
+				["1577836900005000002", "abc"]
+			]
+		},
+		{
+			"stream": {
+				"x": "y"
+			},
+			"values": [
+				["1877836900005000002", "yx"]
+			]
+		}
+	]
+}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
+_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
+_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}

app/vlinsert/loki/loki_json_timing_test.go

@@ -0,0 +1,78 @@
+package loki
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func BenchmarkParseJSONRequest(b *testing.B) {
+	for _, streams := range []int{5, 10} {
+		for _, rows := range []int{100, 1000} {
+			for _, labels := range []int{10, 50} {
+				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
+					benchmarkParseJSONRequest(b, streams, rows, labels)
+				})
+			}
+		}
+	}
+}
+
+func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
+	b.ReportAllocs()
+	b.SetBytes(int64(streams * rows))
+	b.RunParallel(func(pb *testing.PB) {
+		data := getJSONBody(streams, rows, labels)
+		for pb.Next() {
+			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
+			if err != nil {
+				panic(fmt.Errorf("unexpected error: %w", err))
+			}
+		}
+	})
+}
+
+func getJSONBody(streams, rows, labels int) []byte {
+	body := append([]byte{}, `{"streams":[`...)
+	now := time.Now().UnixNano()
+	valuePrefix := fmt.Sprintf(`["%d","value_`, now)
+
+	for i := 0; i < streams; i++ {
+		body = append(body, `{"stream":{`...)
+
+		for j := 0; j < labels; j++ {
+			body = append(body, `"label_`...)
+			body = strconv.AppendInt(body, int64(j), 10)
+			body = append(body, `":"value_`...)
+			body = strconv.AppendInt(body, int64(j), 10)
+			body = append(body, '"')
+			if j < labels-1 {
+				body = append(body, ',')
+			}
+
+		}
+		body = append(body, `}, "values":[`...)
+
+		for j := 0; j < rows; j++ {
+			body = append(body, valuePrefix...)
+			body = strconv.AppendInt(body, int64(j), 10)
+			body = append(body, `"]`...)
+			if j < rows-1 {
+				body = append(body, ',')
+			}
+		}
+
+		body = append(body, `]}`...)
+		if i < streams-1 {
+			body = append(body, ',')
+		}
+
+	}
+
+	body = append(body, `]}`...)
+
+	return body
+}

app/vlinsert/loki/loki_protobuf.go

@@ -0,0 +1,187 @@
+package loki
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+)
+
+var (
+	bytesBufPool bytesutil.ByteBufferPool
+	pushReqsPool sync.Pool
+)
+
+func handleProtobuf(r *http.Request, w http.ResponseWriter) {
+	startTime := time.Now()
+	requestsProtobufTotal.Inc()
+	wcr := writeconcurrencylimiter.GetReader(r.Body)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
+
+	cp, err := getCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
+		return
+	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+	n, err := parseProtobufRequest(data, processLogMessage)
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
+		return
+	}
+
+	rowsIngestedProtobufTotal.Add(n)
+
+	// update requestProtobufDuration only for successfully parsed requests
+	// There is no need in updating requestProtobufDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	requestProtobufDuration.UpdateDuration(startTime)
+}
+
+var (
+	requestsProtobufTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
+	requestProtobufDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
+)
+
+func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
+	bb := bytesBufPool.Get()
+	defer bytesBufPool.Put(bb)
+
+	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
+	if err != nil {
+		return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
+	}
+	bb.B = buf
+
+	req := getPushRequest()
+	defer putPushRequest(req)
+
+	err = req.Unmarshal(bb.B)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse request body: %w", err)
+	}
+
+	var commonFields []logstorage.Field
+	rowsIngested := 0
+	streams := req.Streams
+	currentTimestamp := time.Now().UnixNano()
+	for i := range streams {
+		stream := &streams[i]
+		// st.Labels contains labels for the stream.
+		// Labels are same for all entries in the stream.
+		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
+		if err != nil {
+			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
+		}
+		fields := commonFields
+
+		entries := stream.Entries
+		for j := range entries {
+			entry := &entries[j]
+			fields = append(fields[:len(commonFields)], logstorage.Field{
+				Name:  "_msg",
+				Value: entry.Line,
+			})
+			ts := entry.Timestamp.UnixNano()
+			if ts == 0 {
+				ts = currentTimestamp
+			}
+			processLogMessage(ts, fields)
+		}
+		rowsIngested += len(stream.Entries)
+	}
+	return rowsIngested, nil
+}
+
+// parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
+//
+// See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
+func parsePromLabels(dst []logstorage.Field, s string) ([]logstorage.Field, error) {
+	// Make sure s is wrapped into `{...}`
+	s = strings.TrimSpace(s)
+	if len(s) < 2 {
+		return nil, fmt.Errorf("too short string to parse: %q", s)
+	}
+	if s[0] != '{' {
+		return nil, fmt.Errorf("missing `{` at the beginning of %q", s)
+	}
+	if s[len(s)-1] != '}' {
+		return nil, fmt.Errorf("missing `}` at the end of %q", s)
+	}
+	s = s[1 : len(s)-1]
+
+	for len(s) > 0 {
+		// Parse label name
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("cannot find `=` char for label value at %s", s)
+		}
+		name := s[:n]
+		s = s[n+1:]
+
+		// Parse label value
+		qs, err := strconv.QuotedPrefix(s)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse value for label %q at %s: %w", name, s, err)
+		}
+		s = s[len(qs):]
+		value, err := strconv.Unquote(qs)
+		if err != nil {
+			return nil, fmt.Errorf("cannot unquote value %q for label %q: %w", qs, name, err)
+		}
+
+		// Append the found field to dst.
+		dst = append(dst, logstorage.Field{
+			Name:  name,
+			Value: value,
+		})
+
+		// Check whether there are other labels remaining
+		if len(s) == 0 {
+			break
+		}
+		if !strings.HasPrefix(s, ",") {
+			return nil, fmt.Errorf("missing `,` char at %s", s)
+		}
+		s = s[1:]
+		s = strings.TrimPrefix(s, " ")
+	}
+	return dst, nil
+}
+
+func getPushRequest() *PushRequest {
+	v := pushReqsPool.Get()
+	if v == nil {
+		return &PushRequest{}
+	}
+	return v.(*PushRequest)
+}
+
+func putPushRequest(req *PushRequest) {
+	req.Reset()
+	pushReqsPool.Put(req)
+}

app/vlinsert/loki/loki_protobuf_test.go

@@ -0,0 +1,171 @@
+package loki
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/golang/snappy"
+)
+
+func TestParseProtobufRequestSuccess(t *testing.T) {
+	f := func(s string, resultExpected string) {
+		t.Helper()
+		var pr PushRequest
+		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
+			msg := ""
+			for _, f := range fields {
+				if f.Name == "_msg" {
+					msg = f.Value
+				}
+			}
+			var a []string
+			for _, f := range fields {
+				if f.Name == "_msg" {
+					continue
+				}
+				item := fmt.Sprintf("%s=%q", f.Name, f.Value)
+				a = append(a, item)
+			}
+			labels := "{" + strings.Join(a, ", ") + "}"
+			pr.Streams = append(pr.Streams, Stream{
+				Labels: labels,
+				Entries: []Entry{
+					{
+						Timestamp: time.Unix(0, timestamp),
+						Line:      msg,
+					},
+				},
+			})
+		})
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if n != len(pr.Streams) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(pr.Streams), n)
+		}
+
+		data, err := pr.Marshal()
+		if err != nil {
+			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
+		}
+		encodedData := snappy.Encode(nil, data)
+
+		var lines []string
+		n, err = parseProtobufRequest(encodedData, func(timestamp int64, fields []logstorage.Field) {
+			var a []string
+			for _, f := range fields {
+				a = append(a, f.String())
+			}
+			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
+			lines = append(lines, line)
+		})
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if n != len(lines) {
+			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
+		}
+		result := strings.Join(lines, "\n")
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	// Empty streams
+	f(`{"streams":[]}`, ``)
+	f(`{"streams":[{"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
+	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
+
+	// Empty stream labels
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
+
+	// Non-empty stream labels
+	f(`{"streams":[{"stream":{
+	"label1": "value1",
+	"label2": "value2"
+},"values":[
+	["1577836800000000001", "foo bar"],
+	["1477836900005000002", "abc"],
+	["147.78369e9", "foobar"]
+]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
+_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
+_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
+
+	// Multiple streams
+	f(`{
+	"streams": [
+		{
+			"stream": {
+				"foo": "bar",
+				"a": "b"
+			},
+			"values": [
+				["1577836800000000001", "foo bar"],
+				["1577836900005000002", "abc"]
+			]
+		},
+		{
+			"stream": {
+				"x": "y"
+			},
+			"values": [
+				["1877836900005000002", "yx"]
+			]
+		}
+	]
+}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
+_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
+_time:1877836900005000002 "x":"y" "_msg":"yx"`)
+}
+
+func TestParsePromLabelsSuccess(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		fields, err := parsePromLabels(nil, s)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		var a []string
+		for _, f := range fields {
+			a = append(a, fmt.Sprintf("%s=%q", f.Name, f.Value))
+		}
+		result := "{" + strings.Join(a, ", ") + "}"
+		if result != s {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, s)
+		}
+	}
+
+	f("{}")
+	f(`{foo="bar"}`)
+	f(`{foo="bar", baz="x", y="z"}`)
+	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
+}
+
+func TestParsePromLabelsFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		fields, err := parsePromLabels(nil, s)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if len(fields) > 0 {
+			t.Fatalf("unexpected non-empty fields: %s", fields)
+		}
+	}
+
+	f("")
+	f("{")
+	f(`{foo}`)
+	f(`{foo=bar}`)
+	f(`{foo="bar}`)
+	f(`{foo="ba\",r}`)
+	f(`{foo="bar" baz="aa"}`)
+	f(`foobar`)
+	f(`foo{bar="baz"}`)
+}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -0,0 +1,66 @@
+package loki
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func BenchmarkParseProtobufRequest(b *testing.B) {
+	for _, streams := range []int{5, 10} {
+		for _, rows := range []int{100, 1000} {
+			for _, labels := range []int{10, 50} {
+				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
+					benchmarkParseProtobufRequest(b, streams, rows, labels)
+				})
+			}
+		}
+	}
+}
+
+func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
+	b.ReportAllocs()
+	b.SetBytes(int64(streams * rows))
+	b.RunParallel(func(pb *testing.PB) {
+		body := getProtobufBody(streams, rows, labels)
+		for pb.Next() {
+			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
+			if err != nil {
+				panic(fmt.Errorf("unexpected error: %w", err))
+			}
+		}
+	})
+}
+
+func getProtobufBody(streams, rows, labels int) []byte {
+	var pr PushRequest
+
+	for i := 0; i < streams; i++ {
+		var st Stream
+
+		st.Labels = `{`
+		for j := 0; j < labels; j++ {
+			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
+			if j < labels-1 {
+				st.Labels += `,`
+			}
+		}
+		st.Labels += `}`
+
+		for j := 0; j < rows; j++ {
+			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
+		}
+
+		pr.Streams = append(pr.Streams, st)
+	}
+
+	body, _ := pr.Marshal()
+	encodedBody := snappy.Encode(nil, body)
+
+	return encodedBody
+}

app/vlinsert/loki/push_request.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package logproto;
+
+import "gogoproto/gogo.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
+
+message PushRequest {
+  repeated StreamAdapter streams = 1 [
+    (gogoproto.jsontag) = "streams",
+    (gogoproto.customtype) = "Stream"
+  ];
+}
+
+message StreamAdapter {
+  string labels = 1 [(gogoproto.jsontag) = "labels"];
+  repeated EntryAdapter entries = 2 [
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "entries"
+  ];
+  // hash contains the original hash of the stream.
+  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
+}
+
+message EntryAdapter {
+  google.protobuf.Timestamp timestamp = 1 [
+    (gogoproto.stdtime) = true,
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "ts"
+  ];
+  string line = 2 [(gogoproto.jsontag) = "line"];
+}

app/vlinsert/loki/timestamp.go

@@ -0,0 +1,110 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+)
+
+const (
+	// Seconds field of the earliest valid Timestamp.
+	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	minValidSeconds = -62135596800
+	// Seconds field just after the latest valid Timestamp.
+	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	maxValidSeconds = 253402300800
+)
+
+// validateTimestamp determines whether a Timestamp is valid.
+// A valid timestamp represents a time in the range
+// [0001-01-01, 10000-01-01) and has a Nanos field
+// in the range [0, 1e9).
+//
+// If the Timestamp is valid, validateTimestamp returns nil.
+// Otherwise, it returns an error that describes
+// the problem.
+//
+// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
+func validateTimestamp(ts *types.Timestamp) error {
+	if ts == nil {
+		return errors.New("timestamp: nil Timestamp")
+	}
+	if ts.Seconds < minValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
+	}
+	if ts.Seconds >= maxValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
+	}
+	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
+		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
+	}
+	return nil
+}
+
+// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
+// but avoids the escape incurred by using fmt.Sprintf, eliminating
+// unnecessary heap allocations.
+func formatTimestamp(ts *types.Timestamp) string {
+	if ts == nil {
+		return "nil"
+	}
+
+	seconds := strconv.FormatInt(ts.Seconds, 10)
+	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
+	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
+}
+
+func sizeOfStdTime(t time.Time) int {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0
+	}
+	return ts.Size()
+}
+
+func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0, err
+	}
+	return ts.MarshalTo(data)
+}
+
+func stdTimeUnmarshal(t *time.Time, data []byte) error {
+	ts := &types.Timestamp{}
+	if err := ts.Unmarshal(data); err != nil {
+		return err
+	}
+	tt, err := timestampFromProto(ts)
+	if err != nil {
+		return err
+	}
+	*t = tt
+	return nil
+}
+
+func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
+	// Don't return the zero value on error, because corresponds to a valid
+	// timestamp. Instead return whatever time.Unix gives us.
+	var t time.Time
+	if ts == nil {
+		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
+	} else {
+		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	}
+	return t, validateTimestamp(ts)
+}
+
+func timestampProto(t time.Time) (types.Timestamp, error) {
+	ts := types.Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.Nanosecond()),
+	}
+	return ts, validateTimestamp(&ts)
+}

app/vlinsert/loki/types.go

@@ -0,0 +1,481 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"fmt"
+	"io"
+	"time"
+)
+
+// Stream contains a unique labels set as a string and a set of entries for it.
+// We are not using the proto generated version but this custom one so that we
+// can improve serialization see benchmark.
+type Stream struct {
+	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
+	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
+	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
+}
+
+// Entry is a log entry with a timestamp.
+type Entry struct {
+	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
+	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Stream) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Hash != 0 {
+		i = encodeVarintPush(dAtA, i, m.Hash)
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Entries) > 0 {
+		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintPush(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		i -= len(m.Labels)
+		copy(dAtA[i:], m.Labels)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Entry) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Line) > 0 {
+		i -= len(m.Line)
+		copy(dAtA[i:], m.Line)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
+		i--
+		dAtA[i] = 0x12
+	}
+	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
+	if err7 != nil {
+		return 0, err7
+	}
+	i -= n7
+	i = encodeVarintPush(dAtA, i, uint64(n7))
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Stream) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Labels = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Entries = append(m.Entries, Entry{})
+			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
+			}
+			m.Hash = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Hash |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Entry) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Line = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Size returns the size of the serialized Stream.
+func (m *Stream) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Labels)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	if len(m.Entries) > 0 {
+		for _, e := range m.Entries {
+			l = e.Size()
+			n += 1 + l + sovPush(uint64(l))
+		}
+	}
+	if m.Hash != 0 {
+		n += 1 + sovPush(m.Hash)
+	}
+	return n
+}
+
+// Size returns the size of the serialized Entry
+func (m *Entry) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = sizeOfStdTime(m.Timestamp)
+	n += 1 + l + sovPush(uint64(l))
+	l = len(m.Line)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	return n
+}
+
+// Equal returns true if the two Streams are equal.
+func (m *Stream) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Stream)
+	if !ok {
+		that2, ok := that.(Stream)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if m.Labels != that1.Labels {
+		return false
+	}
+	if len(m.Entries) != len(that1.Entries) {
+		return false
+	}
+	for i := range m.Entries {
+		if !m.Entries[i].Equal(that1.Entries[i]) {
+			return false
+		}
+	}
+	return m.Hash == that1.Hash
+}
+
+// Equal returns true if the two Entries are equal.
+func (m *Entry) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Entry)
+	if !ok {
+		that2, ok := that.(Entry)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if !m.Timestamp.Equal(that1.Timestamp) {
+		return false
+	}
+	if m.Line != that1.Line {
+		return false
+	}
+	return true
+}

app/vlinsert/main.go

@@ -0,0 +1,47 @@
+package vlinsert
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
+)
+
+// Init initializes vlinsert
+func Init() {
+	syslog.MustInit()
+}
+
+// Stop stops vlinsert
+func Stop() {
+	syslog.MustStop()
+}
+
+// RequestHandler handles insert requests for VictoriaLogs
+func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := r.URL.Path
+	if !strings.HasPrefix(path, "/insert/") {
+		// Skip requests, which do not start with /insert/, since these aren't our requests.
+		return false
+	}
+	path = strings.TrimPrefix(path, "/insert")
+	path = strings.ReplaceAll(path, "//", "/")
+
+	if path == "/jsonline" {
+		jsonline.RequestHandler(w, r)
+		return true
+	}
+	switch {
+	case strings.HasPrefix(path, "/elasticsearch/"):
+		path = strings.TrimPrefix(path, "/elasticsearch")
+		return elasticsearch.RequestHandler(path, w, r)
+	case strings.HasPrefix(path, "/loki/"):
+		path = strings.TrimPrefix(path, "/loki")
+		return loki.RequestHandler(path, w, r)
+	default:
+		return false
+	}
+}

app/vlinsert/syslog/syslog.go

@@ -0,0 +1,389 @@
+package syslog
+
+import (
+	"bufio"
+	"crypto/tls"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/klauspost/compress/gzip"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	syslogTenantID = flag.String("syslog.tenantID", "0:0", "TenantID for logs ingested via Syslog protocol. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
+		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	listenAddrTCP = flag.String("syslog.listenAddr.tcp", "", "Optional TCP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	listenAddrUDP = flag.String("syslog.listenAddr.udp", "", "Optional UDP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	tlsEnable = flag.Bool("syslog.tls", false, "Whether to use TLS for receiving syslog messages at -syslog.listenAddr.tcp. -syslog.tlsCertFile and -syslog.tlsKeyFile must be set "+
+		"if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	tlsCertFile = flag.String("syslog.tlsCertFile", "", "Path to file with TLS certificate for -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	tlsKeyFile = flag.String("syslog.tlsKeyFile", "", "Path to file with TLS key for  -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"The provided key file is automatically re-read every second, so it can be dynamically updated")
+	tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+
+		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+
+		"Supported values: TLS10, TLS11, TLS12, TLS13. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+
+	compressMethod = flag.String("syslog.compressMethod", "", "Compression method for syslog messages received at -syslog.listenAddr.tcp and -syslog.listenAddr.udp. "+
+		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+)
+
+// MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports
+//
+// This function must be called after flag.Parse().
+//
+// MustStop() must be called in order to free up resources occupied by the initialized syslog parser.
+func MustInit() {
+	if workersStopCh != nil {
+		logger.Panicf("BUG: MustInit() called twice without MustStop() call")
+	}
+	workersStopCh = make(chan struct{})
+
+	tenantID, err := logstorage.GetTenantIDFromString(*syslogTenantID)
+	if err != nil {
+		logger.Fatalf("cannot parse -syslog.tenantID=%q: %s", *syslogTenantID, err)
+	}
+	globalTenantID = tenantID
+
+	switch *compressMethod {
+	case "", "none", "gzip", "deflate":
+	default:
+		logger.Fatalf("unexpected -syslog.compressLevel=%q; supported values: none, gzip, deflate", *compressMethod)
+	}
+
+	if *listenAddrTCP != "" {
+		workersWG.Add(1)
+		go func() {
+			runTCPListener(*listenAddrTCP)
+			workersWG.Done()
+		}()
+	}
+
+	if *listenAddrUDP != "" {
+		workersWG.Add(1)
+		go func() {
+			runUDPListener(*listenAddrUDP)
+			workersWG.Done()
+		}()
+	}
+
+	currentYear := time.Now().Year()
+	globalCurrentYear.Store(int64(currentYear))
+	workersWG.Add(1)
+	go func() {
+		ticker := time.NewTicker(time.Minute)
+		for {
+			select {
+			case <-workersStopCh:
+				ticker.Stop()
+				workersWG.Done()
+				return
+			case <-ticker.C:
+				currentYear := time.Now().Year()
+				globalCurrentYear.Store(int64(currentYear))
+			}
+		}
+	}()
+
+	if *syslogTimezone != "" {
+		tz, err := time.LoadLocation(*syslogTimezone)
+		if err != nil {
+			logger.Fatalf("cannot parse -syslog.timezone=%q: %s", *syslogTimezone, err)
+		}
+		globalTimezone = tz
+	} else {
+		globalTimezone = time.Local
+	}
+}
+
+var (
+	globalTenantID    logstorage.TenantID
+	globalCurrentYear atomic.Int64
+	globalTimezone    *time.Location
+)
+
+var (
+	workersWG     sync.WaitGroup
+	workersStopCh chan struct{}
+)
+
+// MustStop stops syslog parser initialized via MustInit()
+func MustStop() {
+	close(workersStopCh)
+	workersWG.Wait()
+	workersStopCh = nil
+}
+
+func runUDPListener(addr string) {
+	ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr)
+	if err != nil {
+		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
+	}
+
+	doneCh := make(chan struct{})
+	go func() {
+		serveUDP(ln)
+		close(doneCh)
+	}()
+
+	<-workersStopCh
+	if err := ln.Close(); err != nil {
+		logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err)
+	}
+	<-doneCh
+}
+
+func runTCPListener(addr string) {
+	var tlsConfig *tls.Config
+	if *tlsEnable {
+		tc, err := netutil.GetServerTLSConfig(*tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites)
+		if err != nil {
+			logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s",
+				*tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites, err)
+		}
+		tlsConfig = tc
+	}
+	ln, err := netutil.NewTCPListener("syslog", addr, false, tlsConfig)
+	if err != nil {
+		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
+	}
+
+	doneCh := make(chan struct{})
+	go func() {
+		serveTCP(ln)
+		close(doneCh)
+	}()
+
+	<-workersStopCh
+	if err := ln.Close(); err != nil {
+		logger.Fatalf("syslog: cannot close TCP listener at %s: %s", addr, err)
+	}
+	<-doneCh
+}
+
+func serveUDP(ln net.PacketConn) {
+	gomaxprocs := cgroup.AvailableCPUs()
+	var wg sync.WaitGroup
+	localAddr := ln.LocalAddr()
+	for i := 0; i < gomaxprocs; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cp := insertutils.GetCommonParamsForSyslog(globalTenantID)
+			var bb bytesutil.ByteBuffer
+			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
+			for {
+				bb.Reset()
+				bb.B = bb.B[:cap(bb.B)]
+				n, remoteAddr, err := ln.ReadFrom(bb.B)
+				if err != nil {
+					udpErrorsTotal.Inc()
+					var ne net.Error
+					if errors.As(err, &ne) {
+						if ne.Temporary() {
+							logger.Errorf("syslog: temporary error when listening for UDP at %q: %s", localAddr, err)
+							time.Sleep(time.Second)
+							continue
+						}
+						if strings.Contains(err.Error(), "use of closed network connection") {
+							break
+						}
+					}
+					logger.Errorf("syslog: cannot read UDP data from %s at %s: %s", remoteAddr, localAddr, err)
+					continue
+				}
+				bb.B = bb.B[:n]
+				udpRequestsTotal.Inc()
+				if err := processStream(bb.NewReader(), cp); err != nil {
+					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func serveTCP(ln net.Listener) {
+	var cm ingestserver.ConnsMap
+	cm.Init("syslog")
+
+	var wg sync.WaitGroup
+	addr := ln.Addr()
+	for {
+		c, err := ln.Accept()
+		if err != nil {
+			var ne net.Error
+			if errors.As(err, &ne) {
+				if ne.Temporary() {
+					logger.Errorf("syslog: temporary error when listening for TCP addr %q: %s", addr, err)
+					time.Sleep(time.Second)
+					continue
+				}
+				if strings.Contains(err.Error(), "use of closed network connection") {
+					break
+				}
+				logger.Fatalf("syslog: unrecoverable error when accepting TCP connections at %q: %s", addr, err)
+			}
+			logger.Fatalf("syslog: unexpected error when accepting TCP connections at %q: %s", addr, err)
+		}
+		if !cm.Add(c) {
+			_ = c.Close()
+			break
+		}
+
+		wg.Add(1)
+		go func() {
+			cp := insertutils.GetCommonParamsForSyslog(globalTenantID)
+			if err := processStream(c, cp); err != nil {
+				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
+			}
+
+			cm.Delete(c)
+			_ = c.Close()
+			wg.Done()
+		}()
+	}
+
+	cm.CloseAll(0)
+	wg.Wait()
+}
+
+// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
+func processStream(r io.Reader, cp *insertutils.CommonParams) error {
+	switch *compressMethod {
+	case "", "none":
+	case "gzip":
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read gzipped data: %w", err)
+		}
+		r = zr
+	case "deflate":
+		zr, err := common.GetZlibReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read deflated data: %w", err)
+		}
+		r = zr
+	default:
+		logger.Panicf("BUG: compressLevel=%q; supported values: none, gzip, deflate", *compressMethod)
+	}
+
+	err := processUncompressedStream(r, cp)
+
+	switch *compressMethod {
+	case "gzip":
+		zr := r.(*gzip.Reader)
+		common.PutGzipReader(zr)
+	case "deflate":
+		zr := r.(io.ReadCloser)
+		common.PutZlibReader(zr)
+	}
+
+	return err
+}
+
+func processUncompressedStream(r io.Reader, cp *insertutils.CommonParams) error {
+	if err := vlstorage.CanWriteData(); err != nil {
+		return err
+	}
+	lr := logstorage.GetLogRows(cp.StreamFields, nil)
+	processLogMessage := cp.GetProcessLogMessageFunc(lr)
+
+	wcr := writeconcurrencylimiter.GetReader(r)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
+
+	n := 0
+	for {
+		currentYear := int(globalCurrentYear.Load())
+		ok, err := readLine(sc, currentYear, globalTimezone, processLogMessage)
+		wcr.DecConcurrency()
+		if err != nil {
+			errorsTotal.Inc()
+			return fmt.Errorf("cannot read line #%d: %s", n, err)
+		}
+		if !ok {
+			break
+		}
+		n++
+		rowsIngestedTotal.Inc()
+	}
+
+	vlstorage.MustAddRows(lr)
+	logstorage.PutLogRows(lr)
+
+	return nil
+}
+
+func readLine(sc *bufio.Scanner, currentYear int, timezone *time.Location, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
+	var line []byte
+	for len(line) == 0 {
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`line size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
+		}
+		line = sc.Bytes()
+	}
+
+	p := logstorage.GetSyslogParser(currentYear, timezone)
+	lineStr := bytesutil.ToUnsafeString(line)
+	p.Parse(lineStr)
+	ts, err := insertutils.ExtractTimestampISO8601FromFields("timestamp", p.Fields)
+	if err != nil {
+		return false, fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
+	}
+	logstorage.RenameField(p.Fields, "message", "_msg")
+	processLogMessage(ts, p.Fields)
+	logstorage.PutSyslogParser(p)
+
+	return true, nil
+}
+
+var lineBufferPool bytesutil.ByteBufferPool
+
+var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
+
+	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
+
+	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
+	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
+)

app/vlinsert/syslog/syslog_test.go

@@ -0,0 +1,65 @@
+package syslog
+
+import (
+	"bufio"
+	"bytes"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestReadLine_Success(t *testing.T) {
+	f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		MustInit()
+		defer MustStop()
+
+		var timestamps []int64
+		var result string
+		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
+			timestamps = append(timestamps, timestamp)
+			result += string(logstorage.MarshalFieldsToJSON(nil, fields)) + "\n"
+		}
+
+		r := bytes.NewBufferString(data)
+		sc := bufio.NewScanner(r)
+		rows := 0
+		timezone := time.UTC
+		for {
+			ok, err := readLine(sc, currentYear, timezone, processLogMessage)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if !ok {
+				break
+			}
+			rows++
+		}
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
+		}
+
+		if !reflect.DeepEqual(timestamps, timestampsExpected) {
+			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
+		}
+	}
+
+	data := `Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
+<165>Jun  4 12:08:33 abcd systemd[345]: abc defg
+<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
+`
+	currentYear := 2023
+	rowsExpected := 3
+	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
+	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473":"iut=\"3\" eventSource=\"Application 123 = ] 56\" eventID=\"11211\"","_msg":"This is a test message with structured data."}
+`
+	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
+}

app/vlogscli/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlogsgenerator/Makefile

@@ -0,0 +1,7 @@
+# All these commands must run from repository root.
+
+vlogsgenerator:
+	APP_NAME=vlogsgenerator $(MAKE) app-local
+
+vlogsgenerator-race:
+	APP_NAME=vlogsgenerator RACE=-race $(MAKE) app-local

app/vlogsgenerator/README.md

@@ -1 +1,158 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).
+# vlogsgenerator
+
+Logs generator for [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
+
+## How to build vlogsgenerator?
+
+Run `make vlogsgenerator` from the repository root. This builds `bin/vlogsgenerator` binary.
+
+## How run vlogsgenerator?
+
+`vlogsgenerator` generates logs in [JSON line format](https://jsonlines.org/) suitable for the ingestion
+via [`/insert/jsonline` endpoint at VictoriaLogs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api).
+
+By default it writes the generated logs into `stdout`. For example, the following command writes generated logs to `stdout`:
+
+```
+bin/vlogsgenerator
+```
+
+It is possible to redirect the generated logs to file. For example, the following command writes the generated logs to `logs.json` file:
+
+```
+bin/vlogsgenerator > logs.json
+```
+
+The generated logs at `logs.json` file can be inspected with the following command:
+
+```
+head logs.json | jq .
+```
+
+Below is an example output:
+
+```json
+{
+  "_time": "2024-05-08T14:34:00.854Z",
+  "_msg": "message for the stream 8 and worker 0; ip=185.69.136.129; uuid=b4fe8f1a-c93c-dea3-ba11-5b9f0509291e; u64=8996587920687045253",
+  "host": "host_8",
+  "worker_id": "0",
+  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
+  "const_0": "some value 0 8",
+  "const_1": "some value 1 8",
+  "const_2": "some value 2 8",
+  "var_0": "some value 0 12752539384823438260",
+  "dict_0": "warn",
+  "dict_1": "info",
+  "u8_0": "6",
+  "u16_0": "35202",
+  "u32_0": "1964973739",
+  "u64_0": "4810489083243239145",
+  "float_0": "1.868",
+  "ip_0": "250.34.75.125",
+  "timestamp_0": "1799-03-16T01:34:18.311Z",
+  "json_0": "{\"foo\":\"bar_3\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":32}"
+}
+{
+  "_time": "2024-05-08T14:34:00.854Z",
+  "_msg": "message for the stream 9 and worker 0; ip=164.244.254.194; uuid=7e8373b1-ce0d-1ce7-8e96-4bcab8955598; u64=13949903463741076522",
+  "host": "host_9",
+  "worker_id": "0",
+  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
+  "const_0": "some value 0 9",
+  "const_1": "some value 1 9",
+  "const_2": "some value 2 9",
+  "var_0": "some value 0 5371555382075206134",
+  "dict_0": "INFO",
+  "dict_1": "FATAL",
+  "u8_0": "219",
+  "u16_0": "31459",
+  "u32_0": "3918836777",
+  "u64_0": "6593354256620219850",
+  "float_0": "1.085",
+  "ip_0": "253.151.88.158",
+  "timestamp_0": "2042-10-05T16:42:57.082Z",
+  "json_0": "{\"foo\":\"bar_5\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":27}"
+}
+```
+
+The `run_id` field uniquely identifies every `vlogsgenerator` invocation.
+
+### How to write logs to VictoriaLogs?
+
+The generated logs can be written directly to VictoriaLogs by passing the address of [`/insert/jsonline` endpoint](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api)
+to `-addr` command-line flag. For example, the following command writes the generated logs to VictoriaLogs running at `localhost`:
+
+```
+bin/vlogsgenerator -addr=http://localhost:9428/insert/jsonline
+```
+
+### Configuration
+
+`vlogsgenerator` accepts various command-line flags, which can be used for configuring the number and the shape of the generated logs.
+These flags can be inspected by running `vlogsgenerator -help`. Below are the most interesting flags:
+
+* `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
+* `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
+* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to generate.
+* `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
+
+The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
+
+For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
+[log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields), where every logs stream contains `10_000` log entries,
+and writes them to `http://localhost:9428/insert/jsonline`:
+
+```
+bin/vlogsgenerator \
+  -start=2024-01-01 -end=2024-02-01 \
+  -activeStreams=100 \
+  -logsPerStream=10_000 \
+  -addr=http://localhost:9428/insert/jsonline
+```
+
+### Churn rate
+
+It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
+by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
+logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
+but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:
+
+```
+bin/vlogsgenerator \
+  -start=2024-01-01 -end=2024-02-01 \
+  -activeStreams=100 \
+  -totalStreams=1_000 \
+  -logsPerStream=10_000 \
+  -addr=http://localhost:9428/insert/jsonline
+```
+
+In this case the total number of generated logs equals to `-totalStreams` * `-logsPerStream` = `10_000_000`.
+
+### Benchmark tuning
+
+By default `vlogsgenerator` generates and writes logs by a single worker. This may limit the maximum data ingestion rate during benchmarks.
+The number of workers can be changed via `-workers` command-line flag. For example, the following command generates and writes logs with `16` workers:
+
+```
+bin/vlogsgenerator \
+  -start=2024-01-01 -end=2024-02-01 \
+  -activeStreams=100 \
+  -logsPerStream=10_000 \
+  -addr=http://localhost:9428/insert/jsonline \
+  -workers=16
+```
+
+### Output statistics
+
+Every 10 seconds `vlogsgenerator` writes statistics about the generated logs into `stderr`. The frequency of the generated statistics can be adjusted via `-statInterval` command-line flag.
+For example, the following command writes statistics every 2 seconds:
+
+```
+bin/vlogsgenerator \
+  -start=2024-01-01 -end=2024-02-01 \
+  -activeStreams=100 \
+  -logsPerStream=10_000 \
+  -addr=http://localhost:9428/insert/jsonline \
+  -statInterval=2s
+```

app/vlogsgenerator/main.go

@@ -0,0 +1,344 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+var (
+	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
+	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
+
+	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
+	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
+	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
+	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
+		"during data generation")
+	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
+	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+	jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
+
+	statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
+)
+
+func main() {
+	// Write flags and help message to stdout, since it is easier to grep or pipe.
+	flag.CommandLine.SetOutput(os.Stdout)
+	envflag.Parse()
+	buildinfo.Init()
+	logger.Init()
+
+	var remoteWriteURL *url.URL
+	if *addr != "stdout" {
+		urlParsed, err := url.Parse(*addr)
+		if err != nil {
+			logger.Fatalf("cannot parse -addr=%q: %s", *addr, err)
+		}
+		qs, err := url.ParseQuery(urlParsed.RawQuery)
+		if err != nil {
+			logger.Fatalf("cannot parse query string in -addr=%q: %w", *addr, err)
+		}
+		qs.Set("_stream_fields", "host,worker_id")
+		urlParsed.RawQuery = qs.Encode()
+		remoteWriteURL = urlParsed
+	}
+
+	if start.nsec >= end.nsec {
+		logger.Fatalf("-start=%s must be smaller than -end=%s", start, end)
+	}
+	if *activeStreams <= 0 {
+		logger.Fatalf("-activeStreams must be bigger than 0; got %d", *activeStreams)
+	}
+	if *logsPerStream <= 0 {
+		logger.Fatalf("-logsPerStream must be bigger than 0; got %d", *logsPerStream)
+	}
+	if *totalStreams < *activeStreams {
+		*totalStreams = *activeStreams
+	}
+
+	cfg := &workerConfig{
+		url:           remoteWriteURL,
+		activeStreams: *activeStreams,
+		totalStreams:  *totalStreams,
+	}
+
+	// divide total and active streams among workers
+	if *workers <= 0 {
+		logger.Fatalf("-workers must be bigger than 0; got %d", *workers)
+	}
+	if *workers > *activeStreams {
+		logger.Fatalf("-workers=%d cannot exceed -activeStreams=%d", *workers, *activeStreams)
+	}
+	cfg.activeStreams /= *workers
+	cfg.totalStreams /= *workers
+
+	logger.Infof("start -workers=%d workers for ingesting -logsPerStream=%d log entries per each -totalStreams=%d (-activeStreams=%d) on a time range -start=%s, -end=%s to -addr=%s",
+		*workers, *logsPerStream, *totalStreams, *activeStreams, toRFC3339(start.nsec), toRFC3339(end.nsec), *addr)
+
+	startTime := time.Now()
+	var wg sync.WaitGroup
+	for i := 0; i < *workers; i++ {
+		wg.Add(1)
+		go func(workerID int) {
+			defer wg.Done()
+			generateAndPushLogs(cfg, workerID)
+		}(i)
+	}
+
+	go func() {
+		prevEntries := uint64(0)
+		prevBytes := uint64(0)
+		ticker := time.NewTicker(*statInterval)
+		for range ticker.C {
+			currEntries := logEntriesCount.Load()
+			deltaEntries := currEntries - prevEntries
+			rateEntries := float64(deltaEntries) / statInterval.Seconds()
+
+			currBytes := bytesGenerated.Load()
+			deltaBytes := currBytes - prevBytes
+			rateBytes := float64(deltaBytes) / statInterval.Seconds()
+			logger.Infof("generated %dK log entries (%dK total) at %.0fK entries/sec, %dMB (%dMB total) at %.0fMB/sec",
+				deltaEntries/1e3, currEntries/1e3, rateEntries/1e3, deltaBytes/1e6, currBytes/1e6, rateBytes/1e6)
+
+			prevEntries = currEntries
+			prevBytes = currBytes
+		}
+	}()
+
+	wg.Wait()
+
+	dSecs := time.Since(startTime).Seconds()
+	currEntries := logEntriesCount.Load()
+	currBytes := bytesGenerated.Load()
+	rateEntries := float64(currEntries) / dSecs
+	rateBytes := float64(currBytes) / dSecs
+	logger.Infof("ingested %dK log entries (%dMB) in %.3f seconds; avg ingestion rate: %.0fK entries/sec, %.0fMB/sec", currEntries/1e3, currBytes/1e6, dSecs, rateEntries/1e3, rateBytes/1e6)
+}
+
+var logEntriesCount atomic.Uint64
+
+var bytesGenerated atomic.Uint64
+
+type workerConfig struct {
+	url           *url.URL
+	activeStreams int
+	totalStreams  int
+}
+
+type statWriter struct {
+	w io.Writer
+}
+
+func (sw *statWriter) Write(p []byte) (int, error) {
+	bytesGenerated.Add(uint64(len(p)))
+	return sw.w.Write(p)
+}
+
+func generateAndPushLogs(cfg *workerConfig, workerID int) {
+	pr, pw := io.Pipe()
+	sw := &statWriter{
+		w: pw,
+	}
+	bw := bufio.NewWriter(sw)
+	doneCh := make(chan struct{})
+	go func() {
+		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
+		_ = bw.Flush()
+		_ = pw.Close()
+		close(doneCh)
+	}()
+
+	if cfg.url == nil {
+		_, err := io.Copy(os.Stdout, pr)
+		if err != nil {
+			logger.Fatalf("unexpected error when writing logs to stdout: %s", err)
+		}
+		return
+	}
+
+	req, err := http.NewRequest("POST", cfg.url.String(), pr)
+	if err != nil {
+		logger.Fatalf("cannot create request to %q: %s", cfg.url, err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
+	}
+	if resp.StatusCode/100 != 2 {
+		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
+	}
+
+	// Wait until all the generateLogs goroutine is finished.
+	<-doneCh
+}
+
+func generateLogs(bw *bufio.Writer, workerID, activeStreams, totalStreams int) {
+	streamLifetime := int64(float64(end.nsec-start.nsec) * (float64(activeStreams) / float64(totalStreams)))
+	streamStep := int64(float64(end.nsec-start.nsec) / float64(totalStreams-activeStreams+1))
+	step := streamLifetime / (*logsPerStream - 1)
+
+	currNsec := start.nsec
+	for currNsec < end.nsec {
+		firstStreamID := int((currNsec - start.nsec) / streamStep)
+		generateLogsAtTimestamp(bw, workerID, currNsec, firstStreamID, activeStreams)
+		currNsec += step
+	}
+}
+
+var runID = toUUID(rand.Uint64(), rand.Uint64())
+
+func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStreamID, activeStreams int) {
+	streamID := firstStreamID
+	timeStr := toRFC3339(ts)
+	for i := 0; i < activeStreams; i++ {
+		ip := toIPv4(rand.Uint32())
+		uuid := toUUID(rand.Uint64(), rand.Uint64())
+		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
+			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
+		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
+		for j := 0; j < *constFieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"const_%d":"some value %d %d"`, j, j, streamID)
+		}
+		for j := 0; j < *varFieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"var_%d":"some value %d %d"`, j, j, rand.Uint64())
+		}
+		for j := 0; j < *dictFieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"dict_%d":"%s"`, j, dictValues[rand.Intn(len(dictValues))])
+		}
+		for j := 0; j < *u8FieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"u8_%d":"%d"`, j, uint8(rand.Uint32()))
+		}
+		for j := 0; j < *u16FieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"u16_%d":"%d"`, j, uint16(rand.Uint32()))
+		}
+		for j := 0; j < *u32FieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"u32_%d":"%d"`, j, rand.Uint32())
+		}
+		for j := 0; j < *u64FieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
+		}
+		for j := 0; j < *floatFieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
+		}
+		for j := 0; j < *ipFieldsPerLog; j++ {
+			ip := toIPv4(rand.Uint32())
+			fmt.Fprintf(bw, `,"ip_%d":"%s"`, j, ip)
+		}
+		for j := 0; j < *timestampFieldsPerLog; j++ {
+			timestamp := toISO8601(int64(rand.Uint64()))
+			fmt.Fprintf(bw, `,"timestamp_%d":"%s"`, j, timestamp)
+		}
+		for j := 0; j < *jsonFieldsPerLog; j++ {
+			fmt.Fprintf(bw, `,"json_%d":"{\"foo\":\"bar_%d\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":%d}"`, j, rand.Intn(10), rand.Intn(100))
+		}
+		fmt.Fprintf(bw, "}\n")
+
+		logEntriesCount.Add(1)
+		streamID++
+	}
+}
+
+var dictValues = []string{
+	"debug",
+	"info",
+	"warn",
+	"error",
+	"fatal",
+	"ERROR",
+	"FATAL",
+	"INFO",
+}
+
+func newTimeFlag(name, defaultValue, description string) *timeFlag {
+	var tf timeFlag
+	if err := tf.Set(defaultValue); err != nil {
+		logger.Panicf("invalid defaultValue=%q for flag %q: %w", defaultValue, name, err)
+	}
+	flag.Var(&tf, name, description)
+	return &tf
+}
+
+type timeFlag struct {
+	s    string
+	nsec int64
+}
+
+func (tf *timeFlag) Set(s string) error {
+	msec, err := promutils.ParseTimeMsec(s)
+	if err != nil {
+		return fmt.Errorf("cannot parse time from %q: %w", s, err)
+	}
+	tf.s = s
+	tf.nsec = msec * 1e6
+	return nil
+}
+
+func (tf *timeFlag) String() string {
+	return tf.s
+}
+
+func toRFC3339(nsec int64) string {
+	return time.Unix(0, nsec).UTC().Format(time.RFC3339Nano)
+}
+
+func toISO8601(nsec int64) string {
+	return time.Unix(0, nsec).UTC().Format("2006-01-02T15:04:05.000Z")
+}
+
+func toIPv4(n uint32) string {
+	dst := make([]byte, 0, len("255.255.255.255"))
+	dst = marshalUint64(dst, uint64(n>>24))
+	dst = append(dst, '.')
+	dst = marshalUint64(dst, uint64((n>>16)&0xff))
+	dst = append(dst, '.')
+	dst = marshalUint64(dst, uint64((n>>8)&0xff))
+	dst = append(dst, '.')
+	dst = marshalUint64(dst, uint64(n&0xff))
+	return string(dst)
+}
+
+func toUUID(a, b uint64) string {
+	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", a&(1<<32-1), (a>>32)&(1<<16-1), (a >> 48), b&(1<<16-1), b>>16)
+}
+
+// marshalUint64 appends string representation of n to dst and returns the result.
+func marshalUint64(dst []byte, n uint64) []byte {
+	return strconv.AppendUint(dst, n, 10)
+}

app/vlselect/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlselect/logsql/buffered_writer.go

@@ -0,0 +1,47 @@
+package logsql
+
+import (
+	"bufio"
+	"io"
+	"sync"
+)
+
+func getBufferedWriter(w io.Writer) *bufferedWriter {
+	v := bufferedWriterPool.Get()
+	if v == nil {
+		return &bufferedWriter{
+			bw: bufio.NewWriter(w),
+		}
+	}
+	bw := v.(*bufferedWriter)
+	bw.bw.Reset(w)
+	return bw
+}
+
+func putBufferedWriter(bw *bufferedWriter) {
+	bw.reset()
+	bufferedWriterPool.Put(bw)
+}
+
+var bufferedWriterPool sync.Pool
+
+type bufferedWriter struct {
+	mu sync.Mutex
+	bw *bufio.Writer
+}
+
+func (bw *bufferedWriter) reset() {
+	// nothing to do
+}
+
+func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
+	bw.mu.Lock()
+	_, _ = bw.bw.Write(p)
+	bw.mu.Unlock()
+}
+
+func (bw *bufferedWriter) FlushIgnoreErrors() {
+	bw.mu.Lock()
+	_ = bw.bw.Flush()
+	bw.mu.Unlock()
+}

app/vlselect/logsql/hits_response.qtpl

@@ -0,0 +1,69 @@
+{% import (
+	"slices"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+) %}
+
+{% stripspace %}
+
+// FieldsForHits formats labels for /select/logsql/hits response
+{% func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) %}
+{
+	{% if len(columns) > 0 %}
+		{%q= columns[0].Name %}:{%q= columns[0].Values[rowIdx] %}
+		{% for _, c := range columns[1:] %}
+			,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
+		{% endfor %}
+	{% endif %}
+}
+{% endfunc %}
+
+{% func HitsSeries(m map[string]*hitsSeries) %}
+{
+	{% code
+		sortedKeys := make([]string, 0, len(m))
+		for k := range m {
+			sortedKeys = append(sortedKeys, k)
+		}
+		slices.Sort(sortedKeys)
+	%}
+	"hits":[
+		{% if len(sortedKeys) > 0 %}
+			{%= hitsSeriesLine(m, sortedKeys[0]) %}
+			{% for _, k := range sortedKeys[1:] %}
+				,{%= hitsSeriesLine(m, k) %}
+			{% endfor %}
+		{% endif %}
+	]
+}
+{% endfunc %}
+
+{% func hitsSeriesLine(m map[string]*hitsSeries, k string) %}
+{
+	{% code
+		hs := m[k]
+		hs.sort()
+		timestamps := hs.timestamps
+		values := hs.values
+	%}
+	"fields":{%s= k %},
+	"timestamps":[
+		{% if len(timestamps) > 0 %}
+			{%q= timestamps[0] %}
+			{% for _, ts := range timestamps[1:] %}
+				,{%q= ts %}
+			{% endfor %}
+		{% endif %}
+	],
+	"values":[
+		{% if len(values) > 0 %}
+			{%s= values[0] %}
+			{% for _, v := range values[1:] %}
+				,{%s= v %}
+			{% endfor %}
+		{% endif %}
+	]
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/hits_response.qtpl.go

@@ -0,0 +1,219 @@
+// Code generated by qtc from "hits_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlselect/logsql/hits_response.qtpl:1
+package logsql
+
+//line app/vlselect/logsql/hits_response.qtpl:1
+import (
+	"slices"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// FieldsForHits formats labels for /select/logsql/hits response
+
+//line app/vlselect/logsql/hits_response.qtpl:10
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/hits_response.qtpl:10
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/hits_response.qtpl:10
+func StreamFieldsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/hits_response.qtpl:10
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/hits_response.qtpl:12
+	if len(columns) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:13
+		qw422016.N().Q(columns[0].Name)
+//line app/vlselect/logsql/hits_response.qtpl:13
+		qw422016.N().S(`:`)
+//line app/vlselect/logsql/hits_response.qtpl:13
+		qw422016.N().Q(columns[0].Values[rowIdx])
+//line app/vlselect/logsql/hits_response.qtpl:14
+		for _, c := range columns[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:14
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:15
+			qw422016.N().Q(c.Name)
+//line app/vlselect/logsql/hits_response.qtpl:15
+			qw422016.N().S(`:`)
+//line app/vlselect/logsql/hits_response.qtpl:15
+			qw422016.N().Q(c.Values[rowIdx])
+//line app/vlselect/logsql/hits_response.qtpl:16
+		}
+//line app/vlselect/logsql/hits_response.qtpl:17
+	}
+//line app/vlselect/logsql/hits_response.qtpl:17
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/hits_response.qtpl:19
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:19
+func WriteFieldsForHits(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	StreamFieldsForHits(qw422016, columns, rowIdx)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/hits_response.qtpl:19
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:19
+func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) string {
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/hits_response.qtpl:19
+	WriteFieldsForHits(qb422016, columns, rowIdx)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/hits_response.qtpl:19
+	return qs422016
+//line app/vlselect/logsql/hits_response.qtpl:19
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:21
+func StreamHitsSeries(qw422016 *qt422016.Writer, m map[string]*hitsSeries) {
+//line app/vlselect/logsql/hits_response.qtpl:21
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/hits_response.qtpl:24
+	sortedKeys := make([]string, 0, len(m))
+	for k := range m {
+		sortedKeys = append(sortedKeys, k)
+	}
+	slices.Sort(sortedKeys)
+
+//line app/vlselect/logsql/hits_response.qtpl:29
+	qw422016.N().S(`"hits":[`)
+//line app/vlselect/logsql/hits_response.qtpl:31
+	if len(sortedKeys) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:32
+		streamhitsSeriesLine(qw422016, m, sortedKeys[0])
+//line app/vlselect/logsql/hits_response.qtpl:33
+		for _, k := range sortedKeys[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:33
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:34
+			streamhitsSeriesLine(qw422016, m, k)
+//line app/vlselect/logsql/hits_response.qtpl:35
+		}
+//line app/vlselect/logsql/hits_response.qtpl:36
+	}
+//line app/vlselect/logsql/hits_response.qtpl:36
+	qw422016.N().S(`]}`)
+//line app/vlselect/logsql/hits_response.qtpl:39
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:39
+func WriteHitsSeries(qq422016 qtio422016.Writer, m map[string]*hitsSeries) {
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	StreamHitsSeries(qw422016, m)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/hits_response.qtpl:39
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:39
+func HitsSeries(m map[string]*hitsSeries) string {
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/hits_response.qtpl:39
+	WriteHitsSeries(qb422016, m)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/hits_response.qtpl:39
+	return qs422016
+//line app/vlselect/logsql/hits_response.qtpl:39
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:41
+func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k string) {
+//line app/vlselect/logsql/hits_response.qtpl:41
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/hits_response.qtpl:44
+	hs := m[k]
+	hs.sort()
+	timestamps := hs.timestamps
+	values := hs.values
+
+//line app/vlselect/logsql/hits_response.qtpl:48
+	qw422016.N().S(`"fields":`)
+//line app/vlselect/logsql/hits_response.qtpl:49
+	qw422016.N().S(k)
+//line app/vlselect/logsql/hits_response.qtpl:49
+	qw422016.N().S(`,"timestamps":[`)
+//line app/vlselect/logsql/hits_response.qtpl:51
+	if len(timestamps) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:52
+		qw422016.N().Q(timestamps[0])
+//line app/vlselect/logsql/hits_response.qtpl:53
+		for _, ts := range timestamps[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:53
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:54
+			qw422016.N().Q(ts)
+//line app/vlselect/logsql/hits_response.qtpl:55
+		}
+//line app/vlselect/logsql/hits_response.qtpl:56
+	}
+//line app/vlselect/logsql/hits_response.qtpl:56
+	qw422016.N().S(`],"values":[`)
+//line app/vlselect/logsql/hits_response.qtpl:59
+	if len(values) > 0 {
+//line app/vlselect/logsql/hits_response.qtpl:60
+		qw422016.N().S(values[0])
+//line app/vlselect/logsql/hits_response.qtpl:61
+		for _, v := range values[1:] {
+//line app/vlselect/logsql/hits_response.qtpl:61
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/hits_response.qtpl:62
+			qw422016.N().S(v)
+//line app/vlselect/logsql/hits_response.qtpl:63
+		}
+//line app/vlselect/logsql/hits_response.qtpl:64
+	}
+//line app/vlselect/logsql/hits_response.qtpl:64
+	qw422016.N().S(`]}`)
+//line app/vlselect/logsql/hits_response.qtpl:67
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:67
+func writehitsSeriesLine(qq422016 qtio422016.Writer, m map[string]*hitsSeries, k string) {
+//line app/vlselect/logsql/hits_response.qtpl:67
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/hits_response.qtpl:67
+	streamhitsSeriesLine(qw422016, m, k)
+//line app/vlselect/logsql/hits_response.qtpl:67
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/hits_response.qtpl:67
+}
+
+//line app/vlselect/logsql/hits_response.qtpl:67
+func hitsSeriesLine(m map[string]*hitsSeries, k string) string {
+//line app/vlselect/logsql/hits_response.qtpl:67
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/hits_response.qtpl:67
+	writehitsSeriesLine(qb422016, m, k)
+//line app/vlselect/logsql/hits_response.qtpl:67
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/hits_response.qtpl:67
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/hits_response.qtpl:67
+	return qs422016
+//line app/vlselect/logsql/hits_response.qtpl:67
+}

app/vlselect/logsql/logsql.go

@@ -0,0 +1,492 @@
+package logsql
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+)
+
+// ProcessHitsRequest handles /select/logsql/hits request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-hits-stats
+func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Obtain step
+	stepStr := r.FormValue("step")
+	if stepStr == "" {
+		stepStr = "1d"
+	}
+	step, err := promutils.ParseDuration(stepStr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse 'step' arg: %s", err)
+		return
+	}
+	if step <= 0 {
+		httpserver.Errorf(w, r, "'step' must be bigger than zero")
+	}
+
+	// Obtain offset
+	offsetStr := r.FormValue("offset")
+	if offsetStr == "" {
+		offsetStr = "0s"
+	}
+	offset, err := promutils.ParseDuration(offsetStr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse 'offset' arg: %s", err)
+		return
+	}
+
+	// Obtain field entries
+	fields := r.Form["field"]
+
+	// Prepare the query
+	q.AddCountByTimePipe(int64(step), int64(offset), fields)
+	q.Optimize()
+
+	var mLock sync.Mutex
+	m := make(map[string]*hitsSeries)
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
+			return
+		}
+
+		timestampValues := columns[0].Values
+		hitsValues := columns[len(columns)-1].Values
+		columns = columns[1 : len(columns)-1]
+
+		bb := blockResultPool.Get()
+		for i := range timestamps {
+			timestampStr := strings.Clone(timestampValues[i])
+			hitsStr := strings.Clone(hitsValues[i])
+
+			bb.Reset()
+			WriteFieldsForHits(bb, columns, i)
+
+			mLock.Lock()
+			hs, ok := m[string(bb.B)]
+			if !ok {
+				k := string(bb.B)
+				hs = &hitsSeries{}
+				m[k] = hs
+			}
+			hs.timestamps = append(hs.timestamps, timestampStr)
+			hs.values = append(hs.values, hitsStr)
+			mLock.Unlock()
+		}
+		blockResultPool.Put(bb)
+	}
+
+	// Execute the query
+	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
+		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
+		return
+	}
+
+	// Write response
+	w.Header().Set("Content-Type", "application/json")
+	WriteHitsSeries(w, m)
+}
+
+type hitsSeries struct {
+	timestamps []string
+	values     []string
+}
+
+func (hs *hitsSeries) sort() {
+	sort.Sort(hs)
+}
+
+func (hs *hitsSeries) Len() int {
+	return len(hs.timestamps)
+}
+
+func (hs *hitsSeries) Swap(i, j int) {
+	hs.timestamps[i], hs.timestamps[j] = hs.timestamps[j], hs.timestamps[i]
+	hs.values[i], hs.values[j] = hs.values[j], hs.values[i]
+}
+
+func (hs *hitsSeries) Less(i, j int) bool {
+	return hs.timestamps[i] < hs.timestamps[j]
+}
+
+// ProcessFieldNamesRequest handles /select/logsql/field_names request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-field-names
+func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Obtain field names for the given query
+	q.Optimize()
+	fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain field names: %s", err)
+		return
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, fieldNames)
+}
+
+// ProcessFieldValuesRequest handles /select/logsql/field_values request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-field-values
+func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse fieldName query arg
+	fieldName := r.FormValue("field")
+	if fieldName == "" {
+		httpserver.Errorf(w, r, "missing 'field' query arg")
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain unique values for the given field
+	q.Optimize()
+	values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain values for field %q: %s", fieldName, err)
+		return
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, values)
+}
+
+// ProcessStreamFieldNamesRequest processes /select/logsql/stream_field_names request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-names
+func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Obtain stream field names for the given query
+	q.Optimize()
+	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream field names: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, names)
+}
+
+// ProcessStreamFieldValuesRequest processes /select/logsql/stream_field_values request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-values
+func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse fieldName query arg
+	fieldName := r.FormValue("field")
+	if fieldName == "" {
+		httpserver.Errorf(w, r, "missing 'field' query arg")
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain stream field values for the given query and the given fieldName
+	q.Optimize()
+	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain stream field values: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, values)
+}
+
+// ProcessStreamsRequest processes /select/logsql/streams request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-streams
+func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	// Obtain streams for the given query
+	q.Optimize()
+	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot obtain streams: %s", err)
+	}
+
+	// Write results
+	w.Header().Set("Content-Type", "application/json")
+	WriteValuesWithHitsJSON(w, streams)
+}
+
+// ProcessQueryRequest handles /select/logsql/query request.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
+func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	q, tenantIDs, err := parseCommonArgs(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	// Parse limit query arg
+	limit, err := httputils.GetInt(r, "limit")
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	bw := getBufferedWriter(w)
+	defer func() {
+		bw.FlushIgnoreErrors()
+		putBufferedWriter(bw)
+	}()
+	w.Header().Set("Content-Type", "application/stream+json")
+
+	if limit > 0 {
+		if q.CanReturnLastNResults() {
+			rows, err := getLastNQueryResults(ctx, tenantIDs, q, limit)
+			if err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+				return
+			}
+			bb := blockResultPool.Get()
+			b := bb.B
+			for i := range rows {
+				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
+				b = append(b, '\n')
+				bw.WriteIgnoreErrors(b)
+			}
+			bb.B = b
+			blockResultPool.Put(bb)
+			return
+		}
+
+		q.AddPipeLimit(uint64(limit))
+	}
+	q.Optimize()
+
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
+			return
+		}
+
+		bb := blockResultPool.Get()
+		for i := range timestamps {
+			WriteJSONRow(bb, columns, i)
+		}
+		bw.WriteIgnoreErrors(bb.B)
+		blockResultPool.Put(bb)
+	}
+
+	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
+		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
+	}
+}
+
+var blockResultPool bytesutil.ByteBufferPool
+
+type row struct {
+	timestamp int64
+	fields    []logstorage.Field
+}
+
+func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+	limitUpper := 2 * limit
+	q.AddPipeLimit(uint64(limitUpper))
+	q.Optimize()
+	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
+	if err != nil {
+		return nil, err
+	}
+	if len(rows) < limitUpper {
+		// Fast path - the requested time range contains up to limitUpper rows.
+		rows = getLastNRows(rows, limit)
+		return rows, nil
+	}
+
+	// Slow path - search for the time range containing up to limitUpper rows.
+	start, end := q.GetFilterTimeRange()
+	d := end/2 - start/2
+	start += d
+
+	qOrig := q
+	for {
+		q = qOrig.Clone()
+		q.AddTimeFilter(start, end)
+		rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
+			rows = getLastNRows(rows, limit)
+			return rows, nil
+		}
+
+		lastBit := d & 1
+		d /= 2
+		if len(rows) > limit {
+			start += d
+		} else {
+			start -= d + lastBit
+		}
+	}
+}
+
+func getLastNRows(rows []row, limit int) []row {
+	sort.Slice(rows, func(i, j int) bool {
+		return rows[i].timestamp < rows[j].timestamp
+	})
+	if len(rows) > limit {
+		rows = rows[len(rows)-limit:]
+	}
+	return rows
+}
+
+func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var rows []row
+	var rowsLock sync.Mutex
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		rowsLock.Lock()
+		defer rowsLock.Unlock()
+
+		for i, timestamp := range timestamps {
+			fields := make([]logstorage.Field, len(columns))
+			for j := range columns {
+				f := &fields[j]
+				f.Name = strings.Clone(columns[j].Name)
+				f.Value = strings.Clone(columns[j].Values[i])
+			}
+			rows = append(rows, row{
+				timestamp: timestamp,
+				fields:    fields,
+			})
+		}
+
+		if len(rows) >= limit {
+			cancel()
+		}
+	}
+	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
+		return nil, err
+	}
+
+	return rows, nil
+}
+
+func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {
+	// Extract tenantID
+	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot obtain tenanID: %w", err)
+	}
+	tenantIDs := []logstorage.TenantID{tenantID}
+
+	// Parse query
+	qStr := r.FormValue("query")
+	q, err := logstorage.ParseQuery(qStr)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
+	}
+
+	// Parse optional start and end args
+	start, okStart, err := getTimeNsec(r, "start")
+	if err != nil {
+		return nil, nil, err
+	}
+	end, okEnd, err := getTimeNsec(r, "end")
+	if err != nil {
+		return nil, nil, err
+	}
+	if okStart || okEnd {
+		if !okStart {
+			start = math.MinInt64
+		}
+		if !okEnd {
+			end = math.MaxInt64
+		}
+		q.AddTimeFilter(start, end)
+	}
+
+	return q, tenantIDs, nil
+}
+
+func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
+	s := r.FormValue(argName)
+	if s == "" {
+		return 0, false, nil
+	}
+	currentTimestamp := time.Now().UnixNano()
+	nsecs, err := promutils.ParseTimeAt(s, currentTimestamp)
+	if err != nil {
+		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
+	}
+	return nsecs, true, nil
+}

app/vlselect/logsql/logsql.qtpl

@@ -0,0 +1,32 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+) %}
+
+{% stripspace %}
+
+// ValuesWithHitsJSON generates JSON from the given values.
+{% func ValuesWithHitsJSON(values []logstorage.ValueWithHits) %}
+{
+	"values":{%= valuesWithHitsJSONArray(values) %}
+}
+{% endfunc %}
+
+{% func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) %}
+[
+	{% if len(values) > 0 %}
+		{%= valueWithHitsJSON(values[0]) %}
+		{% for _, v := range values[1:] %}
+			,{%= valueWithHitsJSON(v) %}
+		{% endfor %}
+	{% endif %}
+]
+{% endfunc %}
+
+{% func valueWithHitsJSON(v logstorage.ValueWithHits) %}
+{
+	"value":{%q= v.Value %},
+	"hits":{%dul= v.Hits %}
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/logsql.qtpl.go

@@ -0,0 +1,152 @@
+// Code generated by qtc from "logsql.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlselect/logsql/logsql.qtpl:1
+package logsql
+
+//line app/vlselect/logsql/logsql.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ValuesWithHitsJSON generates JSON from the given values.
+
+//line app/vlselect/logsql/logsql.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/logsql.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/logsql.qtpl:8
+func StreamValuesWithHitsJSON(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:8
+	qw422016.N().S(`{"values":`)
+//line app/vlselect/logsql/logsql.qtpl:10
+	streamvaluesWithHitsJSONArray(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:10
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:12
+func WriteValuesWithHitsJSON(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:12
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+	StreamValuesWithHitsJSON(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:12
+func ValuesWithHitsJSON(values []logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:12
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:12
+	WriteValuesWithHitsJSON(qb422016, values)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:12
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:12
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:12
+}
+
+//line app/vlselect/logsql/logsql.qtpl:14
+func streamvaluesWithHitsJSONArray(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:14
+	qw422016.N().S(`[`)
+//line app/vlselect/logsql/logsql.qtpl:16
+	if len(values) > 0 {
+//line app/vlselect/logsql/logsql.qtpl:17
+		streamvalueWithHitsJSON(qw422016, values[0])
+//line app/vlselect/logsql/logsql.qtpl:18
+		for _, v := range values[1:] {
+//line app/vlselect/logsql/logsql.qtpl:18
+			qw422016.N().S(`,`)
+//line app/vlselect/logsql/logsql.qtpl:19
+			streamvalueWithHitsJSON(qw422016, v)
+//line app/vlselect/logsql/logsql.qtpl:20
+		}
+//line app/vlselect/logsql/logsql.qtpl:21
+	}
+//line app/vlselect/logsql/logsql.qtpl:21
+	qw422016.N().S(`]`)
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:23
+func writevaluesWithHitsJSONArray(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:23
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+	streamvaluesWithHitsJSONArray(qw422016, values)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:23
+func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:23
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:23
+	writevaluesWithHitsJSONArray(qb422016, values)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:23
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:23
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:23
+}
+
+//line app/vlselect/logsql/logsql.qtpl:25
+func streamvalueWithHitsJSON(qw422016 *qt422016.Writer, v logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:25
+	qw422016.N().S(`{"value":`)
+//line app/vlselect/logsql/logsql.qtpl:27
+	qw422016.N().Q(v.Value)
+//line app/vlselect/logsql/logsql.qtpl:27
+	qw422016.N().S(`,"hits":`)
+//line app/vlselect/logsql/logsql.qtpl:28
+	qw422016.N().DUL(v.Hits)
+//line app/vlselect/logsql/logsql.qtpl:28
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/logsql.qtpl:30
+}
+
+//line app/vlselect/logsql/logsql.qtpl:30
+func writevalueWithHitsJSON(qq422016 qtio422016.Writer, v logstorage.ValueWithHits) {
+//line app/vlselect/logsql/logsql.qtpl:30
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+	streamvalueWithHitsJSON(qw422016, v)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+}
+
+//line app/vlselect/logsql/logsql.qtpl:30
+func valueWithHitsJSON(v logstorage.ValueWithHits) string {
+//line app/vlselect/logsql/logsql.qtpl:30
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/logsql.qtpl:30
+	writevalueWithHitsJSON(qb422016, v)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/logsql.qtpl:30
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/logsql.qtpl:30
+	return qs422016
+//line app/vlselect/logsql/logsql.qtpl:30
+}

app/vlselect/logsql/query_response.qtpl

@@ -0,0 +1,41 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+) %}
+
+{% stripspace %}
+
+// JSONRow creates JSON row from the given fields.
+{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
+{
+	{% code c := &columns[0] %}
+	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
+	{% code columns = columns[1:] %}
+	{% for colIdx := range columns %}
+		{% code c := &columns[colIdx] %}
+		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
+	{% endfor %}
+}{% newline %}
+{% endfunc %}
+
+// JSONRows prints formatted rows
+{% func JSONRows(rows [][]logstorage.Field) %}
+	{% if len(rows) == 0 %}
+		{% return %}
+	{% endif %}
+	{% for _, fields := range rows %}
+	{
+		{% if len(fields) > 0 %}
+			{% code
+				f := fields[0]
+				fields = fields[1:]
+			%}
+			{%q= f.Name %}:{%q= f.Value %}
+			{% for _, f := range fields %}
+				,{%q= f.Name %}:{%q= f.Value %}
+			{% endfor %}
+		{% endif %}
+	}{% newline %}
+	{% endfor %}
+{% endfunc %}
+
+{% endstripspace %}

app/vlselect/logsql/query_response.qtpl.go

@@ -0,0 +1,166 @@
+// Code generated by qtc from "query_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vlselect/logsql/query_response.qtpl:1
+package logsql
+
+//line app/vlselect/logsql/query_response.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// JSONRow creates JSON row from the given fields.
+
+//line app/vlselect/logsql/query_response.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vlselect/logsql/query_response.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vlselect/logsql/query_response.qtpl:8
+func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/query_response.qtpl:8
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/query_response.qtpl:10
+	c := &columns[0]
+
+//line app/vlselect/logsql/query_response.qtpl:11
+	qw422016.N().Q(c.Name)
+//line app/vlselect/logsql/query_response.qtpl:11
+	qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:11
+	qw422016.N().Q(c.Values[rowIdx])
+//line app/vlselect/logsql/query_response.qtpl:12
+	columns = columns[1:]
+
+//line app/vlselect/logsql/query_response.qtpl:13
+	for colIdx := range columns {
+//line app/vlselect/logsql/query_response.qtpl:14
+		c := &columns[colIdx]
+
+//line app/vlselect/logsql/query_response.qtpl:14
+		qw422016.N().S(`,`)
+//line app/vlselect/logsql/query_response.qtpl:15
+		qw422016.N().Q(c.Name)
+//line app/vlselect/logsql/query_response.qtpl:15
+		qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:15
+		qw422016.N().Q(c.Values[rowIdx])
+//line app/vlselect/logsql/query_response.qtpl:16
+	}
+//line app/vlselect/logsql/query_response.qtpl:16
+	qw422016.N().S(`}`)
+//line app/vlselect/logsql/query_response.qtpl:17
+	qw422016.N().S(`
+`)
+//line app/vlselect/logsql/query_response.qtpl:18
+}
+
+//line app/vlselect/logsql/query_response.qtpl:18
+func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
+//line app/vlselect/logsql/query_response.qtpl:18
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/query_response.qtpl:18
+	StreamJSONRow(qw422016, columns, rowIdx)
+//line app/vlselect/logsql/query_response.qtpl:18
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/query_response.qtpl:18
+}
+
+//line app/vlselect/logsql/query_response.qtpl:18
+func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
+//line app/vlselect/logsql/query_response.qtpl:18
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/query_response.qtpl:18
+	WriteJSONRow(qb422016, columns, rowIdx)
+//line app/vlselect/logsql/query_response.qtpl:18
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/query_response.qtpl:18
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/query_response.qtpl:18
+	return qs422016
+//line app/vlselect/logsql/query_response.qtpl:18
+}
+
+// JSONRows prints formatted rows
+
+//line app/vlselect/logsql/query_response.qtpl:21
+func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
+//line app/vlselect/logsql/query_response.qtpl:22
+	if len(rows) == 0 {
+//line app/vlselect/logsql/query_response.qtpl:23
+		return
+//line app/vlselect/logsql/query_response.qtpl:24
+	}
+//line app/vlselect/logsql/query_response.qtpl:25
+	for _, fields := range rows {
+//line app/vlselect/logsql/query_response.qtpl:25
+		qw422016.N().S(`{`)
+//line app/vlselect/logsql/query_response.qtpl:27
+		if len(fields) > 0 {
+//line app/vlselect/logsql/query_response.qtpl:29
+			f := fields[0]
+			fields = fields[1:]
+
+//line app/vlselect/logsql/query_response.qtpl:32
+			qw422016.N().Q(f.Name)
+//line app/vlselect/logsql/query_response.qtpl:32
+			qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:32
+			qw422016.N().Q(f.Value)
+//line app/vlselect/logsql/query_response.qtpl:33
+			for _, f := range fields {
+//line app/vlselect/logsql/query_response.qtpl:33
+				qw422016.N().S(`,`)
+//line app/vlselect/logsql/query_response.qtpl:34
+				qw422016.N().Q(f.Name)
+//line app/vlselect/logsql/query_response.qtpl:34
+				qw422016.N().S(`:`)
+//line app/vlselect/logsql/query_response.qtpl:34
+				qw422016.N().Q(f.Value)
+//line app/vlselect/logsql/query_response.qtpl:35
+			}
+//line app/vlselect/logsql/query_response.qtpl:36
+		}
+//line app/vlselect/logsql/query_response.qtpl:36
+		qw422016.N().S(`}`)
+//line app/vlselect/logsql/query_response.qtpl:37
+		qw422016.N().S(`
+`)
+//line app/vlselect/logsql/query_response.qtpl:38
+	}
+//line app/vlselect/logsql/query_response.qtpl:39
+}
+
+//line app/vlselect/logsql/query_response.qtpl:39
+func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
+//line app/vlselect/logsql/query_response.qtpl:39
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vlselect/logsql/query_response.qtpl:39
+	StreamJSONRows(qw422016, rows)
+//line app/vlselect/logsql/query_response.qtpl:39
+	qt422016.ReleaseWriter(qw422016)
+//line app/vlselect/logsql/query_response.qtpl:39
+}
+
+//line app/vlselect/logsql/query_response.qtpl:39
+func JSONRows(rows [][]logstorage.Field) string {
+//line app/vlselect/logsql/query_response.qtpl:39
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vlselect/logsql/query_response.qtpl:39
+	WriteJSONRows(qb422016, rows)
+//line app/vlselect/logsql/query_response.qtpl:39
+	qs422016 := string(qb422016.B)
+//line app/vlselect/logsql/query_response.qtpl:39
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vlselect/logsql/query_response.qtpl:39
+	return qs422016
+//line app/vlselect/logsql/query_response.qtpl:39
+}

app/vlselect/main.go

@@ -0,0 +1,198 @@
+package vlselect
+
+import (
+	"embed"
+	"flag"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
+		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
+		"See also -search.maxQueueDuration")
+	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
+		"limit is reached; see also -search.maxQueryDuration")
+	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
+)
+
+func getDefaultMaxConcurrentRequests() int {
+	n := cgroup.AvailableCPUs()
+	if n <= 4 {
+		n *= 2
+	}
+	if n > 16 {
+		// A single request can saturate all the CPU cores, so there is no sense
+		// in allowing higher number of concurrent requests - they will just contend
+		// for unavailable CPU time.
+		n = 16
+	}
+	return n
+}
+
+// Init initializes vlselect
+func Init() {
+	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
+}
+
+// Stop stops vlselect
+func Stop() {
+}
+
+var concurrencyLimitCh chan struct{}
+
+var (
+	concurrencyLimitReached = metrics.NewCounter(`vl_concurrent_select_limit_reached_total`)
+	concurrencyLimitTimeout = metrics.NewCounter(`vl_concurrent_select_limit_timeout_total`)
+
+	_ = metrics.NewGauge(`vl_concurrent_select_capacity`, func() float64 {
+		return float64(cap(concurrencyLimitCh))
+	})
+	_ = metrics.NewGauge(`vl_concurrent_select_current`, func() float64 {
+		return float64(len(concurrencyLimitCh))
+	})
+)
+
+//go:embed vmui
+var vmuiFiles embed.FS
+
+var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
+
+// RequestHandler handles select requests for VictoriaLogs
+func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := r.URL.Path
+	if !strings.HasPrefix(path, "/select/") {
+		// Skip requests, which do not start with /select/, since these aren't our requests.
+		return false
+	}
+	path = strings.ReplaceAll(path, "//", "/")
+
+	if path == "/select/vmui" {
+		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
+		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
+		// is hidden behind vmauth or similar proxy.
+		_ = r.ParseForm()
+		newURL := "vmui/?" + r.Form.Encode()
+		httpserver.Redirect(w, newURL)
+		return true
+	}
+	if strings.HasPrefix(path, "/select/vmui/") {
+		if strings.HasPrefix(path, "/select/vmui/static/") {
+			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
+			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
+			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
+			w.Header().Set("Cache-Control", "max-age=31536000")
+		}
+		r.URL.Path = strings.TrimPrefix(path, "/select")
+		vmuiFileServer.ServeHTTP(w, r)
+		return true
+	}
+
+	// Limit the number of concurrent queries, which can consume big amounts of CPU.
+	startTime := time.Now()
+	ctx := r.Context()
+	stopCh := ctx.Done()
+	select {
+	case concurrencyLimitCh <- struct{}{}:
+		defer func() { <-concurrencyLimitCh }()
+	default:
+		// Sleep for a while until giving up. This should resolve short bursts in requests.
+		concurrencyLimitReached.Inc()
+		d := getMaxQueryDuration(r)
+		if d > *maxQueueDuration {
+			d = *maxQueueDuration
+		}
+		t := timerpool.Get(d)
+		select {
+		case concurrencyLimitCh <- struct{}{}:
+			timerpool.Put(t)
+			defer func() { <-concurrencyLimitCh }()
+		case <-stopCh:
+			timerpool.Put(t)
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			requestURI := httpserver.GetRequestURI(r)
+			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+				time.Since(startTime).Seconds(), remoteAddr, requestURI)
+			return true
+		case <-t.C:
+			timerpool.Put(t)
+			concurrencyLimitTimeout.Inc()
+			err := &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
+					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
+					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
+					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
+				StatusCode: http.StatusServiceUnavailable,
+			}
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+	}
+
+	httpserver.EnableCORS(w, r)
+	switch path {
+	case "/select/logsql/field_names":
+		logsqlFieldNamesRequests.Inc()
+		logsql.ProcessFieldNamesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/field_values":
+		logsqlFieldValuesRequests.Inc()
+		logsql.ProcessFieldValuesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/hits":
+		logsqlHitsRequests.Inc()
+		logsql.ProcessHitsRequest(ctx, w, r)
+		return true
+	case "/select/logsql/query":
+		logsqlQueryRequests.Inc()
+		logsql.ProcessQueryRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stream_field_names":
+		logsqlStreamFieldNamesRequests.Inc()
+		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/stream_field_values":
+		logsqlStreamFieldValuesRequests.Inc()
+		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
+		return true
+	case "/select/logsql/streams":
+		logsqlStreamsRequests.Inc()
+		logsql.ProcessStreamsRequest(ctx, w, r)
+		return true
+	default:
+		return false
+	}
+}
+
+// getMaxQueryDuration returns the maximum duration for query from r.
+func getMaxQueryDuration(r *http.Request) time.Duration {
+	dms, err := httputils.GetDuration(r, "timeout", 0)
+	if err != nil {
+		dms = 0
+	}
+	d := time.Duration(dms) * time.Millisecond
+	if d <= 0 || d > *maxQueryDuration {
+		d = *maxQueryDuration
+	}
+	return d
+}
+
+var (
+	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
+	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
+	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
+	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
+	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
+	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
+)

app/vlselect/vmui/asset-manifest.json

@@ -0,0 +1,13 @@
+{
+  "files": {
+    "main.css": "./static/css/main.2fa7c03f.css",
+    "main.js": "./static/js/main.68f1bd69.js",
+    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.cb83d071da309a358bc0.md",
+    "index.html": "./index.html"
+  },
+  "entrypoints": [
+    "static/css/main.2fa7c03f.css",
+    "static/js/main.68f1bd69.js"
+  ]
+}

app/vlselect/vmui/index.html

@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.68f1bd69.js"></script><link href="./static/css/main.2fa7c03f.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/manifest.json

@@ -0,0 +1,20 @@
+{
+  "short_name": "Victoria Metrics UI",
+  "name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
+  "icons": [
+    {
+      "src": "favicon-32x32.png",
+      "sizes": "32x32",
+      "type": "image/png"
+    },
+    {
+      "src": "apple-touch-icon.png",
+      "type": "image/png",
+      "sizes": "192x192"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

app/vlselect/vmui/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:

app/vlselect/vmui/static/js/main.68f1bd69.js.LICENSE.txt

@@ -0,0 +1,38 @@
+/*!
+	Copyright (c) 2018 Jed Watson.
+	Licensed under the MIT License (MIT), see
+	http://jedwatson.github.io/classnames
+*/
+
+/**
+ * @remix-run/router v1.15.1
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */
+
+/**
+ * React Router DOM v6.22.1
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */
+
+/**
+ * React Router v6.22.1
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */

app/vlstorage/README.md

@@ -1 +0,0 @@
-VictoriaLogs source code has been moved to [github.com/VictoriaMetrics/VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/).

app/vlstorage/main.go

@@ -0,0 +1,197 @@
+package vlstorage
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+var (
+	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
+		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
+		"see https://docs.victoriametrics.com/victorialogs/#retention")
+	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
+		"see https://docs.victoriametrics.com/victorialogs/#retention")
+	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory with the VictoriaLogs data; "+
+		"see https://docs.victoriametrics.com/victorialogs/#storage")
+	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
+		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
+		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
+		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
+	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
+		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows")
+	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
+		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
+	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
+		"the storage stops accepting new data")
+)
+
+// Init initializes vlstorage.
+//
+// Stop must be called when vlstorage is no longer needed
+func Init() {
+	if strg != nil {
+		logger.Panicf("BUG: Init() has been already called")
+	}
+
+	if retentionPeriod.Duration() < 24*time.Hour {
+		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
+	}
+	cfg := &logstorage.StorageConfig{
+		Retention:             retentionPeriod.Duration(),
+		FlushInterval:         *inmemoryDataFlushInterval,
+		FutureRetention:       futureRetention.Duration(),
+		LogNewStreams:         *logNewStreams,
+		LogIngestedRows:       *logIngestedRows,
+		MinFreeDiskSpaceBytes: minFreeDiskSpaceBytes.N,
+	}
+	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
+	startTime := time.Now()
+	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
+
+	var ss logstorage.StorageStats
+	strg.UpdateStats(&ss)
+	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
+		"smallPartSize: %d bytes; bigPartSize: %d bytes",
+		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
+		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
+
+	// register storage metrics
+	storageMetrics = metrics.NewSet()
+	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
+		writeStorageMetrics(w, strg)
+	})
+	metrics.RegisterSet(storageMetrics)
+}
+
+// Stop stops vlstorage.
+func Stop() {
+	metrics.UnregisterSet(storageMetrics)
+	storageMetrics = nil
+
+	strg.MustClose()
+	strg = nil
+}
+
+var strg *logstorage.Storage
+var storageMetrics *metrics.Set
+
+// CanWriteData returns non-nil error if it cannot write data to vlstorage.
+func CanWriteData() error {
+	if strg.IsReadOnly() {
+		return &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
+				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
+			StatusCode: http.StatusTooManyRequests,
+		}
+	}
+	return nil
+}
+
+// MustAddRows adds lr to vlstorage
+//
+// It is advised to call CanWriteData() before calling MustAddRows()
+func MustAddRows(lr *logstorage.LogRows) {
+	strg.MustAddRows(lr)
+}
+
+// RunQuery runs the given q and calls writeBlock for the returned data blocks
+func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
+	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
+}
+
+// GetFieldNames executes q and returns field names seen in results.
+func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return strg.GetFieldNames(ctx, tenantIDs, q)
+}
+
+// GetFieldValues executes q and returns unique values for the fieldName seen in results.
+//
+// If limit > 0, then up to limit unique values are returned.
+func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+}
+
+// GetStreamFieldNames executes q and returns stream field names seen in results.
+func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
+}
+
+// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
+//
+// If limit > 0, then up to limit unique stream field values are returned.
+func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+}
+
+// GetStreams executes q and returns streams seen in query results.
+//
+// If limit > 0, then up to limit unique streams are returned.
+func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return strg.GetStreams(ctx, tenantIDs, q, limit)
+}
+
+func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
+	var ss logstorage.StorageStats
+	strg.UpdateStats(&ss)
+
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
+
+	isReadOnly := uint64(0)
+	if ss.IsReadOnly {
+		isReadOnly = 1
+	}
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), isReadOnly)
+
+	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/inmemory"}`, ss.InmemoryActiveMerges)
+	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/small"}`, ss.SmallPartActiveMerges)
+	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/big"}`, ss.BigPartActiveMerges)
+
+	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/inmemory"}`, ss.InmemoryMergesTotal)
+	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/small"}`, ss.SmallPartMergesTotal)
+	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/big"}`, ss.BigPartMergesTotal)
+
+	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/inmemory"}`, ss.InmemoryRowsCount)
+	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/small"}`, ss.SmallPartRowsCount)
+	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/big"}`, ss.BigPartRowsCount)
+
+	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/inmemory"}`, ss.InmemoryParts)
+	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/small"}`, ss.SmallParts)
+	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/big"}`, ss.BigParts)
+
+	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/inmemory"}`, ss.InmemoryBlocks)
+	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/small"}`, ss.SmallPartBlocks)
+	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/big"}`, ss.BigPartBlocks)
+
+	metrics.WriteGaugeUint64(w, `vl_partitions`, ss.PartitionsCount)
+	metrics.WriteCounterUint64(w, `vl_streams_created_total`, ss.StreamsCreatedTotal)
+
+	metrics.WriteGaugeUint64(w, `vl_indexdb_rows`, ss.IndexdbItemsCount)
+	metrics.WriteGaugeUint64(w, `vl_indexdb_parts`, ss.IndexdbPartsCount)
+	metrics.WriteGaugeUint64(w, `vl_indexdb_blocks`, ss.IndexdbBlocksCount)
+
+	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="indexdb"}`, ss.IndexdbSizeBytes)
+	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="storage"}`, ss.CompressedInmemorySize+ss.CompressedSmallPartSize+ss.CompressedBigPartSize)
+
+	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/inmemory"}`, ss.CompressedInmemorySize)
+	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/small"}`, ss.CompressedSmallPartSize)
+	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/big"}`, ss.CompressedBigPartSize)
+
+	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/inmemory"}`, ss.UncompressedInmemorySize)
+	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/small"}`, ss.UncompressedSmallPartSize)
+	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/big"}`, ss.UncompressedBigPartSize)
+
+	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
+	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
+}

app/vmagent/README.md

@@ -1,3 +1,3 @@
-See vmagent docs [here](https://docs.victoriametrics.com/victoriametrics/vmagent/).
+See vmagent docs [here](https://docs.victoriametrics.com/vmagent/).
 
-vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmagent.md).
+vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmagent.md).

app/vmagent/common/push_ctx.go

@@ -3,7 +3,7 @@ package common
 import (
 	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
@@ -12,13 +12,13 @@ type PushCtx struct {
 	// WriteRequest contains the WriteRequest, which must be pushed later to remote storage.
 	//
 	// The actual labels and samples for the time series are stored in Labels and Samples fields.
-	WriteRequest prompb.WriteRequest
+	WriteRequest prompbmarshal.WriteRequest
 
 	// Labels contains flat list of all the labels used in WriteRequest.
-	Labels []prompb.Label
+	Labels []prompbmarshal.Label
 
 	// Samples contains flat list of all the samples used in WriteRequest.
-	Samples []prompb.Sample
+	Samples []prompbmarshal.Sample
 }
 
 // Reset resets ctx.

app/vmagent/csvimport/request_handler.go

@@ -6,10 +6,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -22,16 +22,16 @@ var (
 
 // InsertHandler processes csv data from req.
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []csvimport.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -41,23 +41,23 @@ func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompb.Label
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/datadogsketches/request_handler.go

@@ -6,11 +6,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -23,7 +23,7 @@ var (
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
@@ -33,7 +33,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -45,22 +45,22 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 		ms := sketch.ToSummary()
 		for _, m := range ms {
 			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: m.Name,
 			})
 			for _, label := range m.Labels {
-				labels = append(labels, prompb.Label{
+				labels = append(labels, prompbmarshal.Label{
 					Name:  label.Name,
 					Value: label.Value,
 				})
 			}
 			for _, tag := range sketch.Tags {
-				name, value := datadogutil.SplitTag(tag)
+				name, value := datadogutils.SplitTag(tag)
 				if name == "host" {
 					name = "exported_host"
 				}
-				labels = append(labels, prompb.Label{
+				labels = append(labels, prompbmarshal.Label{
 					Name:  name,
 					Value: value,
 				})
@@ -68,13 +68,13 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 			labels = append(labels, extraLabels...)
 			samplesLen := len(samples)
 			for _, p := range m.Points {
-				samples = append(samples, prompb.Sample{
+				samples = append(samples, prompbmarshal.Sample{
 					Timestamp: p.Timestamp,
 					Value:     p.Value,
 				})
 			}
 			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompb.TimeSeries{
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
 				Labels:  labels[labelsLen:],
 				Samples: samples[samplesLen:],
 			})

app/vmagent/datadogv1/request_handler.go

@@ -6,11 +6,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -23,7 +23,7 @@ var (
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
@@ -33,7 +33,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -45,28 +45,28 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.
 		ss := &series[i]
 		rowsTotal += len(ss.Points)
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: ss.Metric,
 		})
 		if ss.Host != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "host",
 				Value: ss.Host,
 			})
 		}
 		if ss.Device != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "device",
 				Value: ss.Device,
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := datadogutil.SplitTag(tag)
+			name, value := datadogutils.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  name,
 				Value: value,
 			})
@@ -74,12 +74,12 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompb.
 		labels = append(labels, extraLabels...)
 		samplesLen := len(samples)
 		for _, pt := range ss.Points {
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Timestamp: pt.Timestamp(),
 				Value:     pt.Value(),
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})

app/vmagent/datadogv2/request_handler.go

@@ -6,11 +6,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -25,7 +25,7 @@ var (
 //
 // See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
@@ -36,7 +36,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	})
 }
 
-func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -48,28 +48,28 @@ func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.
 		ss := &series[i]
 		rowsTotal += len(ss.Points)
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: ss.Metric,
 		})
 		for _, rs := range ss.Resources {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  rs.Type,
 				Value: rs.Name,
 			})
 		}
 		if ss.SourceTypeName != "" {
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "source_type_name",
 				Value: ss.SourceTypeName,
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := datadogutil.SplitTag(tag)
+			name, value := datadogutils.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  name,
 				Value: value,
 			})
@@ -77,12 +77,12 @@ func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompb.
 		labels = append(labels, extraLabels...)
 		samplesLen := len(samples)
 		for _, pt := range ss.Points {
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Timestamp: pt.Timestamp * 1000,
 				Value:     pt.Value,
 			})
 		}
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})

app/vmagent/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 8429
 
 ENTRYPOINT ["/vmagent-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./vmagent-prod

app/vmagent/graphite/request_handler.go

@@ -6,7 +6,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
 	"github.com/VictoriaMetrics/metrics"
@@ -21,7 +21,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, "", func(rows []parser.Row) error {
+	return stream.Parse(r, false, func(rows []parser.Row) error {
 		return insertRows(nil, rows)
 	})
 }
@@ -36,22 +36,22 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/influx/request_handler.go

@@ -10,11 +10,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -35,8 +35,8 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
-	return stream.Parse(r, encoding, true, "", "", func(db string, rows []influx.Row) error {
+func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
+	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, nil)
 	})
 }
@@ -45,22 +45,21 @@ func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error
 //
 // See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	encoding := req.Header.Get("Content-Encoding")
-	isStreamMode := req.Header.Get("Stream-Mode") == "1"
-	return stream.Parse(req.Body, encoding, isStreamMode, precision, db, func(db string, rows []influx.Row) error {
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 
@@ -80,13 +79,13 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 			if tag.Key == *dbLabel {
 				hasDBKey = true
 			}
-			commonLabels = append(commonLabels, prompb.Label{
+			commonLabels = append(commonLabels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		if len(db) > 0 && !hasDBKey {
-			commonLabels = append(commonLabels, prompb.Label{
+			commonLabels = append(commonLabels, prompbmarshal.Label{
 				Name:  *dbLabel,
 				Value: db,
 			})
@@ -110,16 +109,16 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 			}
 			metricGroup := bytesutil.ToUnsafeString(buf[bufLen:])
 			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: metricGroup,
 			})
 			labels = append(labels, commonLabels...)
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Timestamp: r.Timestamp,
 				Value:     f.Value,
 			})
-			tssDst = append(tssDst, prompb.TimeSeries{
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
 				Labels:  labels[labelsLen:],
 				Samples: samples[len(samples)-1:],
 			})
@@ -144,7 +143,7 @@ func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prom
 
 type pushCtx struct {
 	ctx            common.PushCtx
-	commonLabels   []prompb.Label
+	commonLabels   []prompbmarshal.Label
 	metricGroupBuf []byte
 	buf            []byte
 }

app/vmagent/main.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
@@ -26,27 +24,27 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
 	opentsdbhttpserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdbhttp"
+	statsdserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/statsd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -65,6 +63,12 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	statsdListenAddr = flag.String("statsdListenAddr", "", "TCP and UDP address to listen for Statsd plaintext data. Usually :8125 must be set. Doesn't work if empty. "+
+		"See also -statsdListenAddr.useProxyProtocol")
+	statsdUseProxyProtocol = flag.Bool("statsdListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -statsdListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	statsdDisableAggregationEnforcemenet = flag.Bool(`statsd.disableAggregationEnforcement`, false, "Whether to disable streaming aggregation requirement check. "+
+		"It's recommended to run statsdServer with pre-configured streaming aggregation to decrease load at database.")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
@@ -74,19 +78,17 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 0, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
 )
 
 var (
 	influxServer       *influxserver.Server
 	graphiteServer     *graphiteserver.Server
+	statsdServer       *statsdserver.Server
 	opentsdbServer     *opentsdbserver.Server
 	opentsdbhttpServer *opentsdbhttpserver.Server
 )
@@ -98,15 +100,6 @@ var (
 )
 
 func main() {
-	// vmagent is optimized for reduced memory allocations,
-	// so it can run with the reduced GOGC in order to reduce the used memory,
-	// while keeping CPU usage spent in GC at low levels.
-	//
-	// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
-	// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
-	// exceeds 0.05 for extended periods of time.
-	cgroup.SetGOGC(50)
-
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
@@ -114,7 +107,6 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -131,7 +123,7 @@ func main() {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
+			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
@@ -145,15 +137,21 @@ func main() {
 	startTime := time.Now()
 	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
-	protoparserutil.StartUnmarshalWorkers()
+	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, "")
+			return influx.InsertHandlerForReader(nil, r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
 	}
+	if len(*statsdListenAddr) > 0 {
+		if !remotewrite.HasAnyStreamAggrConfigured() && !*statsdDisableAggregationEnforcemenet {
+			logger.Fatalf("streaming aggregation must be configured with enabled statsd server. It's recommended  to aggregate metrics received at statsd listener. This check could be disabled with flag -statsd.disableAggregationEnforcement")
+		}
+		statsdServer = statsdserver.MustStart(*statsdListenAddr, *statsdUseProxyProtocol, statsd.InsertHandler)
+	}
 	if len(*opentsdbListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
 		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
@@ -165,9 +163,7 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -191,13 +187,16 @@ func main() {
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer.MustStop()
 	}
+	if len(*statsdListenAddr) > 0 {
+		statsdServer.MustStop()
+	}
 	if len(*opentsdbListenAddr) > 0 {
 		opentsdbServer.MustStop()
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
-	protoparserutil.StopUnmarshalWorkers()
+	common.StopUnmarshalWorkers()
 	remotewrite.Stop()
 
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
@@ -206,7 +205,7 @@ func main() {
 func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 	if !remotewrite.MultitenancyEnabled() {
 		return func(req *http.Request) error {
-			path := strings.ReplaceAll(req.URL.Path, "//", "/")
+			path := strings.Replace(req.URL.Path, "//", "/", -1)
 			if path != "/api/put" {
 				return fmt.Errorf("unsupported path requested: %q; expecting '/api/put'", path)
 			}
@@ -214,7 +213,7 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 		}
 	}
 	return func(req *http.Request) error {
-		path := strings.ReplaceAll(req.URL.Path, "//", "/")
+		path := strings.Replace(req.URL.Path, "//", "/", -1)
 		at, err := getAuthTokenFromPath(path)
 		if err != nil {
 			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
@@ -244,7 +243,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent/'>https://docs.victoriametrics.com/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
@@ -259,7 +258,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	path := strings.ReplaceAll(r.URL.Path, "//", "/")
+	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus") || strings.HasPrefix(path, "/api/v1/import/prometheus") {
 		prometheusimportRequests.Inc()
 		if err := prometheusimport.InsertHandler(nil, r); err != nil {
@@ -284,7 +283,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	switch path {
 	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
-		if protoparserutil.HandleVMProtoServerHandshake(w, r) {
+		if common.HandleVMProtoServerHandshake(w, r) {
 			return true
 		}
 		prometheusWriteRequests.Inc()
@@ -333,11 +332,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/influx/query", "/query":
 		influxQueryRequests.Inc()
-		influxutil.WriteDatabaseNames(w)
-		return true
-	case "/influx/health":
-		influxHealthRequests.Inc()
-		influxutil.WriteHealthCheckResponse(w)
+		influxutils.WriteDatabaseNames(w)
 		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
@@ -445,10 +440,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		// https://prometheus.io/docs/prometheus/latest/querying/api/#targets
 		state := r.FormValue("state")
-		scrapePool := r.FormValue("scrapePool")
-		promscrape.WriteAPIV1Targets(w, state, scrapePool)
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/prometheus/target_response", "/target_response":
 		promscrapeTargetResponseRequests.Inc()
@@ -459,7 +452,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -468,17 +461,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		var bb bytesutil.ByteBuffer
 		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		promscrapeConfigReloadRequests.Inc()
@@ -517,7 +510,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
 		return true
 	}
-	at, err := auth.NewTokenPossibleMultitenant(p.AuthToken)
+	at, err := auth.NewToken(p.AuthToken)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
@@ -529,13 +522,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		statusCode := http.StatusNoContent
-		if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus/metrics/job/") {
-			// Return 200 status code for pushgateway requests.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
-			statusCode = http.StatusOK
-		}
-		w.WriteHeader(statusCode)
+		w.WriteHeader(http.StatusNoContent)
 		return true
 	}
 	if strings.HasPrefix(p.Suffix, "datadog/") {
@@ -591,11 +578,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		return true
 	case "influx/query":
 		influxQueryRequests.Inc()
-		influxutil.WriteDatabaseNames(w)
-		return true
-	case "influx/health":
-		influxHealthRequests.Inc()
-		influxutil.WriteHealthCheckResponse(w)
+		influxutils.WriteDatabaseNames(w)
 		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
@@ -707,8 +690,7 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
-	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
@@ -754,7 +736,7 @@ func usage() {
 	const s = `
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
 
-See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
+See the docs at https://docs.victoriametrics.com/vmagent/ .
 `
 	flagutil.Usage(s)
 }

app/vmagent/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
+ARG certs_image
+ARG root_image
+FROM $certs_image as certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
@@ -9,5 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
 ARG TARGETARCH
-ARG BINARY_SUFFIX=non-existing
-COPY vmagent-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmagent-prod
+COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod

app/vmagent/native/request_handler.go

@@ -8,9 +8,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -25,17 +25,17 @@ var (
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(block *stream.Block) error {
+	isGzip := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
 		return insertRows(at, block, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -53,13 +53,13 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompb.Label)
 	samples := ctx.Samples[:0]
 	mn := &block.MetricName
 	labelsLen := len(labels)
-	labels = append(labels, prompb.Label{
+	labels = append(labels, prompbmarshal.Label{
 		Name:  "__name__",
 		Value: bytesutil.ToUnsafeString(mn.MetricGroup),
 	})
 	for j := range mn.Tags {
 		tag := &mn.Tags[j]
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  bytesutil.ToUnsafeString(tag.Key),
 			Value: bytesutil.ToUnsafeString(tag.Value),
 		})
@@ -72,12 +72,12 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompb.Label)
 	}
 	samplesLen := len(samples)
 	for j, value := range values {
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     value,
 			Timestamp: timestamps[j],
 		})
 	}
-	tssDst = append(tssDst, prompb.TimeSeries{
+	tssDst = append(tssDst, prompbmarshal.TimeSeries{
 		Labels:  labels[labelsLen:],
 		Samples: samples[samplesLen:],
 	})

app/vmagent/newrelic/request_handler.go

@@ -9,10 +9,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 )
 
@@ -24,17 +24,18 @@ var (
 
 // InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(rows []newrelic.Row) error {
+	ce := req.Header.Get("Content-Encoding")
+	isGzip := ce == "gzip"
+	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -49,22 +50,22 @@ func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label)
 		for j := range srcSamples {
 			s := &srcSamples[j]
 			labelsLen := len(labels)
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  "__name__",
 				Value: bytesutil.ToUnsafeString(s.Name),
 			})
 			for k := range tags {
 				t := &tags[k]
-				labels = append(labels, prompb.Label{
+				labels = append(labels, prompbmarshal.Label{
 					Name:  bytesutil.ToUnsafeString(t.Key),
 					Value: bytesutil.ToUnsafeString(t.Value),
 				})
 			}
-			samples = append(samples, prompb.Sample{
+			samples = append(samples, prompbmarshal.Sample{
 				Value:     s.Value,
 				Timestamp: r.Timestamp,
 			})
-			tssDst = append(tssDst, prompb.TimeSeries{
+			tssDst = append(tssDst, prompbmarshal.TimeSeries{
 				Labels:  labels[labelsLen:],
 				Samples: samples[len(samples)-1:],
 			})

app/vmagent/opentelemetry/request_handler.go

@@ -7,30 +7,27 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="opentelemetry"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="opentelemetry"}`)
-	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
 // InsertHandler processes opentelemetry metrics.
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
 		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
@@ -39,12 +36,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
-		return insertRows(at, tss, mms, extraLabels)
+	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
+		return insertRows(at, tss, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -60,39 +57,20 @@ func insertRows(at *auth.Token, tss []prompb.TimeSeries, mms []prompb.MetricMeta
 		labels = append(labels, extraLabels...)
 		samplesLen := len(samples)
 		samples = append(samples, ts.Samples...)
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[samplesLen:],
 		})
 	}
 	ctx.WriteRequest.Timeseries = tssDst
-
-	var metadataTotal int
-	if promscrape.IsMetadataEnabled() {
-		var accountID, projectID uint32
-		if at != nil {
-			accountID = at.AccountID
-			projectID = at.ProjectID
-			for i := range mms {
-				mm := &mms[i]
-				mm.AccountID = accountID
-				mm.ProjectID = projectID
-			}
-		}
-		ctx.WriteRequest.Metadata = mms
-		metadataTotal = len(mms)
-	}
-
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
 	rowsInserted.Add(rowsTotal)
-	metadataInserted.Add(metadataTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
-		metadataTenantInserted.Get(at).Add(metadataTotal)
 	}
 	rowsPerInsert.Update(float64(rowsTotal))
 	return nil

app/vmagent/opentsdb/request_handler.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
 	"github.com/VictoriaMetrics/metrics"
@@ -33,22 +33,22 @@ func insertRows(rows []parser.Row) error {
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/opentsdbhttp/request_handler.go

@@ -6,10 +6,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -21,16 +21,16 @@ var (
 // InsertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []opentsdbhttp.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -40,23 +40,23 @@ func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompb.La
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})

app/vmagent/prometheusimport/request_handler.go

@@ -7,103 +7,78 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted           = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
-	metadataInserted       = metrics.NewCounter(`vmagent_metadata_inserted_total{type="prometheus"}`)
-	rowsTenantInserted     = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
-	metadataTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_metadata_total{type="prometheus"}`)
-
-	rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="prometheus"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="prometheus"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	defaultTimestamp, err := protoparserutil.GetTimestamp(req)
+	defaultTimestamp, err := parserCommon.GetTimestamp(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, promscrape.IsMetadataEnabled(), func(rows []prometheus.Row, mms []prometheus.Metadata) error {
-		return insertRows(at, rows, mms, extraLabels)
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
+		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)
 	})
 }
 
-func insertRows(at *auth.Token, rows []prometheus.Row, mms []prometheus.Metadata, extraLabels []prompb.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
 	tssDst := ctx.WriteRequest.Timeseries[:0]
-	mmsDst := ctx.WriteRequest.Metadata[:0]
 	labels := ctx.Labels[:0]
 	samples := ctx.Samples[:0]
 	for i := range rows {
 		r := &rows[i]
 		labelsLen := len(labels)
-		labels = append(labels, prompb.Label{
+		labels = append(labels, prompbmarshal.Label{
 			Name:  "__name__",
 			Value: r.Metric,
 		})
 		for j := range r.Tags {
 			tag := &r.Tags[j]
-			labels = append(labels, prompb.Label{
+			labels = append(labels, prompbmarshal.Label{
 				Name:  tag.Key,
 				Value: tag.Value,
 			})
 		}
 		labels = append(labels, extraLabels...)
-		samples = append(samples, prompb.Sample{
+		samples = append(samples, prompbmarshal.Sample{
 			Value:     r.Value,
 			Timestamp: r.Timestamp,
 		})
-		tssDst = append(tssDst, prompb.TimeSeries{
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: samples[len(samples)-1:],
 		})
 	}
-	var accountID, projectID uint32
-	if at != nil {
-		accountID = at.AccountID
-		projectID = at.ProjectID
-	}
-	for i := range mms {
-		mm := &mms[i]
-		mmsDst = append(mmsDst, prompb.MetricMetadata{
-			MetricFamilyName: mm.Metric,
-			Help:             mm.Help,
-			Type:             mm.Type,
-			// there is no unit in Prometheus exposition formats
-
-			AccountID: accountID,
-			ProjectID: projectID,
-		})
-	}
 	ctx.WriteRequest.Timeseries = tssDst
-	ctx.WriteRequest.Metadata = mmsDst
 	ctx.Labels = labels
 	ctx.Samples = samples
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
 	rowsInserted.Add(len(rows))
-	metadataInserted.Add(len(mms))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))
-		metadataTenantInserted.Get(at).Add(len(mms))
 	}
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 )
 
 var (
@@ -26,11 +26,11 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 
@@ -44,17 +44,17 @@ func setUp() {
 		log.Fatalf("unable to set %q with value %q, err: %v", remoteWriteFlag, srv.URL, err)
 	}
 	logger.Init()
-	protoparserutil.StartUnmarshalWorkers()
+	common.StartUnmarshalWorkers()
 	remotewrite.Init()
 	testOutput = &bytes.Buffer{}
 	logger.SetOutputForTests(testOutput)
 }
 
 func tearDown() {
-	protoparserutil.StopUnmarshalWorkers()
+	common.StopUnmarshalWorkers()
 	srv.Close()
 	logger.ResetOutputForTest()
 	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()
-	fs.MustRemoveDir(tmpDataDir)
+	fs.MustRemoveAll(tmpDataDir)
 
 }