CI: disable docker scan, enable auto release to sandbox (#4476)

* disable docker scan

* disable nightly, enable auto release to sandbox

* remove whitespace

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -6,9 +6,9 @@ body:
     attributes:
       value: |
         Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
-        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
         and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/troubleshooting/) first.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
   - type: textarea
     id: describe-the-bug
     attributes:
@@ -60,12 +60,12 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229-victoriametrics/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176-victoriametrics-cluster/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
     validations:
       required: false
   - type: textarea

.github/ISSUE_TEMPLATE/question.yml

@@ -1,32 +0,0 @@
-name: Question
-description: Ask a question regarding VictoriaMetrics or its components
-labels: [question]
-body:
-  - type: textarea
-    id: describe-the-component
-    attributes:
-      label: Is your question request related to a specific component?
-      placeholder: |
-        VictoriaMetrics, vmagent, vmalert, vmui, etc...
-    validations:
-      required: false
-  - type: textarea
-    id: describe-the-question
-    attributes:
-      label: Describe the question in detail
-      description: |
-        A clear and concise description of the issue and the question.
-    validations:
-      required: true
-  - type: checkboxes
-    id: troubleshooting
-    attributes:
-      label: Troubleshooting docs
-      description: I am familiar with the following troubleshooting docs
-      options:
-        - label: General - https://docs.victoriametrics.com/troubleshooting/
-          required: false
-        - label: vmagent - https://docs.victoriametrics.com/vmagent/#troubleshooting
-          required: false
-        - label: vmalert - https://docs.victoriametrics.com/vmalert/#troubleshooting
-          required: false

.github/dependabot.yml

@@ -8,12 +8,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    open-pull-requests-limit: 0
   - package-ecosystem: "bundler"
     directory: "/docs"
     schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 0
+      interval: "daily"
   - package-ecosystem: "gomod"
     directory: "/app/vmui/packages/vmui/web"
     schedule:

.github/pull_request_template.md

@@ -1,9 +0,0 @@
-### Describe Your Changes
-
-Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications.
-
-### Checklist
-
-The following checks are **mandatory**:
-
-- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).

.github/workflows/build.yml

@@ -1,54 +0,0 @@
-name: build
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
-      - '**/Makefile'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-      - '**/Dockerfile*' # The trailing * is for app/vmui/Dockerfile-*.
-      - '**/Makefile'
-
-permissions:
-  contents: read
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
-      - name: Run crossbuild
-        run: make crossbuild

.github/workflows/check-licenses.yml

@@ -14,25 +14,13 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
+      - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.21.4
+        id: go
       - name: Code checkout
         uses: actions/checkout@master
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
-
       - name: Check License
-        run: make check-licenses
+        run: |
+          make check-licenses

.github/workflows/codeql-analysis-go.yml

@@ -1,62 +0,0 @@
-name: 'CodeQL Go'
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.go'
-
-concurrency:
-  cancel-in-progress: true
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: go
-
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: 'language:go'

.github/workflows/codeql-analysis-js-ts.yml

@@ -1,46 +0,0 @@
-name: 'CodeQL JS/TS'
-
-on:
-  push:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
-  pull_request:
-    branches:
-      - cluster
-      - master
-    paths:
-      - '**.js'
-      - '**.ts'
-      - '**.tsx'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: javascript-typescript
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: 'language:js/ts'

.github/workflows/codeql-analysis-js.yml

@@ -0,0 +1,46 @@
+name: "CodeQL - JS"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths:
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+  
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["javascript"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master, cluster]
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "**.txt"
+      - "**.js"
+  schedule:
+    - cron: "30 18 * * 2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["go"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.21.4
+          check-latest: true
+          cache: true
+        if: ${{ matrix.language == 'go' }}
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -1,25 +1,25 @@
 name: main
-
 on:
   push:
     branches:
-      - cluster
       - master
-    paths:
-      - '**.go'
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
   pull_request:
     branches:
-      - cluster
       - master
-    paths:
-      - '**.go'
-
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
 permissions:
   contents: read
 
 concurrency:
-  cancel-in-progress: true
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   lint:
@@ -27,67 +27,69 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v3
         with:
-          cache: false
-          go-version: stable
+          go-version: 1.21.4
+          check-latest: true
+          cache: true
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-all-
-
-      - name: Run check-all
+      - name: Dependencies
         run: |
+          make install-golangci-lint
           make check-all
           git diff --exit-code
 
   test:
-    name: test
     needs: lint
-    runs-on: ubuntu-latest
-
     strategy:
       matrix:
-        scenario:
-          - 'test-full'
-          - 'test-full-386'
-          - 'test-pure'
-
+        scenario: ["test-full", "test-pure", "test-full-386"]
+    name: test
+    runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.21.4
+          check-latest: true
+          cache: true
+
+      - name: run tests
+        run: |
+          make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.txt
+
+  build:
+    needs: test
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v3
         with:
-          cache: false
-          go-version: stable
+          go-version: 1.21.4
+          check-latest: true
+          cache: true
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
+      - uses: actions/cache@v3
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
+          path: gocache-for-docker
+          key:  gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
 
-      - name: Run tests
-        run: make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v4
-        with:
-          file: ./coverage.txt
+      - name: Build
+        run: |
+          make vmcluster-crossbuild

.github/workflows/sync-docs.yml

@@ -1,51 +0,0 @@
-name: publish-docs
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'docs/**'
-  workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-        with:
-          path: main
-      - name: Checkout private code
-        uses: actions/checkout@v4
-        with:
-          repository: VictoriaMetrics/vmdocs
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: docs
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v6
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-          workdir: docs
-      - name: Set short git commit SHA
-        id: vars
-        run: |
-          calculatedSha=$(git rev-parse --short ${{ github.sha }})
-          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
-        working-directory: main
-      - name: update code and commit
-        run: |
-          rm -rf content
-          cp -r ../main/docs content
-          make clean-after-copy
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          git add .
-          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
-          git push
-        working-directory: docs

.gitignore

@@ -8,7 +8,6 @@
 *.test
 *.swp
 /gocache-for-docker
-/victoria-logs-data
 /victoria-metrics-data
 /vmagent-remotewrite-data
 /vmstorage-data
@@ -21,7 +20,4 @@
 Gemfile.lock
 /_site
 _site
-*.tmp
-/docs/.jekyll-metadata
-coverage.txt
-cspell.json
+*.tmp

.golangci.yml

@@ -1,22 +1,15 @@
 run:
   timeout: 2m
 
-linters:
-  enable:
+enable:
   - revive
 
 issues:
   exclude-rules:
-  - linters:
-    - staticcheck
-    text: "SA(4003|1019|5011):"
-  include:
-  - EXC0012
-  - EXC0014
+    - linters:
+      - staticcheck
+      text: "SA(4003|1019|5011):"
 
 linters-settings:
   errcheck:
-    exclude-functions:
-      - "fmt.Fprintf"
-      - "fmt.Fprint"
-      - "(net/http.ResponseWriter).Write"
+    exclude: ./errcheck_excludes.txt

CODE_OF_CONDUCT_RU.md

@@ -0,0 +1,120 @@
+
+# Кодекс Поведения участника
+
+## Наши обязательства
+
+Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
+свободным от притеснений для всех, независимо от возраста, телосложения,
+видимых или невидимых ограничений способности, этнической принадлежности,
+половых признаков, гендерной идентичности и выражения, уровня опыта,
+образования, социо-экономического статуса, национальности, внешности,
+расы, религии, или сексуальной идентичности и ориентации.
+
+Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
+дружелюбное, многообразное, инклюзивное и здоровое сообщество.
+
+## Наши стандарты
+
+Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
+
+* Проявление доброты и эмпатии к другим участникам проекта
+* Уважение к чужой точке зрения и опыту
+* Конструктивная критика и принятие конструктивной критики
+* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
+  и извлечение уроков из опыта
+* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
+
+Примеры неприемлемого поведения участников включают в себя:
+
+* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
+* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
+* Публичное или приватное домогательство
+* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
+* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
+
+## Обязанности
+
+Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
+поведения и будут предпринимать соответствующие и честные меры по исправлению положения
+в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
+
+Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
+комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
+с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
+
+## Область применения
+
+Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
+а также когда человек официально представляет сообщество в публичных местах.
+Примеры представления проекта или сообщества включают использование официальной электронной почты,
+публикации в официальном аккаунте в социальных сетях,
+или упоминания как представителя в онлайн или оффлайн мероприятии.
+
+## Приведение в исполнение
+
+О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
+поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
+Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
+
+Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
+неприкосновенность автора сообщения.
+
+## Руководство по исполнению
+
+Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
+чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
+
+### 1. Исправление
+
+**Общественное влияние**: Использование недопустимой лексики или другое поведение,
+считающиеся непрофессиональным или нежелательным в сообществе.
+
+**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
+объясняющее суть нарушения и почему такое поведение
+было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
+
+### 2. Предупреждение
+
+**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
+
+**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
+На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
+включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
+в публичных пространствах, а так же во внешних каналах,
+таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
+
+### 3. Временный бан
+
+**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
+включая продолжительное неуместное поведение.
+
+**Последствия**: Временный запрет (бан) на любое взаимодействие
+или публичное общение с сообществом на определенный период времени.
+На этот период не допускается публичное или личное взаимодействие с людьми,
+вовлеченными в инцидент, включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса.
+Нарушение этих правил влечет за собой вечный бан.
+
+### 4. Вечный бан
+
+**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
+включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
+или проявление агрессии либо пренебрежительного отношения к категориям лиц.
+
+**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
+
+## Атрибуция
+
+Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
+версии 2.0, доступной по адресу
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
+
+Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

CONTRIBUTING.md

@@ -1 +1,16 @@
-The document has been moved [here](https://docs.victoriametrics.com/contributing/).
+If you like VictoriaMetrics and want to contribute, then we need the following:
+
+- Filing issues and feature requests [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
+- Spreading a word about VictoriaMetrics: conference talks, articles, comments, experience sharing with colleagues.
+- Updating documentation.
+
+We are open to third-party pull requests provided they follow [KISS design principle](https://en.wikipedia.org/wiki/KISS_principle):
+
+- Prefer simple code and architecture.
+- Avoid complex abstractions.
+- Avoid magic code and fancy algorithms.
+- Avoid [big external dependencies](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d).
+- Minimize the number of moving parts in the distributed system.
+- Avoid automated decisions, which may hurt cluster availability, consistency or performance.
+
+Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2024 VictoriaMetrics, Inc.
+   Copyright 2019-2023 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,7 +1,5 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
-MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
@@ -17,10 +15,7 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include cspell/Makefile
-include docs/Makefile
 include deployment/*/Makefile
-include dashboards/Makefile
 include package/release/Makefile
 
 all: \
@@ -71,21 +66,14 @@ vmcluster-openbsd-amd64: \
 	vmselect-openbsd-amd64 \
 	vmstorage-openbsd-amd64
 
-vmcluster-windows-amd64: \
-	vminsert-windows-amd64 \
-	vmselect-windows-amd64 \
-	vmstorage-windows-amd64
-
-crossbuild: vmcluster-crossbuild
-
-vmcluster-crossbuild:
-	$(MAKE_PARALLEL) vmcluster-linux-amd64 \
-		vmcluster-linux-arm64 \
-		vmcluster-linux-arm \
-		vmcluster-linux-ppc64le \
-		vmcluster-linux-386 \
-		vmcluster-freebsd-amd64 \
-		vmcluster-openbsd-amd64
+vmcluster-crossbuild: \
+	vmcluster-linux-amd64 \
+	vmcluster-linux-arm64 \
+	vmcluster-linux-arm \
+	vmcluster-linux-ppc64le \
+	vmcluster-linux-386 \
+	vmcluster-freebsd-amd64 \
+	vmcluster-openbsd-amd64
 
 publish: \
 	publish-vminsert \
@@ -99,20 +87,19 @@ package: \
 
 publish-release:
 	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
+	git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && LATEST_TAG=enterprise-cluster-stable $(MAKE) release publish
 
-release:
-	$(MAKE_PARALLEL) release-vmcluster
+release: \
+	release-vmcluster
 
 release-vmcluster: \
 	release-vmcluster-linux-amd64 \
 	release-vmcluster-linux-arm64 \
 	release-vmcluster-freebsd-amd64 \
-	release-vmcluster-openbsd-amd64 \
-	release-vmcluster-windows-amd64
+	release-vmcluster-openbsd-amd64
 
 release-vmcluster-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
@@ -126,9 +113,6 @@ release-vmcluster-freebsd-amd64:
 release-vmcluster-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
 
-release-vmcluster-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-vmcluster-windows-goarch
-
 release-vmcluster-goos-goarch: \
 	vminsert-$(GOOS)-$(GOARCH)-prod \
 	vmselect-$(GOOS)-$(GOARCH)-prod \
@@ -148,25 +132,6 @@ release-vmcluster-goos-goarch: \
 		vmselect-$(GOOS)-$(GOARCH)-prod \
 		vmstorage-$(GOOS)-$(GOARCH)-prod
 
-release-vmcluster-windows-goarch: \
-	vminsert-windows-$(GOARCH)-prod \
-	vmselect-windows-$(GOARCH)-prod \
-	vmstorage-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vminsert-windows-$(GOARCH)-prod.exe \
-			vmselect-windows-$(GOARCH)-prod.exe \
-			vmstorage-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			vminsert-windows-$(GOARCH)-prod.exe \
-			vmselect-windows-$(GOARCH)-prod.exe \
-			vmstorage-windows-$(GOARCH)-prod.exe \
-		> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		vminsert-windows-$(GOARCH)-prod.exe \
-		vmselect-windows-$(GOARCH)-prod.exe \
-		vmstorage-windows-$(GOARCH)-prod.exe
-
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -180,8 +145,6 @@ vet:
 
 check-all: fmt vet golangci-lint govulncheck
 
-clean-checkers: remove-golangci-lint remove-govulncheck
-
 test:
 	go test ./lib/... ./app/...
 
@@ -208,7 +171,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.22
+	go mod tidy -compat=1.20
 	go mod vendor
 
 app-local:
@@ -234,10 +197,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
-
-remove-golangci-lint:
-	rm -rf `which golangci-lint`
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.2
 
 govulncheck: install-govulncheck
 	govulncheck ./...
@@ -245,9 +205,6 @@ govulncheck: install-govulncheck
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 
-remove-govulncheck:
-	rm -rf `which govulncheck`
-
 install-wwhrd:
 	which wwhrd || go install github.com/frapposelli/wwhrd@latest
 
@@ -255,32 +212,26 @@ check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
 
 copy-docs:
-# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
-# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
-	echo "---" > ${DST}
+	echo '' > ${DST}
 	@if [ ${ORDER} -ne 0 ]; then \
-		echo "sort: ${ORDER}" >> ${DST}; \
-		echo "weight: ${ORDER}" >> ${DST}; \
-		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
+		echo "---\nsort: ${ORDER}\n---\n" > ${DST}; \
 	fi
-
-	echo "title: ${TITLE}" >> ${DST}
-	@if [ ${OLD_URL} ]; then \
-		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
-	fi
-	echo "---" >> ${DST}
 	cat ${SRC} >> ${DST}
-	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"\//' ${DST}
-	sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"\//' ${DST}
-	sed -i='.tmp' 's/](docs\//](/' ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
 	rm -rf docs/*.tmp
 
-# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
-# For ORDER=0 it adds no order tag/weight tag.
-# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
+# Copies docs for all components and adds the order tag.
+# For ORDER=0 it adds no order tag.
 # Images starting with <img src="docs/ are replaced with <img src="
-# Cluster docs are supposed to be ordered as 2nd.
+# Cluster docs are supposed to be ordered as 9th.
 # The rest of docs is ordered manually.
 docs-sync:
-	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md OLD_URL='/Cluster-VictoriaMetrics.html' ORDER=2 TITLE='Cluster version' $(MAKE) copy-docs
+	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md ORDER=2 $(MAKE) copy-docs
+	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=3 $(MAKE) copy-docs
+	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=4 $(MAKE) copy-docs
+	SRC=app/vmauth/README.md DST=docs/vmauth.md ORDER=5 $(MAKE) copy-docs
+	SRC=app/vmbackup/README.md DST=docs/vmbackup.md ORDER=6 $(MAKE) copy-docs
+	SRC=app/vmrestore/README.md DST=docs/vmrestore.md ORDER=7 $(MAKE) copy-docs
+	SRC=app/vmctl/README.md DST=docs/vmctl.md ORDER=8 $(MAKE) copy-docs
+	SRC=app/vmgateway/README.md DST=docs/vmgateway.md ORDER=9 $(MAKE) copy-docs
+	SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md ORDER=10 $(MAKE) copy-docs

SECURITY.md

@@ -2,16 +2,12 @@
 
 ## Supported Versions
 
-The following versions of VictoriaMetrics receive regular security fixes:
-
 | Version | Supported          |
 |---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
-| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| other releases  | :x:                |
-
-See [this page](https://victoriametrics.com/security/) for more details.
+| 1.81.x  | :white_check_mark: |
+| 1.80.x  | :x:                |
+| 1.79.x  | :white_check_mark: |
+| < 1.78  | :x:                |
 
 ## Reporting a Vulnerability
 

app/victoria-logs/Makefile

@@ -1,113 +0,0 @@
-# All these commands must run from repository root.
-
-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) app-local
-
-victoria-logs-race:
-	APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
-
-victoria-logs-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker
-
-victoria-logs-pure-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
-
-victoria-logs-linux-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
-
-victoria-logs-linux-arm-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
-
-victoria-logs-linux-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
-
-victoria-logs-linux-ppc64le-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
-
-victoria-logs-linux-386-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
-
-victoria-logs-darwin-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
-
-victoria-logs-darwin-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
-
-victoria-logs-freebsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
-
-victoria-logs-openbsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
-
-victoria-logs-windows-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
-
-package-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker
-
-package-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
-
-package-victoria-logs-amd64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
-
-package-victoria-logs-arm:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
-
-package-victoria-logs-arm64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
-
-package-victoria-logs-ppc64le:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
-
-package-victoria-logs-386:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-386
-
-publish-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) publish-via-docker
-
-victoria-logs-linux-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-ppc64le:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-s390x:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-loong64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-386:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-freebsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-openbsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-windows-amd64:
-	GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
-
-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) app-local-pure
-
-run-victoria-logs:
-	mkdir -p victoria-logs-data
-	DOCKER_OPTS='-v $(shell pwd)/victoria-logs-data:/victoria-logs-data' \
-	APP_NAME=victoria-logs \
-	ARGS='' \
-	$(MAKE) run-via-docker

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image
-FROM $base_image
-
-EXPOSE 9428
-
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary
-COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -1,105 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-)
-
-var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9428"}
-	}
-	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
-	startTime := time.Now()
-
-	vlstorage.Init()
-	vlselect.Init()
-	vlinsert.Init()
-
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
-	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
-
-	pushmetrics.Init()
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
-
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
-
-	vlinsert.Stop()
-	vlselect.Stop()
-	vlstorage.Stop()
-
-	fs.MustStopDirRemover()
-
-	logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
-}
-
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
-			return false
-		}
-		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victorialogs/'>https://docs.victoriametrics.com/victorialogs/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints:</br>")
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"select/vmui", "Web UI for VictoriaLogs"},
-			{"metrics", "available service metrics"},
-			{"flags", "command-line flags"},
-		})
-		return true
-	}
-	if vlinsert.RequestHandler(w, r) {
-		return true
-	}
-	if vlselect.RequestHandler(w, r) {
-		return true
-	}
-	return false
-}
-
-func usage() {
-	const s = `
-victoria-logs is a log management and analytics service.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/
-`
-	flagutil.Usage(s)
-}

app/victoria-logs/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 9428
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG TARGETARCH
-COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/vlinsert/elasticsearch/bulk_response.qtpl

@@ -1,20 +0,0 @@
-{% stripspace %}
-
-{% func BulkResponse(n int, tookMs int64) %}
-{
-	"took":{%dl tookMs %},
-	"errors":false,
-	"items":[
-		{% for i := 0; i < n; i++ %}
-		{
-			"create":{
-				"status":201
-			}
-		}
-		{% if i+1 < n %},{% endif %}
-		{% endfor %}
-	]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlinsert/elasticsearch/bulk_response.qtpl.go

@@ -1,69 +0,0 @@
-// Code generated by qtc from "bulk_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-package elasticsearch
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-func StreamBulkResponse(qw422016 *qt422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-	qw422016.N().S(`{"took":`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().DL(tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().S(`,"errors":false,"items":[`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-	for i := 0; i < n; i++ {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-		qw422016.N().S(`{"create":{"status":201}}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		if i+1 < n {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	qw422016.N().S(`]}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func WriteBulkResponse(qq422016 qtio422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	StreamBulkResponse(qw422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func BulkResponse(n int, tookMs int64) string {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	WriteBulkResponse(qb422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	return qs422016
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -1,274 +0,0 @@
-package elasticsearch
-
-import (
-	"bufio"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-)
-
-var (
-	elasticsearchVersion = flag.String("elasticsearch.version", "8.9.0", "Elasticsearch version to report to client")
-)
-
-// RequestHandler processes Elasticsearch insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	// This header is needed for Logstash
-	w.Header().Set("X-Elastic-Product", "Elasticsearch")
-
-	if strings.HasPrefix(path, "/_ilm/policy") {
-		// Return fake response for Elasticsearch ilm request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_index_template") {
-		// Return fake response for Elasticsearch index template request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_ingest") {
-		// Return fake response for Elasticsearch ingest pipeline request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_nodes") {
-		// Return fake response for Elasticsearch nodes discovery request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	switch path {
-	case "/":
-		switch r.Method {
-		case http.MethodGet:
-			// Return fake response for Elasticsearch ping request.
-			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
-			fmt.Fprintf(w, `{
-			"version": {
-				"number": %q
-			}
-		}`, *elasticsearchVersion)
-		case http.MethodHead:
-			// Return empty response for Logstash ping request.
-		}
-
-		return true
-	case "/_license":
-		// Return fake response for Elasticsearch license request.
-		fmt.Fprintf(w, `{
-			"license": {
-				"uid": "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
-				"type": "oss",
-				"status": "active",
-				"expiry_date_in_millis" : 4000000000000
-			}
-		}`)
-		return true
-	case "/_bulk":
-		startTime := time.Now()
-		bulkRequestsTotal.Inc()
-
-		cp, err := insertutils.GetCommonParams(r)
-		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		if err := vlstorage.CanWriteData(); err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		lmp := cp.NewLogMessageProcessor()
-		isGzip := r.Header.Get("Content-Encoding") == "gzip"
-		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, lmp)
-		lmp.MustClose()
-		if err != nil {
-			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
-			return true
-		}
-
-		tookMs := time.Since(startTime).Milliseconds()
-		bw := bufferedwriter.Get(w)
-		defer bufferedwriter.Put(bw)
-		WriteBulkResponse(bw, n, tookMs)
-		_ = bw.Flush()
-
-		// update bulkRequestDuration only for successfully parsed requests
-		// There is no need in updating bulkRequestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		bulkRequestDuration.UpdateDuration(startTime)
-
-		return true
-	default:
-		return false
-	}
-}
-
-var (
-	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
-	rowsIngestedTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
-	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
-)
-
-func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp insertutils.LogMessageProcessor) (int, error) {
-	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
-
-	if isGzip {
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
-		}
-		defer common.PutGzipReader(zr)
-		r = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lb := lineBufferPool.Get()
-	defer lineBufferPool.Put(lb)
-
-	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
-	sc := bufio.NewScanner(wcr)
-	sc.Buffer(lb.B, len(lb.B))
-
-	n := 0
-	nCheckpoint := 0
-	for {
-		ok, err := readBulkLine(sc, timeField, msgField, lmp)
-		wcr.DecConcurrency()
-		if err != nil || !ok {
-			rowsIngestedTotal.Add(n - nCheckpoint)
-			return n, err
-		}
-		n++
-		if batchSize := n - nCheckpoint; n >= 1000 {
-			rowsIngestedTotal.Add(batchSize)
-			nCheckpoint = n
-		}
-	}
-}
-
-var lineBufferPool bytesutil.ByteBufferPool
-
-func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
-	var line []byte
-
-	// Read the command, must be "create" or "index"
-	for len(line) == 0 {
-		if !sc.Scan() {
-			if err := sc.Err(); err != nil {
-				if errors.Is(err, bufio.ErrTooLong) {
-					return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
-						insertutils.MaxLineSizeBytes.IntN())
-				}
-				return false, err
-			}
-			return false, nil
-		}
-		line = sc.Bytes()
-	}
-	lineStr := bytesutil.ToUnsafeString(line)
-	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
-		return false, fmt.Errorf(`unexpected command %q; expecting "create" or "index"`, line)
-	}
-
-	// Decode log message
-	if !sc.Scan() {
-		if err := sc.Err(); err != nil {
-			if errors.Is(err, bufio.ErrTooLong) {
-				return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
-			}
-			return false, err
-		}
-		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
-	}
-	line = sc.Bytes()
-	p := logstorage.GetJSONParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
-	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	logstorage.RenameField(p.Fields, msgField, "_msg")
-	lmp.AddRow(ts, p.Fields)
-	logstorage.PutJSONParser(p)
-
-	return true, nil
-}
-
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		timestamp, err := parseElasticsearchTimestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
-	}
-	return 0, nil
-}
-
-func parseElasticsearchTimestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case - zero or empty timestamp must be substituted
-		// with the current time by the caller.
-		return 0, nil
-	}
-	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in milliseconds
-		n, err := strconv.ParseInt(s, 10, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
-		}
-		if n > int64(math.MaxInt64)/1e6 {
-			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
-		}
-		if n < int64(math.MinInt64)/1e6 {
-			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
-		}
-		n *= 1e6
-		return n, nil
-	}
-	if len(s) == len("YYYY-MM-DD") {
-		t, err := time.Parse("2006-01-02", s)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse date %q: %w", s, err)
-		}
-		return t.UnixNano(), nil
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot parse timestamp %q", s)
-	}
-	return nsecs, nil
-}

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -1,103 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-)
-
-func TestReadBulkRequest_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, "_time", "_msg", tlp)
-		if err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if rows != 0 {
-			t.Fatalf("unexpected non-zero rows=%d", rows)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`{"create":{}}`)
-	f(`{"creat":{}}
-{}`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadBulkRequest_Success(t *testing.T) {
-	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-
-		// Read the request without compression
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, timeField, msgField, tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-
-		// Read the request with compression
-		tlp = &insertutils.TestLogMessageProcessor{}
-		compressedData := compressData(data)
-		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest(r, true, timeField, msgField, tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("verification failure after compression: %s", err)
-		}
-	}
-
-	// Verify an empty data
-	f("", "_time", "_msg", 0, nil, "")
-	f("\n", "_time", "_msg", 0, nil, "")
-	f("\n\n", "_time", "_msg", 0, nil, "")
-
-	// Verify non-empty data
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	rowsExpected := 3
-	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}`
-	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
-}
-
-func compressData(s string) string {
-	var bb bytes.Buffer
-	zw := gzip.NewWriter(&bb)
-	if _, err := zw.Write([]byte(s)); err != nil {
-		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
-	}
-	if err := zw.Close(); err != nil {
-		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
-	}
-	return bb.String()
-}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -1,50 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-)
-
-func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("gzip:off", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, false)
-	})
-	b.Run("gzip:on", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, true)
-	})
-}
-
-func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	if isGzip {
-		data = compressData(data)
-	}
-	dataBytes := bytesutil.ToUnsafeBytes(data)
-
-	timeField := "@timestamp"
-	msgField := "message"
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		for pb.Next() {
-			r.Reset(dataBytes)
-			_, err := readBulkRequest(r, isGzip, timeField, msgField, blp)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}

app/vlinsert/insertutils/common_params.go

@@ -1,200 +0,0 @@
-package insertutils
-
-import (
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-// CommonParams contains common HTTP parameters used by log ingestion APIs.
-//
-// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
-type CommonParams struct {
-	TenantID     logstorage.TenantID
-	TimeField    string
-	MsgField     string
-	StreamFields []string
-	IgnoreFields []string
-
-	Debug           bool
-	DebugRequestURI string
-	DebugRemoteAddr string
-}
-
-// GetCommonParams returns CommonParams from r.
-func GetCommonParams(r *http.Request) (*CommonParams, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// Extract time field name from _time_field query arg
-	var timeField = "_time"
-	if tf := r.FormValue("_time_field"); tf != "" {
-		timeField = tf
-	}
-
-	// Extract message field name from _msg_field query arg
-	var msgField = ""
-	if msgf := r.FormValue("_msg_field"); msgf != "" {
-		msgField = msgf
-	}
-
-	streamFields := httputils.GetArray(r, "_stream_fields")
-	ignoreFields := httputils.GetArray(r, "ignore_fields")
-
-	debug := httputils.GetBool(r, "debug")
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-	if debug {
-		debugRequestURI = httpserver.GetRequestURI(r)
-		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
-	}
-
-	cp := &CommonParams{
-		TenantID:        tenantID,
-		TimeField:       timeField,
-		MsgField:        msgField,
-		StreamFields:    streamFields,
-		IgnoreFields:    ignoreFields,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
-	}
-	return cp, nil
-}
-
-// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
-	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
-	cp := &CommonParams{
-		TenantID:  tenantID,
-		TimeField: "timestamp",
-		MsgField:  "message",
-		StreamFields: []string{
-			"hostname",
-			"app_name",
-			"proc_id",
-		},
-	}
-
-	return cp
-}
-
-// LogMessageProcessor is an interface for log message processors.
-type LogMessageProcessor interface {
-	// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
-	//
-	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
-	AddRow(timestamp int64, fields []logstorage.Field)
-
-	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
-	MustClose()
-}
-
-type logMessageProcessor struct {
-	mu            sync.Mutex
-	wg            sync.WaitGroup
-	stopCh        chan struct{}
-	lastFlushTime time.Time
-
-	cp *CommonParams
-	lr *logstorage.LogRows
-}
-
-func (lmp *logMessageProcessor) initPeriodicFlush() {
-	lmp.lastFlushTime = time.Now()
-
-	lmp.wg.Add(1)
-	go func() {
-		defer lmp.wg.Done()
-
-		d := timeutil.AddJitterToDuration(time.Second)
-		ticker := time.NewTicker(d)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-lmp.stopCh:
-				return
-			case <-ticker.C:
-				lmp.mu.Lock()
-				if time.Since(lmp.lastFlushTime) >= d {
-					lmp.flushLocked()
-				}
-				lmp.mu.Unlock()
-			}
-		}
-	}()
-}
-
-// AddRow adds new log message to lmp with the given timestamp and fields.
-func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	if len(fields) > *MaxFieldsPerLine {
-		rf := logstorage.RowFormatter(fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// flushLocked must be called under locked lmp.mu.
-func (lmp *logMessageProcessor) flushLocked() {
-	lmp.lastFlushTime = time.Now()
-	vlstorage.MustAddRows(lmp.lr)
-	lmp.lr.ResetKeepSettings()
-}
-
-// MustClose flushes the remaining data to the underlying storage and closes lmp.
-func (lmp *logMessageProcessor) MustClose() {
-	close(lmp.stopCh)
-	lmp.wg.Wait()
-
-	lmp.flushLocked()
-	logstorage.PutLogRows(lmp.lr)
-	lmp.lr = nil
-}
-
-// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
-//
-// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
-func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	lmp := &logMessageProcessor{
-		cp: cp,
-		lr: lr,
-
-		stopCh: make(chan struct{}),
-	}
-	lmp.initPeriodicFlush()
-
-	return lmp
-}
-
-var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/flags.go

@@ -1,15 +0,0 @@
-package insertutils
-
-import (
-	"flag"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-)
-
-var (
-	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
-
-	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
-)

app/vlinsert/insertutils/testutils.go

@@ -1,53 +0,0 @@
-package insertutils
-
-import (
-	"fmt"
-	"reflect"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// TestLogMessageProcessor implements LogMessageProcessor for testing.
-type TestLogMessageProcessor struct {
-	timestamps []int64
-	rows       []string
-}
-
-// AddRow adds row with the given timestamp and fields to tlp
-func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
-	tlp.timestamps = append(tlp.timestamps, timestamp)
-	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
-}
-
-// MustClose closes tlp.
-func (tlp *TestLogMessageProcessor) MustClose() {
-}
-
-// Verify verifies the number of rows, timestamps and results after AddRow calls.
-func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error {
-	result := strings.Join(tlp.rows, "\n")
-	if len(tlp.rows) != rowsExpected {
-		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), rowsExpected, result, resultExpected)
-	}
-
-	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
-		return fmt.Errorf("unexpected timestamps;\ngot\n%d\nwant\n%d", tlp.timestamps, timestampsExpected)
-	}
-	if result != resultExpected {
-		return fmt.Errorf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-	}
-
-	return nil
-}
-
-// BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.
-type BenchmarkLogMessageProcessor struct{}
-
-// AddRow implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field) {
-}
-
-// MustClose implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) MustClose() {
-}

app/vlinsert/insertutils/timestamp.go

@@ -1,33 +0,0 @@
-package insertutils
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
-//
-// The value for the timeField is set to empty string after returning from the function,
-// so it could be ignored during data ingestion.
-//
-// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		if f.Value == "" || f.Value == "0" {
-			return time.Now().UnixNano(), nil
-		}
-		nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
-		if !ok {
-			return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
-		}
-		f.Value = ""
-		return nsecs, nil
-	}
-	return time.Now().UnixNano(), nil
-}

app/vlinsert/insertutils/timestamp_test.go

@@ -1,75 +0,0 @@
-package insertutils
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs != nsecsExpected {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
-		}
-
-		for _, f := range fields {
-			if f.Name == timeField {
-				if f.Value != "" {
-					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
-				}
-			}
-		}
-	}
-
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20Z"},
-	}, 1718753840000000000)
-
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
-	}, 1718725040000000000)
-
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
-	}, 1718773640123000000)
-
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
-		{Name: "foo", Value: "bar"},
-	}, 1718773640123456789)
-}
-
-func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		fields := []logstorage.Field{
-			{Name: "time", Value: s},
-		}
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if nsecs != 0 {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
-		}
-	}
-
-	f("foobar")
-
-	// no Z at the end
-	f("2024-06-18T23:37:20")
-
-	// incomplete time
-	f("2024-06-18")
-	f("2024-06-18T23:37")
-}

app/vlinsert/jsonline/jsonline.go

@@ -1,135 +0,0 @@
-package jsonline
-
-import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// RequestHandler processes jsonline insert requests
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	startTime := time.Now()
-	w.Header().Add("Content-Type", "application/json")
-
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			logger.Errorf("cannot read gzipped jsonline request: %s", err)
-			return
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	lmp := cp.NewLogMessageProcessor()
-	err = processStreamInternal(reader, cp.TimeField, cp.MsgField, lmp)
-	lmp.MustClose()
-
-	if err != nil {
-		logger.Errorf("jsonline: %s", err)
-	} else {
-		// update requestDuration only for successfully parsed requests.
-		// There is no need in updating requestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		requestDuration.UpdateDuration(startTime)
-	}
-}
-
-func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertutils.LogMessageProcessor) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lb := lineBufferPool.Get()
-	defer lineBufferPool.Put(lb)
-
-	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
-	sc := bufio.NewScanner(wcr)
-	sc.Buffer(lb.B, len(lb.B))
-
-	n := 0
-	for {
-		ok, err := readLine(sc, timeField, msgField, lmp)
-		wcr.DecConcurrency()
-		if err != nil {
-			errorsTotal.Inc()
-			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
-		}
-		if !ok {
-			return nil
-		}
-		n++
-		rowsIngestedTotal.Inc()
-	}
-}
-
-func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
-	var line []byte
-	for len(line) == 0 {
-		if !sc.Scan() {
-			if err := sc.Err(); err != nil {
-				if errors.Is(err, bufio.ErrTooLong) {
-					return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
-				}
-				return false, err
-			}
-			return false, nil
-		}
-		line = sc.Bytes()
-	}
-
-	p := logstorage.GetJSONParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot get timestamp: %w", err)
-	}
-	logstorage.RenameField(p.Fields, msgField, "_msg")
-	lmp.AddRow(ts, p.Fields)
-	logstorage.PutJSONParser(p)
-
-	return true, nil
-}
-
-var lineBufferPool bytesutil.ByteBufferPool
-
-var (
-	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
-
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
-
-	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
-)

app/vlinsert/jsonline/jsonline_test.go

@@ -1,55 +0,0 @@
-package jsonline
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-)
-
-func TestProcessStreamInternal_Success(t *testing.T) {
-	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, timeField, msgField, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	rowsExpected := 3
-	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}`
-	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "time", "", tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid json
-	f("foobar")
-
-	// invalid timestamp field
-	f(`{"time":"foobar"}`)
-}

app/vlinsert/loki/loki.go

@@ -1,59 +0,0 @@
-package loki
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// RequestHandler processes Loki insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/api/v1/push":
-		handleInsert(r, w)
-		return true
-	case "/ready":
-		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("ready"))
-		return true
-	default:
-		return false
-	}
-}
-
-// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-func handleInsert(r *http.Request, w http.ResponseWriter) {
-	contentType := r.Header.Get("Content-Type")
-	switch contentType {
-	case "application/json":
-		handleJSON(r, w)
-	default:
-		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		handleProtobuf(r, w)
-	}
-}
-
-func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// If parsed tenant is (0,0) it is likely to be default tenant
-	// Try parsing tenant from Loki headers
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		org := r.Header.Get("X-Scope-OrgID")
-		if org != "" {
-			tenantID, err := logstorage.ParseTenantID(org)
-			if err != nil {
-				return nil, err
-			}
-			cp.TenantID = tenantID
-		}
-
-	}
-
-	return cp, nil
-}

app/vlinsert/loki/loki_json.go

@@ -1,202 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-)
-
-var parserPool fastjson.ParserPool
-
-func handleJSON(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsJSONTotal.Inc()
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	lmp := cp.NewLogMessageProcessor()
-	n, err := parseJSONRequest(data, lmp)
-	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
-		return
-	}
-
-	rowsIngestedJSONTotal.Add(n)
-
-	// update requestJSONDuration only for successfully parsed requests
-	// There is no need in updating requestJSONDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestJSONDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
-	requestJSONDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
-)
-
-func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-
-	streamsV := v.Get("streams")
-	if streamsV == nil {
-		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
-	}
-	streams, err := streamsV.Array()
-	if err != nil {
-		return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
-	}
-
-	currentTimestamp := time.Now().UnixNano()
-	var commonFields []logstorage.Field
-	rowsIngested := 0
-	for _, stream := range streams {
-		// populate common labels from `stream` dict
-		commonFields = commonFields[:0]
-		labelsV := stream.Get("stream")
-		var labels *fastjson.Object
-		if labelsV != nil {
-			o, err := labelsV.Object()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
-			}
-			labels = o
-		}
-		labels.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
-			vStr, errLocal := v.StringBytes()
-			if errLocal != nil {
-				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-				return
-			}
-			commonFields = append(commonFields, logstorage.Field{
-				Name:  bytesutil.ToUnsafeString(k),
-				Value: bytesutil.ToUnsafeString(vStr),
-			})
-		})
-		if err != nil {
-			return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
-		}
-
-		// populate messages from `values` array
-		linesV := stream.Get("values")
-		if linesV == nil {
-			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
-		}
-		lines, err := linesV.Array()
-		if err != nil {
-			return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
-		}
-
-		fields := commonFields
-		for _, line := range lines {
-			lineA, err := line.Array()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
-			}
-			if len(lineA) != 2 {
-				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
-			}
-
-			// parse timestamp
-			timestamp, err := lineA[0].StringBytes()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
-			}
-			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
-			if err != nil {
-				return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
-			}
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			// parse log message
-			msg, err := lineA[1].StringBytes()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
-			}
-
-			fields = append(fields[:len(commonFields)], logstorage.Field{
-				Name:  "_msg",
-				Value: bytesutil.ToUnsafeString(msg),
-			})
-			lmp.AddRow(ts, fields)
-		}
-		rowsIngested += len(lines)
-	}
-
-	return rowsIngested, nil
-}
-
-func parseLokiTimestamp(s string) (int64, error) {
-	if s == "" {
-		// Special case - an empty timestamp must be substituted with the current time by the caller.
-		return 0, nil
-	}
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		// Fall back to parsing floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, err
-		}
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		n = int64(f)
-	}
-	if n < 0 {
-		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
-	}
-	return n, nil
-}

app/vlinsert/loki/loki_json_test.go

@@ -1,119 +0,0 @@
-package loki
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-)
-
-func TestParseJSONRequest_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		n, err := parseJSONRequest([]byte(s), tlp)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if n != 0 {
-			t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
-		}
-	}
-	f(``)
-
-	// Invalid json
-	f(`{}`)
-	f(`[]`)
-	f(`"foo"`)
-	f(`123`)
-
-	// invalid type for `streams` item
-	f(`{"streams":123}`)
-
-	// Missing `values` item
-	f(`{"streams":[{}]}`)
-
-	// Invalid type for `values` item
-	f(`{"streams":[{"values":"foobar"}]}`)
-
-	// Invalid type for `stream` item
-	f(`{"streams":[{"stream":[],"values":[]}]}`)
-
-	// Invalid type for `values` individual item
-	f(`{"streams":[{"values":[123]}]}`)
-
-	// Invalid length of `values` individual item
-	f(`{"streams":[{"values":[[]]}]}`)
-	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
-
-	// Invalid type for timestamp inside `values` individual item
-	f(`{"streams":[{"values":[[123,"456"]}]}`)
-
-	// Invalid type for log message
-	f(`{"streams":[{"values":[["123",1234]]}]}`)
-}
-
-func TestParseJSONRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-
-		n, err := parseJSONRequest([]byte(s), tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
-}

app/vlinsert/loki/loki_json_timing_test.go

@@ -1,79 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-)
-
-func BenchmarkParseJSONRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseJSONRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		data := getJSONBody(streams, rows, labels)
-		for pb.Next() {
-			_, err := parseJSONRequest(data, blp)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getJSONBody(streams, rows, labels int) []byte {
-	body := append([]byte{}, `{"streams":[`...)
-	now := time.Now().UnixNano()
-	valuePrefix := fmt.Sprintf(`["%d","value_`, now)
-
-	for i := 0; i < streams; i++ {
-		body = append(body, `{"stream":{`...)
-
-		for j := 0; j < labels; j++ {
-			body = append(body, `"label_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `":"value_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, '"')
-			if j < labels-1 {
-				body = append(body, ',')
-			}
-
-		}
-		body = append(body, `}, "values":[`...)
-
-		for j := 0; j < rows; j++ {
-			body = append(body, valuePrefix...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `"]`...)
-			if j < rows-1 {
-				body = append(body, ',')
-			}
-		}
-
-		body = append(body, `]}`...)
-		if i < streams-1 {
-			body = append(body, ',')
-		}
-
-	}
-
-	body = append(body, `]}`...)
-
-	return body
-}

app/vlinsert/loki/loki_protobuf.go

@@ -1,218 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
-)
-
-var (
-	bytesBufPool bytesutil.ByteBufferPool
-	pushReqsPool sync.Pool
-)
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-	wcr := writeconcurrencylimiter.GetReader(r.Body)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	lmp := cp.NewLogMessageProcessor()
-	n, err := parseProtobufRequest(data, lmp)
-	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
-		return
-	}
-
-	rowsIngestedProtobufTotal.Add(n)
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsProtobufTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
-	requestProtobufDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
-)
-
-func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
-	bb := bytesBufPool.Get()
-	defer bytesBufPool.Put(bb)
-
-	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
-	if err != nil {
-		return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
-	}
-	bb.B = buf
-
-	req := getPushRequest()
-	defer putPushRequest(req)
-
-	err = req.UnmarshalProtobuf(bb.B)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse request body: %w", err)
-	}
-
-	fields := getFields()
-	defer putFields(fields)
-
-	rowsIngested := 0
-	streams := req.Streams
-	currentTimestamp := time.Now().UnixNano()
-	for i := range streams {
-		stream := &streams[i]
-		// st.Labels contains labels for the stream.
-		// Labels are same for all entries in the stream.
-		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
-		if err != nil {
-			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
-		}
-		commonFieldsLen := len(fields.fields)
-
-		entries := stream.Entries
-		for j := range entries {
-			e := &entries[j]
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			for _, lp := range e.StructuredMetadata {
-				fields.fields = append(fields.fields, logstorage.Field{
-					Name:  lp.Name,
-					Value: lp.Value,
-				})
-			}
-
-			fields.fields = append(fields.fields, logstorage.Field{
-				Name:  "_msg",
-				Value: e.Line,
-			})
-
-			ts := e.Timestamp.UnixNano()
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			lmp.AddRow(ts, fields.fields)
-		}
-		rowsIngested += len(stream.Entries)
-	}
-	return rowsIngested, nil
-}
-
-func getFields() *fields {
-	v := fieldsPool.Get()
-	if v == nil {
-		return &fields{}
-	}
-	return v.(*fields)
-}
-
-func putFields(f *fields) {
-	f.fields = f.fields[:0]
-	fieldsPool.Put(f)
-}
-
-var fieldsPool sync.Pool
-
-type fields struct {
-	fields []logstorage.Field
-}
-
-// parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
-//
-// See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
-func parsePromLabels(dst []logstorage.Field, s string) ([]logstorage.Field, error) {
-	// Make sure s is wrapped into `{...}`
-	s = strings.TrimSpace(s)
-	if len(s) < 2 {
-		return nil, fmt.Errorf("too short string to parse: %q", s)
-	}
-	if s[0] != '{' {
-		return nil, fmt.Errorf("missing `{` at the beginning of %q", s)
-	}
-	if s[len(s)-1] != '}' {
-		return nil, fmt.Errorf("missing `}` at the end of %q", s)
-	}
-	s = s[1 : len(s)-1]
-
-	for len(s) > 0 {
-		// Parse label name
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("cannot find `=` char for label value at %s", s)
-		}
-		name := s[:n]
-		s = s[n+1:]
-
-		// Parse label value
-		qs, err := strconv.QuotedPrefix(s)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse value for label %q at %s: %w", name, s, err)
-		}
-		s = s[len(qs):]
-		value, err := strconv.Unquote(qs)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unquote value %q for label %q: %w", qs, name, err)
-		}
-
-		// Append the found field to dst.
-		dst = append(dst, logstorage.Field{
-			Name:  name,
-			Value: value,
-		})
-
-		// Check whether there are other labels remaining
-		if len(s) == 0 {
-			break
-		}
-		if !strings.HasPrefix(s, ",") {
-			return nil, fmt.Errorf("missing `,` char at %s", s)
-		}
-		s = s[1:]
-		s = strings.TrimPrefix(s, " ")
-	}
-	return dst, nil
-}
-
-func getPushRequest() *PushRequest {
-	v := pushReqsPool.Get()
-	if v == nil {
-		return &PushRequest{}
-	}
-	return v.(*PushRequest)
-}
-
-func putPushRequest(req *PushRequest) {
-	req.reset()
-	pushReqsPool.Put(req)
-}

app/vlinsert/loki/loki_protobuf_test.go

@@ -1,168 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/golang/snappy"
-)
-
-type testLogMessageProcessor struct {
-	pr PushRequest
-}
-
-func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
-	msg := ""
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			msg = f.Value
-		}
-	}
-	var a []string
-	for _, f := range fields {
-		if f.Name == "_msg" {
-			continue
-		}
-		item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-		a = append(a, item)
-	}
-	labels := "{" + strings.Join(a, ", ") + "}"
-	tlp.pr.Streams = append(tlp.pr.Streams, Stream{
-		Labels: labels,
-		Entries: []Entry{
-			{
-				Timestamp: time.Unix(0, timestamp),
-				Line:      strings.Clone(msg),
-			},
-		},
-	})
-}
-
-func (tlp *testLogMessageProcessor) MustClose() {
-}
-
-func TestParseProtobufRequest_Success(t *testing.T) {
-	f := func(s string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		n, err := parseJSONRequest([]byte(s), tlp)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if n != len(tlp.pr.Streams) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
-		}
-
-		data := tlp.pr.MarshalProtobuf(nil)
-		encodedData := snappy.Encode(nil, data)
-
-		tlp2 := &insertutils.TestLogMessageProcessor{}
-		n, err = parseProtobufRequest(encodedData, tlp2)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp2.Verify(n, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, nil, ``)
-	f(`{"streams":[{"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, nil, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, nil, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
-{"label1":"value1","label2":"value2","_msg":"abc"}
-{"label1":"value1","label2":"value2","_msg":"foobar"}`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
-{"foo":"bar","a":"b","_msg":"abc"}
-{"x":"y","_msg":"yx"}`)
-}
-
-func TestParsePromLabels_Success(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		var a []string
-		for _, f := range fields {
-			a = append(a, fmt.Sprintf("%s=%q", f.Name, f.Value))
-		}
-		result := "{" + strings.Join(a, ", ") + "}"
-		if result != s {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, s)
-		}
-	}
-
-	f("{}")
-	f(`{foo="bar"}`)
-	f(`{foo="bar", baz="x", y="z"}`)
-	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
-}
-
-func TestParsePromLabels_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(fields) > 0 {
-			t.Fatalf("unexpected non-empty fields: %s", fields)
-		}
-	}
-
-	f("")
-	f("{")
-	f(`{foo}`)
-	f(`{foo=bar}`)
-	f(`{foo="bar}`)
-	f(`{foo="ba\",r}`)
-	f(`{foo="bar" baz="aa"}`)
-	f(`foobar`)
-	f(`foo{bar="baz"}`)
-}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -1,86 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/golang/snappy"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			_, err := parseProtobufRequest(body, blp)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
-	var b []byte
-	var entries []Entry
-	streams := make([]Stream, streamsCount)
-	for i := range streams {
-		b = b[:0]
-		b = append(b, '{')
-		for j := 0; j < labelsCount; j++ {
-			b = append(b, "label_"...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, `="value_`...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, '"')
-			if j < labelsCount-1 {
-				b = append(b, ',')
-			}
-		}
-		b = append(b, '}')
-		labels := string(b)
-
-		var rowsBuf []byte
-		entriesLen := len(entries)
-		for j := 0; j < rowsCount; j++ {
-			rowsBufLen := len(rowsBuf)
-			rowsBuf = append(rowsBuf, "value_"...)
-			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
-			entries = append(entries, Entry{
-				Timestamp: time.Now(),
-				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
-			})
-		}
-
-		st := &streams[i]
-		st.Labels = labels
-		st.Entries = entries[entriesLen:]
-	}
-	pr := PushRequest{
-		Streams: streams,
-	}
-
-	body := pr.MarshalProtobuf(nil)
-	encodedBody := snappy.Encode(nil, body)
-
-	return encodedBody
-}

app/vlinsert/loki/pb.go

@@ -1,302 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: push_request.proto
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package loki
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// PushRequest represents Loki PushRequest
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
-type PushRequest struct {
-	Streams []Stream
-
-	entriesBuf   []Entry
-	labelPairBuf []LabelPair
-}
-
-func (pr *PushRequest) reset() {
-	pr.Streams = pr.Streams[:0]
-
-	pr.entriesBuf = pr.entriesBuf[:0]
-	pr.labelPairBuf = pr.labelPairBuf[:0]
-}
-
-// UnmarshalProtobuf unmarshals pr from protobuf message at src.
-//
-// pr remains valid until src is modified.
-func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
-	pr.reset()
-	var err error
-	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
-	return err
-}
-
-// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
-func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	pr.marshalProtobuf(m.MessageMarshaler())
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, s := range pr.Streams {
-		s.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message PushRequest {
-	//   repeated Stream streams = 1;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
-			}
-			pr.Streams = append(pr.Streams, Stream{})
-			s := &pr.Streams[len(pr.Streams)-1]
-			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
-			}
-		}
-	}
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Stream represents Loki stream.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
-type Stream struct {
-	Labels  string
-	Entries []Entry
-}
-
-func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, s.Labels)
-	for _, e := range s.Entries {
-		e.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message Stream {
-	//   string labels = 1;
-	//   repeated Entry entries = 2;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	entriesBufLen := len(entriesBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			labels, ok := fc.String()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
-			}
-			s.Labels = labels
-		case 2:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
-			}
-			entriesBuf = append(entriesBuf, Entry{})
-			e := &entriesBuf[len(entriesBuf)-1]
-			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
-			}
-		}
-	}
-	s.Entries = entriesBuf[entriesBufLen:]
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Entry represents Loki entry.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
-type Entry struct {
-	Timestamp          time.Time
-	Line               string
-	StructuredMetadata []LabelPair
-}
-
-func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	marshalTime(mm, 1, e.Timestamp)
-	mm.AppendString(2, e.Line)
-	for _, lp := range e.StructuredMetadata {
-		lp.marshalProtobuf(mm.AppendMessage(3))
-	}
-}
-
-func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
-	// message Entry {
-	//   Timestamp timestamp = 1;
-	//   string line = 2;
-	//   repeated LabelPair structuredMetadata = 3;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	labelPairBufLen := len(labelPairBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
-			}
-			timestamp, err := unmarshalTime(data)
-			if err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
-			}
-			e.Timestamp = timestamp
-		case 2:
-			line, ok := fc.String()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Line")
-			}
-			e.Line = line
-		case 3:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
-			}
-			labelPairBuf = append(labelPairBuf, LabelPair{})
-			lp := &labelPairBuf[len(labelPairBuf)-1]
-			if err := lp.unmarshalProtobuf(data); err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
-			}
-		}
-	}
-	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
-	return labelPairBuf, nil
-}
-
-// LabelPair represents Loki label pair.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
-type LabelPair struct {
-	Name  string
-	Value string
-}
-
-func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, lp.Name)
-	mm.AppendString(2, lp.Value)
-}
-
-func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
-	// message LabelPair {
-	//   string name = 1;
-	//   string value = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			name, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot read name")
-			}
-			lp.Name = name
-		case 2:
-			value, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot unmarshal value")
-			}
-			lp.Value = value
-		}
-	}
-	return nil
-}
-
-func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
-	nsecs := timestamp.UnixNano()
-	ts := Timestamp{
-		Seconds: nsecs / 1e9,
-		Nanos:   int32(nsecs % 1e9),
-	}
-	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
-}
-
-func unmarshalTime(src []byte) (time.Time, error) {
-	var ts Timestamp
-	if err := ts.unmarshalProtobuf(src); err != nil {
-		return time.Time{}, err
-	}
-	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	return timestamp, nil
-}
-
-// Timestamp is protobuf well-known timestamp type.
-type Timestamp struct {
-	Seconds int64
-	Nanos   int32
-}
-
-func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendInt64(1, ts.Seconds)
-	mm.AppendInt32(2, ts.Nanos)
-}
-
-func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
-	// message Timestamp {
-	//   int64 seconds = 1;
-	//   int32 nanos = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			seconds, ok := fc.Int64()
-			if !ok {
-				return fmt.Errorf("cannot read Seconds")
-			}
-			ts.Seconds = seconds
-		case 2:
-			nanos, ok := fc.Int32()
-			if !ok {
-				return fmt.Errorf("cannot read Nanos")
-			}
-			ts.Nanos = nanos
-		}
-	}
-	return nil
-}

app/vlinsert/main.go

@@ -1,47 +0,0 @@
-package vlinsert
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
-)
-
-// Init initializes vlinsert
-func Init() {
-	syslog.MustInit()
-}
-
-// Stop stops vlinsert
-func Stop() {
-	syslog.MustStop()
-}
-
-// RequestHandler handles insert requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if !strings.HasPrefix(path, "/insert/") {
-		// Skip requests, which do not start with /insert/, since these aren't our requests.
-		return false
-	}
-	path = strings.TrimPrefix(path, "/insert")
-	path = strings.ReplaceAll(path, "//", "/")
-
-	if path == "/jsonline" {
-		jsonline.RequestHandler(w, r)
-		return true
-	}
-	switch {
-	case strings.HasPrefix(path, "/elasticsearch/"):
-		path = strings.TrimPrefix(path, "/elasticsearch")
-		return elasticsearch.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/loki/"):
-		path = strings.TrimPrefix(path, "/loki")
-		return loki.RequestHandler(path, w, r)
-	default:
-		return false
-	}
-}

app/vlinsert/syslog/syslog.go

@@ -1,531 +0,0 @@
-package syslog
-
-import (
-	"bufio"
-	"crypto/tls"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"net"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/klauspost/compress/gzip"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
-		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-	syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-	listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
-
-	tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+
-		"The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
-		"Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+
-		"The provided key file is automatically re-read every second, so it can be dynamically updated. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+
-		"See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+
-		"See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-	tlsMinVersion = flag.String("syslog.tlsMinVersion", "TLS13", "The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. "+
-		"Supported values: TLS10, TLS11, TLS12, TLS13. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#security")
-
-	compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-	compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+
-		"Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#compression")
-
-	useLocalTimestampTCP = flagutil.NewArrayBool("syslog.useLocalTimestamp.tcp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
-	useLocalTimestampUDP = flagutil.NewArrayBool("syslog.useLocalTimestamp.udp", "Whether to use local timestamp instead of the original timestamp for the ingested syslog messages "+
-		"at the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#log-timestamps")
-)
-
-// MustInit initializes syslog parser at the given -syslog.listenAddr.tcp and -syslog.listenAddr.udp ports
-//
-// This function must be called after flag.Parse().
-//
-// MustStop() must be called in order to free up resources occupied by the initialized syslog parser.
-func MustInit() {
-	if workersStopCh != nil {
-		logger.Panicf("BUG: MustInit() called twice without MustStop() call")
-	}
-	workersStopCh = make(chan struct{})
-
-	for argIdx, addr := range *listenAddrTCP {
-		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runTCPListener(addr, argIdx)
-			workersWG.Done()
-		}(addr, argIdx)
-	}
-
-	for argIdx, addr := range *listenAddrUDP {
-		workersWG.Add(1)
-		go func(addr string, argIdx int) {
-			runUDPListener(addr, argIdx)
-			workersWG.Done()
-		}(addr, argIdx)
-	}
-
-	currentYear := time.Now().Year()
-	globalCurrentYear.Store(int64(currentYear))
-	workersWG.Add(1)
-	go func() {
-		ticker := time.NewTicker(time.Minute)
-		for {
-			select {
-			case <-workersStopCh:
-				ticker.Stop()
-				workersWG.Done()
-				return
-			case <-ticker.C:
-				currentYear := time.Now().Year()
-				globalCurrentYear.Store(int64(currentYear))
-			}
-		}
-	}()
-
-	if *syslogTimezone != "" {
-		tz, err := time.LoadLocation(*syslogTimezone)
-		if err != nil {
-			logger.Fatalf("cannot parse -syslog.timezone=%q: %s", *syslogTimezone, err)
-		}
-		globalTimezone = tz
-	} else {
-		globalTimezone = time.Local
-	}
-}
-
-var (
-	globalCurrentYear atomic.Int64
-	globalTimezone    *time.Location
-)
-
-var (
-	workersWG     sync.WaitGroup
-	workersStopCh chan struct{}
-)
-
-// MustStop stops syslog parser initialized via MustInit()
-func MustStop() {
-	close(workersStopCh)
-	workersWG.Wait()
-	workersStopCh = nil
-}
-
-func runUDPListener(addr string, argIdx int) {
-	ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr)
-	if err != nil {
-		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
-	}
-
-	tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodUDP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "udp")
-
-	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
-
-	doneCh := make(chan struct{})
-	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
-		close(doneCh)
-	}()
-
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
-	<-workersStopCh
-	if err := ln.Close(); err != nil {
-		logger.Fatalf("syslog: cannot close UDP listener at %s: %s", addr, err)
-	}
-	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.udp=%q", addr)
-}
-
-func runTCPListener(addr string, argIdx int) {
-	var tlsConfig *tls.Config
-	if tlsEnable.GetOptionalArg(argIdx) {
-		certFile := tlsCertFile.GetOptionalArg(argIdx)
-		keyFile := tlsKeyFile.GetOptionalArg(argIdx)
-		tc, err := netutil.GetServerTLSConfig(certFile, keyFile, *tlsMinVersion, *tlsCipherSuites)
-		if err != nil {
-			logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s",
-				certFile, keyFile, *tlsMinVersion, *tlsCipherSuites, err)
-		}
-		tlsConfig = tc
-	}
-	ln, err := netutil.NewTCPListener("syslog", addr, false, tlsConfig)
-	if err != nil {
-		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
-	}
-
-	tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
-	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
-	}
-
-	compressMethod := compressMethodTCP.GetOptionalArg(argIdx)
-	checkCompressMethod(compressMethod, addr, "tcp")
-
-	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
-
-	doneCh := make(chan struct{})
-	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
-		close(doneCh)
-	}()
-
-	logger.Infof("started accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
-	<-workersStopCh
-	if err := ln.Close(); err != nil {
-		logger.Fatalf("syslog: cannot close TCP listener at %s: %s", addr, err)
-	}
-	<-doneCh
-	logger.Infof("finished accepting syslog messages at -syslog.listenAddr.tcp=%q", addr)
-}
-
-func checkCompressMethod(compressMethod, addr, protocol string) {
-	switch compressMethod {
-	case "", "none", "gzip", "deflate":
-		return
-	default:
-		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
-	}
-}
-
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
-	gomaxprocs := cgroup.AvailableCPUs()
-	var wg sync.WaitGroup
-	localAddr := ln.LocalAddr()
-	for i := 0; i < gomaxprocs; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			cp := insertutils.GetCommonParamsForSyslog(tenantID)
-			var bb bytesutil.ByteBuffer
-			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
-			for {
-				bb.Reset()
-				bb.B = bb.B[:cap(bb.B)]
-				n, remoteAddr, err := ln.ReadFrom(bb.B)
-				if err != nil {
-					udpErrorsTotal.Inc()
-					var ne net.Error
-					if errors.As(err, &ne) {
-						if ne.Temporary() {
-							logger.Errorf("syslog: temporary error when listening for UDP at %q: %s", localAddr, err)
-							time.Sleep(time.Second)
-							continue
-						}
-						if strings.Contains(err.Error(), "use of closed network connection") {
-							break
-						}
-					}
-					logger.Errorf("syslog: cannot read UDP data from %s at %s: %s", remoteAddr, localAddr, err)
-					continue
-				}
-				bb.B = bb.B[:n]
-				udpRequestsTotal.Inc()
-				if err := processStream(bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
-					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
-				}
-			}
-		}()
-	}
-	wg.Wait()
-}
-
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
-	var cm ingestserver.ConnsMap
-	cm.Init("syslog")
-
-	var wg sync.WaitGroup
-	addr := ln.Addr()
-	for {
-		c, err := ln.Accept()
-		if err != nil {
-			var ne net.Error
-			if errors.As(err, &ne) {
-				if ne.Temporary() {
-					logger.Errorf("syslog: temporary error when listening for TCP addr %q: %s", addr, err)
-					time.Sleep(time.Second)
-					continue
-				}
-				if strings.Contains(err.Error(), "use of closed network connection") {
-					break
-				}
-				logger.Fatalf("syslog: unrecoverable error when accepting TCP connections at %q: %s", addr, err)
-			}
-			logger.Fatalf("syslog: unexpected error when accepting TCP connections at %q: %s", addr, err)
-		}
-		if !cm.Add(c) {
-			_ = c.Close()
-			break
-		}
-
-		wg.Add(1)
-		go func() {
-			cp := insertutils.GetCommonParamsForSyslog(tenantID)
-			if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
-				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
-			}
-
-			cm.Delete(c)
-			_ = c.Close()
-			wg.Done()
-		}()
-	}
-
-	cm.CloseAll(0)
-	wg.Wait()
-}
-
-// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
-	if err := vlstorage.CanWriteData(); err != nil {
-		return err
-	}
-
-	lmp := cp.NewLogMessageProcessor()
-	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
-	lmp.MustClose()
-
-	return err
-}
-
-func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
-	switch compressMethod {
-	case "", "none":
-	case "gzip":
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read gzipped data: %w", err)
-		}
-		r = zr
-	case "deflate":
-		zr, err := common.GetZlibReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read deflated data: %w", err)
-		}
-		r = zr
-	default:
-		logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
-	}
-
-	err := processUncompressedStream(r, useLocalTimestamp, lmp)
-
-	switch compressMethod {
-	case "gzip":
-		zr := r.(*gzip.Reader)
-		common.PutGzipReader(zr)
-	case "deflate":
-		zr := r.(io.ReadCloser)
-		common.PutZlibReader(zr)
-	}
-
-	return err
-}
-
-func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	slr := getSyslogLineReader(wcr)
-	defer putSyslogLineReader(slr)
-
-	n := 0
-	for {
-		ok := slr.nextLine()
-		wcr.DecConcurrency()
-		if !ok {
-			break
-		}
-
-		currentYear := int(globalCurrentYear.Load())
-		err := processLine(slr.line, currentYear, globalTimezone, useLocalTimestamp, lmp)
-		if err != nil {
-			errorsTotal.Inc()
-			return fmt.Errorf("cannot read line #%d: %s", n, err)
-		}
-		n++
-		rowsIngestedTotal.Inc()
-	}
-	return slr.Error()
-}
-
-type syslogLineReader struct {
-	line []byte
-
-	br  *bufio.Reader
-	err error
-}
-
-func (slr *syslogLineReader) reset(r io.Reader) {
-	slr.line = slr.line[:0]
-	slr.br.Reset(r)
-	slr.err = nil
-}
-
-// Error returns the last error occurred in slr.
-func (slr *syslogLineReader) Error() error {
-	if slr.err == nil || slr.err == io.EOF {
-		return nil
-	}
-	return slr.err
-}
-
-// nextLine reads the next syslog line from slr and stores it at slr.line.
-//
-// false is returned if the next line cannot be read. Error() must be called in this case
-// in order to verify whether there is an error or just slr stream has been finished.
-func (slr *syslogLineReader) nextLine() bool {
-	if slr.err != nil {
-		return false
-	}
-
-again:
-	prefix, err := slr.br.ReadSlice(' ')
-	if err != nil {
-		if err != io.EOF {
-			slr.err = fmt.Errorf("cannot read message frame prefix: %w", err)
-			return false
-		}
-		if len(prefix) == 0 {
-			slr.err = err
-			return false
-		}
-	}
-	// skip empty lines
-	for len(prefix) > 0 && prefix[0] == '\n' {
-		prefix = prefix[1:]
-	}
-	if len(prefix) == 0 {
-		// An empty prefix or a prefix with empty lines - try reading yet another prefix.
-		goto again
-	}
-
-	if prefix[0] >= '0' && prefix[0] <= '9' {
-		// This is octet-counting method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#msgxfer
-		msgLenStr := bytesutil.ToUnsafeString(prefix[:len(prefix)-1])
-		msgLen, err := strconv.ParseUint(msgLenStr, 10, 64)
-		if err != nil {
-			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
-			return false
-		}
-		if maxMsgLen := insertutils.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
-			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
-			return false
-		}
-		slr.line = slicesutil.SetLength(slr.line, int(msgLen))
-		if _, err := io.ReadFull(slr.br, slr.line); err != nil {
-			slr.err = fmt.Errorf("cannot read message with size %d bytes: %w", msgLen, err)
-			return false
-		}
-		return true
-	}
-
-	// This is octet-stuffing method. See https://www.ietf.org/archive/id/draft-gerhards-syslog-plain-tcp-07.html#octet-stuffing-legacy
-	slr.line = append(slr.line[:0], prefix...)
-	for {
-		line, err := slr.br.ReadSlice('\n')
-		if err == nil {
-			slr.line = append(slr.line, line[:len(line)-1]...)
-			return true
-		}
-		if err == io.EOF {
-			slr.line = append(slr.line, line...)
-			return true
-		}
-		if err == bufio.ErrBufferFull {
-			slr.line = append(slr.line, line...)
-			continue
-		}
-		slr.err = fmt.Errorf("cannot read message in octet-stuffing method: %w", err)
-		return false
-	}
-}
-
-func getSyslogLineReader(r io.Reader) *syslogLineReader {
-	v := syslogLineReaderPool.Get()
-	if v == nil {
-		br := bufio.NewReaderSize(r, 64*1024)
-		return &syslogLineReader{
-			br: br,
-		}
-	}
-	slr := v.(*syslogLineReader)
-	slr.reset(r)
-	return slr
-}
-
-func putSyslogLineReader(slr *syslogLineReader) {
-	syslogLineReaderPool.Put(slr)
-}
-
-var syslogLineReaderPool sync.Pool
-
-func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
-	p := logstorage.GetSyslogParser(currentYear, timezone)
-	lineStr := bytesutil.ToUnsafeString(line)
-	p.Parse(lineStr)
-
-	var ts int64
-	if useLocalTimestamp {
-		ts = time.Now().UnixNano()
-	} else {
-		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
-		if err != nil {
-			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
-		}
-		ts = nsecs
-	}
-	logstorage.RenameField(p.Fields, "message", "_msg")
-	lmp.AddRow(ts, p.Fields)
-	logstorage.PutSyslogParser(p)
-
-	return nil
-}
-
-var (
-	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
-
-	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
-
-	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
-	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
-)

app/vlinsert/syslog/syslog_test.go

@@ -1,130 +0,0 @@
-package syslog
-
-import (
-	"bytes"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-)
-
-func TestSyslogLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		var lines []string
-		for slr.nextLine() {
-			lines = append(lines, string(slr.line))
-		}
-		if err := slr.Error(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines read;\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", nil)
-	f("\n\n\n", nil)
-
-	f("foobar", []string{"foobar"})
-	f("foobar\n", []string{"foobar\n"})
-	f("\n\nfoo\n\nbar\n\n", []string{"foo\n\nbar\n\n"})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...`, []string{"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches..."})
-
-	f(`Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
-
-`, []string{
-		"Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...",
-		"<165>Jun  4 12:08:33 abcd systemd[345]: abc defg",
-		`<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`,
-	})
-}
-
-func TestSyslogLineReader_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		slr := getSyslogLineReader(r)
-		defer putSyslogLineReader(slr)
-
-		if slr.nextLine() {
-			t.Fatalf("expecting failure to read the first line")
-		}
-		if err := slr.Error(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 aa")
-	f("1233423432 abc")
-}
-
-func TestProcessStreamInternal_Success(t *testing.T) {
-	f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		MustInit()
-		defer MustStop()
-
-		globalTimezone = time.UTC
-		globalCurrentYear.Store(int64(currentYear))
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	data := `Jun  3 12:08:33 abcd systemd: Starting Update the local ESM caches...
-
-48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
-`
-	currentYear := 2023
-	rowsExpected := 3
-	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
-	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
-	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		MustInit()
-		defer MustStop()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal(r, "", false, tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// invalid format for message size
-	f("12foo bar")
-
-	// too big message size
-	f("123 foo")
-	f("123456789 bar")
-}

app/vlogsgenerator/Makefile

@@ -1,7 +0,0 @@
-# All these commands must run from repository root.
-
-vlogsgenerator:
-	APP_NAME=vlogsgenerator $(MAKE) app-local
-
-vlogsgenerator-race:
-	APP_NAME=vlogsgenerator RACE=-race $(MAKE) app-local

app/vlogsgenerator/README.md

@@ -1,158 +0,0 @@
-# vlogsgenerator
-
-Logs generator for [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-## How to build vlogsgenerator?
-
-Run `make vlogsgenerator` from the repository root. This builds `bin/vlogsgenerator` binary.
-
-## How run vlogsgenerator?
-
-`vlogsgenerator` generates logs in [JSON line format](https://jsonlines.org/) suitable for the ingestion
-via [`/insert/jsonline` endpoint at VictoriaLogs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api).
-
-By default it writes the generated logs into `stdout`. For example, the following command writes generated logs to `stdout`:
-
-```
-bin/vlogsgenerator
-```
-
-It is possible to redirect the generated logs to file. For example, the following command writes the generated logs to `logs.json` file:
-
-```
-bin/vlogsgenerator > logs.json
-```
-
-The generated logs at `logs.json` file can be inspected with the following command:
-
-```
-head logs.json | jq .
-```
-
-Below is an example output:
-
-```json
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 8 and worker 0; ip=185.69.136.129; uuid=b4fe8f1a-c93c-dea3-ba11-5b9f0509291e; u64=8996587920687045253",
-  "host": "host_8",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 8",
-  "const_1": "some value 1 8",
-  "const_2": "some value 2 8",
-  "var_0": "some value 0 12752539384823438260",
-  "dict_0": "warn",
-  "dict_1": "info",
-  "u8_0": "6",
-  "u16_0": "35202",
-  "u32_0": "1964973739",
-  "u64_0": "4810489083243239145",
-  "float_0": "1.868",
-  "ip_0": "250.34.75.125",
-  "timestamp_0": "1799-03-16T01:34:18.311Z",
-  "json_0": "{\"foo\":\"bar_3\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":32}"
-}
-{
-  "_time": "2024-05-08T14:34:00.854Z",
-  "_msg": "message for the stream 9 and worker 0; ip=164.244.254.194; uuid=7e8373b1-ce0d-1ce7-8e96-4bcab8955598; u64=13949903463741076522",
-  "host": "host_9",
-  "worker_id": "0",
-  "run_id": "f9b3deee-e6b6-7f56-5deb-1586e4e81725",
-  "const_0": "some value 0 9",
-  "const_1": "some value 1 9",
-  "const_2": "some value 2 9",
-  "var_0": "some value 0 5371555382075206134",
-  "dict_0": "INFO",
-  "dict_1": "FATAL",
-  "u8_0": "219",
-  "u16_0": "31459",
-  "u32_0": "3918836777",
-  "u64_0": "6593354256620219850",
-  "float_0": "1.085",
-  "ip_0": "253.151.88.158",
-  "timestamp_0": "2042-10-05T16:42:57.082Z",
-  "json_0": "{\"foo\":\"bar_5\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":27}"
-}
-```
-
-The `run_id` field uniquely identifies every `vlogsgenerator` invocation.
-
-### How to write logs to VictoriaLogs?
-
-The generated logs can be written directly to VictoriaLogs by passing the address of [`/insert/jsonline` endpoint](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api)
-to `-addr` command-line flag. For example, the following command writes the generated logs to VictoriaLogs running at `localhost`:
-
-```
-bin/vlogsgenerator -addr=http://localhost:9428/insert/jsonline
-```
-
-### Configuration
-
-`vlogsgenerator` accepts various command-line flags, which can be used for configuring the number and the shape of the generated logs.
-These flags can be inspected by running `vlogsgenerator -help`. Below are the most interesting flags:
-
-* `-start` - starting timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-end` - ending timestamp for generating logs. Logs are evenly generated on the [`-start` ... `-end`] interval.
-* `-activeStreams` - the number of active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to generate.
-* `-logsPerStream` - the number of log entries to generate per each log stream. Log entries are evenly distributed on the [`-start` ... `-end`] interval.
-
-The total number of generated logs can be calculated as `-activeStreams` * `-logsPerStream`.
-
-For example, the following command generates `1_000_000` log entries on the time range `[2024-01-01 - 2024-02-01]` across `100`
-[log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields), where every logs stream contains `10_000` log entries,
-and writes them to `http://localhost:9428/insert/jsonline`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-### Churn rate
-
-It is possible to generate churn rate for active [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
-by specifying `-totalStreams` command-line flag bigger than `-activeStreams`. For example, the following command generates
-logs for `1000` total streams, while the number of active streams equals to `100`. This means that at every time there are logs for `100` streams,
-but these streams change over the given [`-start` ... `-end`] time range, so the total number of streams on the given time range becomes `1000`:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -totalStreams=1_000 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline
-```
-
-In this case the total number of generated logs equals to `-totalStreams` * `-logsPerStream` = `10_000_000`.
-
-### Benchmark tuning
-
-By default `vlogsgenerator` generates and writes logs by a single worker. This may limit the maximum data ingestion rate during benchmarks.
-The number of workers can be changed via `-workers` command-line flag. For example, the following command generates and writes logs with `16` workers:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -workers=16
-```
-
-### Output statistics
-
-Every 10 seconds `vlogsgenerator` writes statistics about the generated logs into `stderr`. The frequency of the generated statistics can be adjusted via `-statInterval` command-line flag.
-For example, the following command writes statistics every 2 seconds:
-
-```
-bin/vlogsgenerator \
-  -start=2024-01-01 -end=2024-02-01 \
-  -activeStreams=100 \
-  -logsPerStream=10_000 \
-  -addr=http://localhost:9428/insert/jsonline \
-  -statInterval=2s
-```

app/vlogsgenerator/main.go

@@ -1,344 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"math"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"os"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-var (
-	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
-	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
-
-	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
-	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
-	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
-	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
-		"during data generation")
-	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
-	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	dictFieldsPerLog = flag.Int("dictFieldsPerLog", 2, "The number of fields with up to 8 different values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u8FieldsPerLog = flag.Int("u8FieldsPerLog", 1, "The number of fields with uint8 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u16FieldsPerLog = flag.Int("u16FieldsPerLog", 1, "The number of fields with uint16 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u32FieldsPerLog = flag.Int("u32FieldsPerLog", 1, "The number of fields with uint32 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	timestampFieldsPerLog = flag.Int("timestampFieldsPerLog", 1, "The number of fields with ISO8601 timestamps per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	jsonFieldsPerLog = flag.Int("jsonFieldsPerLog", 1, "The number of JSON fields to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-
-	statInterval = flag.Duration("statInterval", 10*time.Second, "The interval between publishing the stats")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	var remoteWriteURL *url.URL
-	if *addr != "stdout" {
-		urlParsed, err := url.Parse(*addr)
-		if err != nil {
-			logger.Fatalf("cannot parse -addr=%q: %s", *addr, err)
-		}
-		qs, err := url.ParseQuery(urlParsed.RawQuery)
-		if err != nil {
-			logger.Fatalf("cannot parse query string in -addr=%q: %w", *addr, err)
-		}
-		qs.Set("_stream_fields", "host,worker_id")
-		urlParsed.RawQuery = qs.Encode()
-		remoteWriteURL = urlParsed
-	}
-
-	if start.nsec >= end.nsec {
-		logger.Fatalf("-start=%s must be smaller than -end=%s", start, end)
-	}
-	if *activeStreams <= 0 {
-		logger.Fatalf("-activeStreams must be bigger than 0; got %d", *activeStreams)
-	}
-	if *logsPerStream <= 0 {
-		logger.Fatalf("-logsPerStream must be bigger than 0; got %d", *logsPerStream)
-	}
-	if *totalStreams < *activeStreams {
-		*totalStreams = *activeStreams
-	}
-
-	cfg := &workerConfig{
-		url:           remoteWriteURL,
-		activeStreams: *activeStreams,
-		totalStreams:  *totalStreams,
-	}
-
-	// divide total and active streams among workers
-	if *workers <= 0 {
-		logger.Fatalf("-workers must be bigger than 0; got %d", *workers)
-	}
-	if *workers > *activeStreams {
-		logger.Fatalf("-workers=%d cannot exceed -activeStreams=%d", *workers, *activeStreams)
-	}
-	cfg.activeStreams /= *workers
-	cfg.totalStreams /= *workers
-
-	logger.Infof("start -workers=%d workers for ingesting -logsPerStream=%d log entries per each -totalStreams=%d (-activeStreams=%d) on a time range -start=%s, -end=%s to -addr=%s",
-		*workers, *logsPerStream, *totalStreams, *activeStreams, toRFC3339(start.nsec), toRFC3339(end.nsec), *addr)
-
-	startTime := time.Now()
-	var wg sync.WaitGroup
-	for i := 0; i < *workers; i++ {
-		wg.Add(1)
-		go func(workerID int) {
-			defer wg.Done()
-			generateAndPushLogs(cfg, workerID)
-		}(i)
-	}
-
-	go func() {
-		prevEntries := uint64(0)
-		prevBytes := uint64(0)
-		ticker := time.NewTicker(*statInterval)
-		for range ticker.C {
-			currEntries := logEntriesCount.Load()
-			deltaEntries := currEntries - prevEntries
-			rateEntries := float64(deltaEntries) / statInterval.Seconds()
-
-			currBytes := bytesGenerated.Load()
-			deltaBytes := currBytes - prevBytes
-			rateBytes := float64(deltaBytes) / statInterval.Seconds()
-			logger.Infof("generated %dK log entries (%dK total) at %.0fK entries/sec, %dMB (%dMB total) at %.0fMB/sec",
-				deltaEntries/1e3, currEntries/1e3, rateEntries/1e3, deltaBytes/1e6, currBytes/1e6, rateBytes/1e6)
-
-			prevEntries = currEntries
-			prevBytes = currBytes
-		}
-	}()
-
-	wg.Wait()
-
-	dSecs := time.Since(startTime).Seconds()
-	currEntries := logEntriesCount.Load()
-	currBytes := bytesGenerated.Load()
-	rateEntries := float64(currEntries) / dSecs
-	rateBytes := float64(currBytes) / dSecs
-	logger.Infof("ingested %dK log entries (%dMB) in %.3f seconds; avg ingestion rate: %.0fK entries/sec, %.0fMB/sec", currEntries/1e3, currBytes/1e6, dSecs, rateEntries/1e3, rateBytes/1e6)
-}
-
-var logEntriesCount atomic.Uint64
-
-var bytesGenerated atomic.Uint64
-
-type workerConfig struct {
-	url           *url.URL
-	activeStreams int
-	totalStreams  int
-}
-
-type statWriter struct {
-	w io.Writer
-}
-
-func (sw *statWriter) Write(p []byte) (int, error) {
-	bytesGenerated.Add(uint64(len(p)))
-	return sw.w.Write(p)
-}
-
-func generateAndPushLogs(cfg *workerConfig, workerID int) {
-	pr, pw := io.Pipe()
-	sw := &statWriter{
-		w: pw,
-	}
-	bw := bufio.NewWriter(sw)
-	doneCh := make(chan struct{})
-	go func() {
-		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
-		_ = bw.Flush()
-		_ = pw.Close()
-		close(doneCh)
-	}()
-
-	if cfg.url == nil {
-		_, err := io.Copy(os.Stdout, pr)
-		if err != nil {
-			logger.Fatalf("unexpected error when writing logs to stdout: %s", err)
-		}
-		return
-	}
-
-	req, err := http.NewRequest("POST", cfg.url.String(), pr)
-	if err != nil {
-		logger.Fatalf("cannot create request to %q: %s", cfg.url, err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
-	}
-	if resp.StatusCode/100 != 2 {
-		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
-	}
-
-	// Wait until all the generateLogs goroutine is finished.
-	<-doneCh
-}
-
-func generateLogs(bw *bufio.Writer, workerID, activeStreams, totalStreams int) {
-	streamLifetime := int64(float64(end.nsec-start.nsec) * (float64(activeStreams) / float64(totalStreams)))
-	streamStep := int64(float64(end.nsec-start.nsec) / float64(totalStreams-activeStreams+1))
-	step := streamLifetime / (*logsPerStream - 1)
-
-	currNsec := start.nsec
-	for currNsec < end.nsec {
-		firstStreamID := int((currNsec - start.nsec) / streamStep)
-		generateLogsAtTimestamp(bw, workerID, currNsec, firstStreamID, activeStreams)
-		currNsec += step
-	}
-}
-
-var runID = toUUID(rand.Uint64(), rand.Uint64())
-
-func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStreamID, activeStreams int) {
-	streamID := firstStreamID
-	timeStr := toRFC3339(ts)
-	for i := 0; i < activeStreams; i++ {
-		ip := toIPv4(rand.Uint32())
-		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
-			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
-		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
-		for j := 0; j < *constFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"const_%d":"some value %d %d"`, j, j, streamID)
-		}
-		for j := 0; j < *varFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"var_%d":"some value %d %d"`, j, j, rand.Uint64())
-		}
-		for j := 0; j < *dictFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"dict_%d":"%s"`, j, dictValues[rand.Intn(len(dictValues))])
-		}
-		for j := 0; j < *u8FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u8_%d":"%d"`, j, uint8(rand.Uint32()))
-		}
-		for j := 0; j < *u16FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u16_%d":"%d"`, j, uint16(rand.Uint32()))
-		}
-		for j := 0; j < *u32FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u32_%d":"%d"`, j, rand.Uint32())
-		}
-		for j := 0; j < *u64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
-		}
-		for j := 0; j < *floatFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
-		}
-		for j := 0; j < *ipFieldsPerLog; j++ {
-			ip := toIPv4(rand.Uint32())
-			fmt.Fprintf(bw, `,"ip_%d":"%s"`, j, ip)
-		}
-		for j := 0; j < *timestampFieldsPerLog; j++ {
-			timestamp := toISO8601(int64(rand.Uint64()))
-			fmt.Fprintf(bw, `,"timestamp_%d":"%s"`, j, timestamp)
-		}
-		for j := 0; j < *jsonFieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"json_%d":"{\"foo\":\"bar_%d\",\"baz\":{\"a\":[\"x\",\"y\"]},\"f3\":NaN,\"f4\":%d}"`, j, rand.Intn(10), rand.Intn(100))
-		}
-		fmt.Fprintf(bw, "}\n")
-
-		logEntriesCount.Add(1)
-		streamID++
-	}
-}
-
-var dictValues = []string{
-	"debug",
-	"info",
-	"warn",
-	"error",
-	"fatal",
-	"ERROR",
-	"FATAL",
-	"INFO",
-}
-
-func newTimeFlag(name, defaultValue, description string) *timeFlag {
-	var tf timeFlag
-	if err := tf.Set(defaultValue); err != nil {
-		logger.Panicf("invalid defaultValue=%q for flag %q: %w", defaultValue, name, err)
-	}
-	flag.Var(&tf, name, description)
-	return &tf
-}
-
-type timeFlag struct {
-	s    string
-	nsec int64
-}
-
-func (tf *timeFlag) Set(s string) error {
-	msec, err := promutils.ParseTimeMsec(s)
-	if err != nil {
-		return fmt.Errorf("cannot parse time from %q: %w", s, err)
-	}
-	tf.s = s
-	tf.nsec = msec * 1e6
-	return nil
-}
-
-func (tf *timeFlag) String() string {
-	return tf.s
-}
-
-func toRFC3339(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format(time.RFC3339Nano)
-}
-
-func toISO8601(nsec int64) string {
-	return time.Unix(0, nsec).UTC().Format("2006-01-02T15:04:05.000Z")
-}
-
-func toIPv4(n uint32) string {
-	dst := make([]byte, 0, len("255.255.255.255"))
-	dst = marshalUint64(dst, uint64(n>>24))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>16)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64((n>>8)&0xff))
-	dst = append(dst, '.')
-	dst = marshalUint64(dst, uint64(n&0xff))
-	return string(dst)
-}
-
-func toUUID(a, b uint64) string {
-	return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", a&(1<<32-1), (a>>32)&(1<<16-1), (a >> 48), b&(1<<16-1), b>>16)
-}
-
-// marshalUint64 appends string representation of n to dst and returns the result.
-func marshalUint64(dst []byte, n uint64) []byte {
-	return strconv.AppendUint(dst, n, 10)
-}

app/vlselect/logsql/buffered_writer.go

@@ -1,47 +0,0 @@
-package logsql
-
-import (
-	"bufio"
-	"io"
-	"sync"
-)
-
-func getBufferedWriter(w io.Writer) *bufferedWriter {
-	v := bufferedWriterPool.Get()
-	if v == nil {
-		return &bufferedWriter{
-			bw: bufio.NewWriter(w),
-		}
-	}
-	bw := v.(*bufferedWriter)
-	bw.bw.Reset(w)
-	return bw
-}
-
-func putBufferedWriter(bw *bufferedWriter) {
-	bw.reset()
-	bufferedWriterPool.Put(bw)
-}
-
-var bufferedWriterPool sync.Pool
-
-type bufferedWriter struct {
-	mu sync.Mutex
-	bw *bufio.Writer
-}
-
-func (bw *bufferedWriter) reset() {
-	// nothing to do
-}
-
-func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
-	bw.mu.Lock()
-	_, _ = bw.bw.Write(p)
-	bw.mu.Unlock()
-}
-
-func (bw *bufferedWriter) FlushIgnoreErrors() {
-	bw.mu.Lock()
-	_ = bw.bw.Flush()
-	bw.mu.Unlock()
-}

app/vlselect/logsql/hits_response.qtpl

@@ -1,70 +0,0 @@
-{% import (
-	"slices"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// FieldsForHits formats labels for /select/logsql/hits response
-{% func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) %}
-{
-	{% if len(columns) > 0 %}
-		{%q= columns[0].Name %}:{%q= columns[0].Values[rowIdx] %}
-		{% for _, c := range columns[1:] %}
-			,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-		{% endfor %}
-	{% endif %}
-}
-{% endfunc %}
-
-{% func HitsSeries(m map[string]*hitsSeries) %}
-{
-	{% code
-		sortedKeys := make([]string, 0, len(m))
-		for k := range m {
-			sortedKeys = append(sortedKeys, k)
-		}
-		slices.Sort(sortedKeys)
-	%}
-	"hits":[
-		{% if len(sortedKeys) > 0 %}
-			{%= hitsSeriesLine(m, sortedKeys[0]) %}
-			{% for _, k := range sortedKeys[1:] %}
-				,{%= hitsSeriesLine(m, k) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func hitsSeriesLine(m map[string]*hitsSeries, k string) %}
-{
-	{% code
-		hs := m[k]
-		hs.sort()
-		timestamps := hs.timestamps
-		hits := hs.hits
-	%}
-	"fields":{%s= k %},
-	"timestamps":[
-		{% if len(timestamps) > 0 %}
-			{%q= timestamps[0] %}
-			{% for _, ts := range timestamps[1:] %}
-				,{%q= ts %}
-			{% endfor %}
-		{% endif %}
-	],
-	"values":[
-		{% if len(hits) > 0 %}
-			{%dul= hits[0] %}
-			{% for _, v := range hits[1:] %}
-				,{%dul= v %}
-			{% endfor %}
-		{% endif %}
-	],
-	"total":{%dul= hs.hitsTotal %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/hits_response.qtpl.go

@@ -1,223 +0,0 @@
-// Code generated by qtc from "hits_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/hits_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/hits_response.qtpl:1
-import (
-	"slices"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// FieldsForHits formats labels for /select/logsql/hits response
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/hits_response.qtpl:10
-func StreamFieldsForHits(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/hits_response.qtpl:10
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:12
-	if len(columns) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().Q(columns[0].Name)
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/hits_response.qtpl:13
-		qw422016.N().Q(columns[0].Values[rowIdx])
-//line app/vlselect/logsql/hits_response.qtpl:14
-		for _, c := range columns[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/hits_response.qtpl:15
-			qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/hits_response.qtpl:16
-		}
-//line app/vlselect/logsql/hits_response.qtpl:17
-	}
-//line app/vlselect/logsql/hits_response.qtpl:17
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:19
-func WriteFieldsForHits(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	StreamFieldsForHits(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:19
-func FieldsForHits(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:19
-	WriteFieldsForHits(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:19
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:19
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:21
-func StreamHitsSeries(qw422016 *qt422016.Writer, m map[string]*hitsSeries) {
-//line app/vlselect/logsql/hits_response.qtpl:21
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:24
-	sortedKeys := make([]string, 0, len(m))
-	for k := range m {
-		sortedKeys = append(sortedKeys, k)
-	}
-	slices.Sort(sortedKeys)
-
-//line app/vlselect/logsql/hits_response.qtpl:29
-	qw422016.N().S(`"hits":[`)
-//line app/vlselect/logsql/hits_response.qtpl:31
-	if len(sortedKeys) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:32
-		streamhitsSeriesLine(qw422016, m, sortedKeys[0])
-//line app/vlselect/logsql/hits_response.qtpl:33
-		for _, k := range sortedKeys[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:33
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:34
-			streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:35
-		}
-//line app/vlselect/logsql/hits_response.qtpl:36
-	}
-//line app/vlselect/logsql/hits_response.qtpl:36
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:39
-func WriteHitsSeries(qq422016 qtio422016.Writer, m map[string]*hitsSeries) {
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	StreamHitsSeries(qw422016, m)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:39
-func HitsSeries(m map[string]*hitsSeries) string {
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:39
-	WriteHitsSeries(qb422016, m)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:39
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:39
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:41
-func streamhitsSeriesLine(qw422016 *qt422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:41
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/hits_response.qtpl:44
-	hs := m[k]
-	hs.sort()
-	timestamps := hs.timestamps
-	hits := hs.hits
-
-//line app/vlselect/logsql/hits_response.qtpl:48
-	qw422016.N().S(`"fields":`)
-//line app/vlselect/logsql/hits_response.qtpl:49
-	qw422016.N().S(k)
-//line app/vlselect/logsql/hits_response.qtpl:49
-	qw422016.N().S(`,"timestamps":[`)
-//line app/vlselect/logsql/hits_response.qtpl:51
-	if len(timestamps) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:52
-		qw422016.N().Q(timestamps[0])
-//line app/vlselect/logsql/hits_response.qtpl:53
-		for _, ts := range timestamps[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:53
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:54
-			qw422016.N().Q(ts)
-//line app/vlselect/logsql/hits_response.qtpl:55
-		}
-//line app/vlselect/logsql/hits_response.qtpl:56
-	}
-//line app/vlselect/logsql/hits_response.qtpl:56
-	qw422016.N().S(`],"values":[`)
-//line app/vlselect/logsql/hits_response.qtpl:59
-	if len(hits) > 0 {
-//line app/vlselect/logsql/hits_response.qtpl:60
-		qw422016.N().DUL(hits[0])
-//line app/vlselect/logsql/hits_response.qtpl:61
-		for _, v := range hits[1:] {
-//line app/vlselect/logsql/hits_response.qtpl:61
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/hits_response.qtpl:62
-			qw422016.N().DUL(v)
-//line app/vlselect/logsql/hits_response.qtpl:63
-		}
-//line app/vlselect/logsql/hits_response.qtpl:64
-	}
-//line app/vlselect/logsql/hits_response.qtpl:64
-	qw422016.N().S(`],"total":`)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().DUL(hs.hitsTotal)
-//line app/vlselect/logsql/hits_response.qtpl:66
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/hits_response.qtpl:68
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:68
-func writehitsSeriesLine(qq422016 qtio422016.Writer, m map[string]*hitsSeries, k string) {
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	streamhitsSeriesLine(qw422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-}
-
-//line app/vlselect/logsql/hits_response.qtpl:68
-func hitsSeriesLine(m map[string]*hitsSeries, k string) string {
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/hits_response.qtpl:68
-	writehitsSeriesLine(qb422016, m, k)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/hits_response.qtpl:68
-	return qs422016
-//line app/vlselect/logsql/hits_response.qtpl:68
-}

app/vlselect/logsql/logsql.go

@@ -1,771 +0,0 @@
-package logsql
-
-import (
-	"context"
-	"fmt"
-	"math"
-	"net/http"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-// ProcessHitsRequest handles /select/logsql/hits request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-hits-stats
-func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Obtain step
-	stepStr := r.FormValue("step")
-	if stepStr == "" {
-		stepStr = "1d"
-	}
-	step, err := promutils.ParseDuration(stepStr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse 'step' arg: %s", err)
-		return
-	}
-	if step <= 0 {
-		httpserver.Errorf(w, r, "'step' must be bigger than zero")
-	}
-
-	// Obtain offset
-	offsetStr := r.FormValue("offset")
-	if offsetStr == "" {
-		offsetStr = "0s"
-	}
-	offset, err := promutils.ParseDuration(offsetStr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse 'offset' arg: %s", err)
-		return
-	}
-
-	// Obtain field entries
-	fields := r.Form["field"]
-
-	// Obtain limit on the number of top fields entries.
-	fieldsLimit, err := httputils.GetInt(r, "fields_limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if fieldsLimit < 0 {
-		fieldsLimit = 0
-	}
-
-	// Prepare the query for hits count.
-	q.Optimize()
-	q.DropAllPipes()
-	q.AddCountByTimePipe(int64(step), int64(offset), fields)
-
-	var mLock sync.Mutex
-	m := make(map[string]*hitsSeries)
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
-			return
-		}
-
-		timestampValues := columns[0].Values
-		hitsValues := columns[len(columns)-1].Values
-		columns = columns[1 : len(columns)-1]
-
-		bb := blockResultPool.Get()
-		for i := range timestamps {
-			timestampStr := strings.Clone(timestampValues[i])
-			hitsStr := strings.Clone(hitsValues[i])
-			hits, err := strconv.ParseUint(hitsStr, 10, 64)
-			if err != nil {
-				logger.Panicf("BUG: cannot parse hitsStr=%q: %s", hitsStr, err)
-			}
-
-			bb.Reset()
-			WriteFieldsForHits(bb, columns, i)
-
-			mLock.Lock()
-			hs, ok := m[string(bb.B)]
-			if !ok {
-				k := string(bb.B)
-				hs = &hitsSeries{}
-				m[k] = hs
-			}
-			hs.timestamps = append(hs.timestamps, timestampStr)
-			hs.hits = append(hs.hits, hits)
-			hs.hitsTotal += hits
-			mLock.Unlock()
-		}
-		blockResultPool.Put(bb)
-	}
-
-	// Execute the query
-	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
-		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
-		return
-	}
-
-	m = getTopHitsSeries(m, fieldsLimit)
-
-	// Write response
-	w.Header().Set("Content-Type", "application/json")
-	WriteHitsSeries(w, m)
-}
-
-func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
-	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
-		return m
-	}
-
-	type fieldsHits struct {
-		fieldsStr string
-		hs        *hitsSeries
-	}
-	a := make([]fieldsHits, 0, len(m))
-	for fieldsStr, hs := range m {
-		a = append(a, fieldsHits{
-			fieldsStr: fieldsStr,
-			hs:        hs,
-		})
-	}
-	sort.Slice(a, func(i, j int) bool {
-		return a[i].hs.hitsTotal > a[j].hs.hitsTotal
-	})
-
-	hitsOther := make(map[string]uint64)
-	for _, x := range a[fieldsLimit:] {
-		for i, timestampStr := range x.hs.timestamps {
-			hitsOther[timestampStr] += x.hs.hits[i]
-		}
-	}
-	var hsOther hitsSeries
-	for timestampStr, hits := range hitsOther {
-		hsOther.timestamps = append(hsOther.timestamps, timestampStr)
-		hsOther.hits = append(hsOther.hits, hits)
-		hsOther.hitsTotal += hits
-	}
-
-	mNew := make(map[string]*hitsSeries, fieldsLimit+1)
-	for _, x := range a[:fieldsLimit] {
-		mNew[x.fieldsStr] = x.hs
-	}
-	mNew["{}"] = &hsOther
-
-	return mNew
-}
-
-type hitsSeries struct {
-	hitsTotal  uint64
-	timestamps []string
-	hits       []uint64
-}
-
-func (hs *hitsSeries) sort() {
-	sort.Sort(hs)
-}
-
-func (hs *hitsSeries) Len() int {
-	return len(hs.timestamps)
-}
-
-func (hs *hitsSeries) Swap(i, j int) {
-	hs.timestamps[i], hs.timestamps[j] = hs.timestamps[j], hs.timestamps[i]
-	hs.hits[i], hs.hits[j] = hs.hits[j], hs.hits[i]
-}
-
-func (hs *hitsSeries) Less(i, j int) bool {
-	return hs.timestamps[i] < hs.timestamps[j]
-}
-
-// ProcessFieldNamesRequest handles /select/logsql/field_names request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-field-names
-func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Obtain field names for the given query
-	q.Optimize()
-	fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain field names: %s", err)
-		return
-	}
-
-	// Write results
-	w.Header().Set("Content-Type", "application/json")
-	WriteValuesWithHitsJSON(w, fieldNames)
-}
-
-// ProcessFieldValuesRequest handles /select/logsql/field_values request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-field-values
-func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Parse fieldName query arg
-	fieldName := r.FormValue("field")
-	if fieldName == "" {
-		httpserver.Errorf(w, r, "missing 'field' query arg")
-		return
-	}
-
-	// Parse limit query arg
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if limit < 0 {
-		limit = 0
-	}
-
-	// Obtain unique values for the given field
-	q.Optimize()
-	values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain values for field %q: %s", fieldName, err)
-		return
-	}
-
-	// Write results
-	w.Header().Set("Content-Type", "application/json")
-	WriteValuesWithHitsJSON(w, values)
-}
-
-// ProcessStreamFieldNamesRequest processes /select/logsql/stream_field_names request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-names
-func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Obtain stream field names for the given query
-	q.Optimize()
-	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain stream field names: %s", err)
-	}
-
-	// Write results
-	w.Header().Set("Content-Type", "application/json")
-	WriteValuesWithHitsJSON(w, names)
-}
-
-// ProcessStreamFieldValuesRequest processes /select/logsql/stream_field_values request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream-field-values
-func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Parse fieldName query arg
-	fieldName := r.FormValue("field")
-	if fieldName == "" {
-		httpserver.Errorf(w, r, "missing 'field' query arg")
-		return
-	}
-
-	// Parse limit query arg
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if limit < 0 {
-		limit = 0
-	}
-
-	// Obtain stream field values for the given query and the given fieldName
-	q.Optimize()
-	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain stream field values: %s", err)
-	}
-
-	// Write results
-	w.Header().Set("Content-Type", "application/json")
-	WriteValuesWithHitsJSON(w, values)
-}
-
-// ProcessStreamIDsRequest processes /select/logsql/stream_ids request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-stream_ids
-func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Parse limit query arg
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if limit < 0 {
-		limit = 0
-	}
-
-	// Obtain streamIDs for the given query
-	q.Optimize()
-	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain stream_ids: %s", err)
-	}
-
-	// Write results
-	w.Header().Set("Content-Type", "application/json")
-	WriteValuesWithHitsJSON(w, streamIDs)
-}
-
-// ProcessStreamsRequest processes /select/logsql/streams request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-streams
-func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Parse limit query arg
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if limit < 0 {
-		limit = 0
-	}
-
-	// Obtain streams for the given query
-	q.Optimize()
-	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot obtain streams: %s", err)
-	}
-
-	// Write results
-	w.Header().Set("Content-Type", "application/json")
-	WriteValuesWithHitsJSON(w, streams)
-}
-
-// ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
-func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	liveTailRequests.Inc()
-	defer liveTailRequests.Dec()
-
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if !q.CanLiveTail() {
-		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
-	}
-	q.Optimize()
-
-	refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
-
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	tp := newTailProcessor(cancel)
-
-	ticker := time.NewTicker(refreshInterval)
-	defer ticker.Stop()
-
-	end := time.Now().UnixNano()
-	doneCh := ctxWithCancel.Done()
-	flusher, ok := w.(http.Flusher)
-	if !ok {
-		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
-	}
-	for {
-		start := end - tailOffsetNsecs
-		end = time.Now().UnixNano()
-
-		qCopy := q.Clone()
-		qCopy.AddTimeFilter(start, end)
-		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
-			httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
-			return
-		}
-		resultRows, err := tp.getTailRows()
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot get tail results for query [%q]: %s", q, err)
-			return
-		}
-		if len(resultRows) > 0 {
-			WriteJSONRows(w, resultRows)
-			flusher.Flush()
-		}
-
-		select {
-		case <-doneCh:
-			return
-		case <-ticker.C:
-		}
-	}
-}
-
-var liveTailRequests = metrics.NewCounter(`vl_live_tailing_requests`)
-
-const tailOffsetNsecs = 5e9
-
-type logRow struct {
-	timestamp int64
-	fields    []logstorage.Field
-}
-
-func sortLogRows(rows []logRow) {
-	sort.SliceStable(rows, func(i, j int) bool {
-		return rows[i].timestamp < rows[j].timestamp
-	})
-}
-
-type tailProcessor struct {
-	cancel func()
-
-	mu sync.Mutex
-
-	perStreamRows  map[string][]logRow
-	lastTimestamps map[string]int64
-
-	err error
-}
-
-func newTailProcessor(cancel func()) *tailProcessor {
-	return &tailProcessor{
-		cancel: cancel,
-
-		perStreamRows:  make(map[string][]logRow),
-		lastTimestamps: make(map[string]int64),
-	}
-}
-
-func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-	if len(timestamps) == 0 {
-		return
-	}
-
-	tp.mu.Lock()
-	defer tp.mu.Unlock()
-
-	if tp.err != nil {
-		return
-	}
-
-	// Make sure columns contain _time field, since it is needed for proper tail work.
-	hasTime := false
-	for _, c := range columns {
-		if c.Name == "_time" {
-			hasTime = true
-			break
-		}
-	}
-	if !hasTime {
-		tp.err = fmt.Errorf("missing _time field")
-		tp.cancel()
-		return
-	}
-
-	// Copy block rows to tp.perStreamRows
-	for i, timestamp := range timestamps {
-		streamID := ""
-		fields := make([]logstorage.Field, len(columns))
-		for j, c := range columns {
-			name := strings.Clone(c.Name)
-			value := strings.Clone(c.Values[i])
-
-			fields[j] = logstorage.Field{
-				Name:  name,
-				Value: value,
-			}
-
-			if name == "_stream_id" {
-				streamID = value
-			}
-		}
-
-		tp.perStreamRows[streamID] = append(tp.perStreamRows[streamID], logRow{
-			timestamp: timestamp,
-			fields:    fields,
-		})
-	}
-}
-
-func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
-	if tp.err != nil {
-		return nil, tp.err
-	}
-
-	var resultRows []logRow
-	for streamID, rows := range tp.perStreamRows {
-		sortLogRows(rows)
-
-		lastTimestamp, ok := tp.lastTimestamps[streamID]
-		if ok {
-			// Skip already written rows
-			for len(rows) > 0 && rows[0].timestamp <= lastTimestamp {
-				rows = rows[1:]
-			}
-		}
-		if len(rows) > 0 {
-			resultRows = append(resultRows, rows...)
-			tp.lastTimestamps[streamID] = rows[len(rows)-1].timestamp
-		}
-	}
-	clear(tp.perStreamRows)
-
-	sortLogRows(resultRows)
-
-	tailRows := make([][]logstorage.Field, len(resultRows))
-	for i, row := range resultRows {
-		tailRows[i] = row.fields
-	}
-
-	return tailRows, nil
-}
-
-// ProcessQueryRequest handles /select/logsql/query request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
-func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	// Parse limit query arg
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	bw := getBufferedWriter(w)
-	defer func() {
-		bw.FlushIgnoreErrors()
-		putBufferedWriter(bw)
-	}()
-	w.Header().Set("Content-Type", "application/stream+json")
-
-	if limit > 0 {
-		if q.CanReturnLastNResults() {
-			rows, err := getLastNQueryResults(ctx, tenantIDs, q, limit)
-			if err != nil {
-				httpserver.Errorf(w, r, "%s", err)
-				return
-			}
-			bb := blockResultPool.Get()
-			b := bb.B
-			for i := range rows {
-				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
-				b = append(b, '\n')
-				bw.WriteIgnoreErrors(b)
-			}
-			bb.B = b
-			blockResultPool.Put(bb)
-			return
-		}
-
-		q.AddPipeLimit(uint64(limit))
-	}
-	q.Optimize()
-
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
-			return
-		}
-
-		bb := blockResultPool.Get()
-		for i := range timestamps {
-			WriteJSONRow(bb, columns, i)
-		}
-		bw.WriteIgnoreErrors(bb.B)
-		blockResultPool.Put(bb)
-	}
-
-	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
-		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
-		return
-	}
-}
-
-var blockResultPool bytesutil.ByteBufferPool
-
-type row struct {
-	timestamp int64
-	fields    []logstorage.Field
-}
-
-func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
-	limitUpper := 2 * limit
-	q.AddPipeLimit(uint64(limitUpper))
-	q.Optimize()
-	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
-	if err != nil {
-		return nil, err
-	}
-	if len(rows) < limitUpper {
-		// Fast path - the requested time range contains up to limitUpper rows.
-		rows = getLastNRows(rows, limit)
-		return rows, nil
-	}
-
-	// Slow path - search for the time range containing up to limitUpper rows.
-	start, end := q.GetFilterTimeRange()
-	d := end/2 - start/2
-	start += d
-
-	qOrig := q
-	for {
-		q = qOrig.Clone()
-		q.AddTimeFilter(start, end)
-		rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
-			rows = getLastNRows(rows, limit)
-			return rows, nil
-		}
-
-		lastBit := d & 1
-		d /= 2
-		if len(rows) > limit {
-			start += d
-		} else {
-			start -= d + lastBit
-		}
-	}
-}
-
-func getLastNRows(rows []row, limit int) []row {
-	sort.Slice(rows, func(i, j int) bool {
-		return rows[i].timestamp < rows[j].timestamp
-	})
-	if len(rows) > limit {
-		rows = rows[len(rows)-limit:]
-	}
-	return rows
-}
-
-func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	var rows []row
-	var rowsLock sync.Mutex
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		rowsLock.Lock()
-		defer rowsLock.Unlock()
-
-		for i, timestamp := range timestamps {
-			fields := make([]logstorage.Field, len(columns))
-			for j := range columns {
-				f := &fields[j]
-				f.Name = strings.Clone(columns[j].Name)
-				f.Value = strings.Clone(columns[j].Values[i])
-			}
-			rows = append(rows, row{
-				timestamp: timestamp,
-				fields:    fields,
-			})
-		}
-
-		if len(rows) >= limit {
-			cancel()
-		}
-	}
-	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
-		return nil, err
-	}
-
-	return rows, nil
-}
-
-func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, nil, fmt.Errorf("cannot obtain tenanID: %w", err)
-	}
-	tenantIDs := []logstorage.TenantID{tenantID}
-
-	// Parse query
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
-	}
-
-	// Parse optional start and end args
-	start, okStart, err := getTimeNsec(r, "start")
-	if err != nil {
-		return nil, nil, err
-	}
-	end, okEnd, err := getTimeNsec(r, "end")
-	if err != nil {
-		return nil, nil, err
-	}
-	if okStart || okEnd {
-		if !okStart {
-			start = math.MinInt64
-		}
-		if !okEnd {
-			end = math.MaxInt64
-		}
-		q.AddTimeFilter(start, end)
-	}
-
-	return q, tenantIDs, nil
-}
-
-func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
-	s := r.FormValue(argName)
-	if s == "" {
-		return 0, false, nil
-	}
-	currentTimestamp := time.Now().UnixNano()
-	nsecs, err := promutils.ParseTimeAt(s, currentTimestamp)
-	if err != nil {
-		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
-	}
-	return nsecs, true, nil
-}

app/vlselect/logsql/logsql.qtpl

@@ -1,32 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// ValuesWithHitsJSON generates JSON from the given values.
-{% func ValuesWithHitsJSON(values []logstorage.ValueWithHits) %}
-{
-	"values":{%= valuesWithHitsJSONArray(values) %}
-}
-{% endfunc %}
-
-{% func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) %}
-[
-	{% if len(values) > 0 %}
-		{%= valueWithHitsJSON(values[0]) %}
-		{% for _, v := range values[1:] %}
-			,{%= valueWithHitsJSON(v) %}
-		{% endfor %}
-	{% endif %}
-]
-{% endfunc %}
-
-{% func valueWithHitsJSON(v logstorage.ValueWithHits) %}
-{
-	"value":{%q= v.Value %},
-	"hits":{%dul= v.Hits %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/logsql.qtpl.go

@@ -1,152 +0,0 @@
-// Code generated by qtc from "logsql.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/logsql.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/logsql.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ValuesWithHitsJSON generates JSON from the given values.
-
-//line app/vlselect/logsql/logsql.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/logsql.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/logsql.qtpl:8
-func StreamValuesWithHitsJSON(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:8
-	qw422016.N().S(`{"values":`)
-//line app/vlselect/logsql/logsql.qtpl:10
-	streamvaluesWithHitsJSONArray(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:10
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:12
-func WriteValuesWithHitsJSON(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:12
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-	StreamValuesWithHitsJSON(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:12
-func ValuesWithHitsJSON(values []logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:12
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:12
-	WriteValuesWithHitsJSON(qb422016, values)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:12
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:12
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:12
-}
-
-//line app/vlselect/logsql/logsql.qtpl:14
-func streamvaluesWithHitsJSONArray(qw422016 *qt422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:14
-	qw422016.N().S(`[`)
-//line app/vlselect/logsql/logsql.qtpl:16
-	if len(values) > 0 {
-//line app/vlselect/logsql/logsql.qtpl:17
-		streamvalueWithHitsJSON(qw422016, values[0])
-//line app/vlselect/logsql/logsql.qtpl:18
-		for _, v := range values[1:] {
-//line app/vlselect/logsql/logsql.qtpl:18
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/logsql.qtpl:19
-			streamvalueWithHitsJSON(qw422016, v)
-//line app/vlselect/logsql/logsql.qtpl:20
-		}
-//line app/vlselect/logsql/logsql.qtpl:21
-	}
-//line app/vlselect/logsql/logsql.qtpl:21
-	qw422016.N().S(`]`)
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:23
-func writevaluesWithHitsJSONArray(qq422016 qtio422016.Writer, values []logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:23
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-	streamvaluesWithHitsJSONArray(qw422016, values)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:23
-func valuesWithHitsJSONArray(values []logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:23
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:23
-	writevaluesWithHitsJSONArray(qb422016, values)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:23
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:23
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:23
-}
-
-//line app/vlselect/logsql/logsql.qtpl:25
-func streamvalueWithHitsJSON(qw422016 *qt422016.Writer, v logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:25
-	qw422016.N().S(`{"value":`)
-//line app/vlselect/logsql/logsql.qtpl:27
-	qw422016.N().Q(v.Value)
-//line app/vlselect/logsql/logsql.qtpl:27
-	qw422016.N().S(`,"hits":`)
-//line app/vlselect/logsql/logsql.qtpl:28
-	qw422016.N().DUL(v.Hits)
-//line app/vlselect/logsql/logsql.qtpl:28
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/logsql.qtpl:30
-}
-
-//line app/vlselect/logsql/logsql.qtpl:30
-func writevalueWithHitsJSON(qq422016 qtio422016.Writer, v logstorage.ValueWithHits) {
-//line app/vlselect/logsql/logsql.qtpl:30
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-	streamvalueWithHitsJSON(qw422016, v)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-}
-
-//line app/vlselect/logsql/logsql.qtpl:30
-func valueWithHitsJSON(v logstorage.ValueWithHits) string {
-//line app/vlselect/logsql/logsql.qtpl:30
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/logsql.qtpl:30
-	writevalueWithHitsJSON(qb422016, v)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/logsql.qtpl:30
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/logsql.qtpl:30
-	return qs422016
-//line app/vlselect/logsql/logsql.qtpl:30
-}

app/vlselect/logsql/query_response.qtpl

@@ -1,41 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// JSONRow creates JSON row from the given fields.
-{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-{
-	{% code c := &columns[0] %}
-	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% code columns = columns[1:] %}
-	{% for colIdx := range columns %}
-		{% code c := &columns[colIdx] %}
-		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% endfor %}
-}{% newline %}
-{% endfunc %}
-
-// JSONRows prints formatted rows
-{% func JSONRows(rows [][]logstorage.Field) %}
-	{% if len(rows) == 0 %}
-		{% return %}
-	{% endif %}
-	{% for _, fields := range rows %}
-	{
-		{% if len(fields) > 0 %}
-			{% code
-				f := fields[0]
-				fields = fields[1:]
-			%}
-			{%q= f.Name %}:{%q= f.Value %}
-			{% for _, f := range fields %}
-				,{%q= f.Name %}:{%q= f.Value %}
-			{% endfor %}
-		{% endif %}
-	}{% newline %}
-	{% endfor %}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/query_response.qtpl.go

@@ -1,166 +0,0 @@
-// Code generated by qtc from "query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/query_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/query_response.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// JSONRow creates JSON row from the given fields.
-
-//line app/vlselect/logsql/query_response.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:8
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:10
-	c := &columns[0]
-
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:12
-	columns = columns[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:13
-	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:14
-		c := &columns[colIdx]
-
-//line app/vlselect/logsql/query_response.qtpl:14
-		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:16
-	}
-//line app/vlselect/logsql/query_response.qtpl:16
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:17
-	qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-//line app/vlselect/logsql/query_response.qtpl:18
-func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-//line app/vlselect/logsql/query_response.qtpl:18
-func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:18
-	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-// JSONRows prints formatted rows
-
-//line app/vlselect/logsql/query_response.qtpl:21
-func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:22
-	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:23
-		return
-//line app/vlselect/logsql/query_response.qtpl:24
-	}
-//line app/vlselect/logsql/query_response.qtpl:25
-	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:25
-		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:27
-		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:29
-			f := fields[0]
-			fields = fields[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:33
-			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:33
-				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:35
-			}
-//line app/vlselect/logsql/query_response.qtpl:36
-		}
-//line app/vlselect/logsql/query_response.qtpl:36
-		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:37
-		qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:38
-	}
-//line app/vlselect/logsql/query_response.qtpl:39
-}
-
-//line app/vlselect/logsql/query_response.qtpl:39
-func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:39
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-}
-
-//line app/vlselect/logsql/query_response.qtpl:39
-func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:39
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:39
-	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:39
-}

app/vlselect/main.go

@@ -1,240 +0,0 @@
-package vlselect
-
-import (
-	"context"
-	"embed"
-	"flag"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration")
-	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
-		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden on a per-query basis via 'timeout' query arg")
-)
-
-func getDefaultMaxConcurrentRequests() int {
-	n := cgroup.AvailableCPUs()
-	if n <= 4 {
-		n *= 2
-	}
-	if n > 16 {
-		// A single request can saturate all the CPU cores, so there is no sense
-		// in allowing higher number of concurrent requests - they will just contend
-		// for unavailable CPU time.
-		n = 16
-	}
-	return n
-}
-
-// Init initializes vlselect
-func Init() {
-	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
-}
-
-// Stop stops vlselect
-func Stop() {
-}
-
-var concurrencyLimitCh chan struct{}
-
-var (
-	concurrencyLimitReached = metrics.NewCounter(`vl_concurrent_select_limit_reached_total`)
-	concurrencyLimitTimeout = metrics.NewCounter(`vl_concurrent_select_limit_timeout_total`)
-
-	_ = metrics.NewGauge(`vl_concurrent_select_capacity`, func() float64 {
-		return float64(cap(concurrencyLimitCh))
-	})
-	_ = metrics.NewGauge(`vl_concurrent_select_current`, func() float64 {
-		return float64(len(concurrencyLimitCh))
-	})
-)
-
-//go:embed vmui
-var vmuiFiles embed.FS
-
-var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
-
-// RequestHandler handles select requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if !strings.HasPrefix(path, "/select/") {
-		// Skip requests, which do not start with /select/, since these aren't our requests.
-		return false
-	}
-	path = strings.ReplaceAll(path, "//", "/")
-
-	if path == "/select/vmui" {
-		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
-		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
-		// is hidden behind vmauth or similar proxy.
-		_ = r.ParseForm()
-		newURL := "vmui/?" + r.Form.Encode()
-		httpserver.Redirect(w, newURL)
-		return true
-	}
-	if strings.HasPrefix(path, "/select/vmui/") {
-		if strings.HasPrefix(path, "/select/vmui/static/") {
-			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
-			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
-			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
-			w.Header().Set("Cache-Control", "max-age=31536000")
-		}
-		r.URL.Path = strings.TrimPrefix(path, "/select")
-		vmuiFileServer.ServeHTTP(w, r)
-		return true
-	}
-
-	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
-	startTime := time.Now()
-	ctx := r.Context()
-	d := getMaxQueryDuration(r)
-	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
-	defer cancel()
-
-	stopCh := ctxWithTimeout.Done()
-	select {
-	case concurrencyLimitCh <- struct{}{}:
-		defer func() { <-concurrencyLimitCh }()
-	default:
-		// Sleep for a while until giving up. This should resolve short bursts in requests.
-		concurrencyLimitReached.Inc()
-		select {
-		case concurrencyLimitCh <- struct{}{}:
-			defer func() { <-concurrencyLimitCh }()
-		case <-stopCh:
-			switch ctxWithTimeout.Err() {
-			case context.Canceled:
-				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-				requestURI := httpserver.GetRequestURI(r)
-				logger.Infof("client has canceled the pending request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-					time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			case context.DeadlineExceeded:
-				concurrencyLimitTimeout.Inc()
-				err := &httpserver.ErrorWithStatusCode{
-					Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-						"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-						"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
-						"to pass bigger value to 'timeout' query arg",
-						d.Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
-					StatusCode: http.StatusServiceUnavailable,
-				}
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return true
-		}
-	}
-
-	if path == "/select/logsql/tail" {
-		logsqlTailRequests.Inc()
-		// Process live tailing request without timeout (e.g. use ctx instead of ctxWithTimeout),
-		// since it is OK to run live tailing requests for very long time.
-		logsql.ProcessLiveTailRequest(ctx, w, r)
-		return true
-	}
-
-	ok := processSelectRequest(ctxWithTimeout, w, r, path)
-	if !ok {
-		return false
-	}
-
-	err := ctxWithTimeout.Err()
-	switch err {
-	case nil:
-		// nothing to do
-	case context.Canceled:
-		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		requestURI := httpserver.GetRequestURI(r)
-		logger.Infof("client has canceled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-			time.Since(startTime).Seconds(), remoteAddr, requestURI)
-	case context.DeadlineExceeded:
-		err = &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
-				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", d.Seconds(), maxQueryDuration),
-			StatusCode: http.StatusServiceUnavailable,
-		}
-		httpserver.Errorf(w, r, "%s", err)
-	default:
-		httpserver.Errorf(w, r, "unexpected error: %s", err)
-	}
-
-	return true
-}
-
-func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
-	httpserver.EnableCORS(w, r)
-	switch path {
-	case "/select/logsql/field_names":
-		logsqlFieldNamesRequests.Inc()
-		logsql.ProcessFieldNamesRequest(ctx, w, r)
-		return true
-	case "/select/logsql/field_values":
-		logsqlFieldValuesRequests.Inc()
-		logsql.ProcessFieldValuesRequest(ctx, w, r)
-		return true
-	case "/select/logsql/hits":
-		logsqlHitsRequests.Inc()
-		logsql.ProcessHitsRequest(ctx, w, r)
-		return true
-	case "/select/logsql/query":
-		logsqlQueryRequests.Inc()
-		logsql.ProcessQueryRequest(ctx, w, r)
-		return true
-	case "/select/logsql/stream_field_names":
-		logsqlStreamFieldNamesRequests.Inc()
-		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
-		return true
-	case "/select/logsql/stream_field_values":
-		logsqlStreamFieldValuesRequests.Inc()
-		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
-		return true
-	case "/select/logsql/stream_ids":
-		logsqlStreamIDsRequests.Inc()
-		logsql.ProcessStreamIDsRequest(ctx, w, r)
-		return true
-	case "/select/logsql/streams":
-		logsqlStreamsRequests.Inc()
-		logsql.ProcessStreamsRequest(ctx, w, r)
-		return true
-	default:
-		return false
-	}
-}
-
-// getMaxQueryDuration returns the maximum duration for query from r.
-func getMaxQueryDuration(r *http.Request) time.Duration {
-	dms, err := httputils.GetDuration(r, "timeout", 0)
-	if err != nil {
-		dms = 0
-	}
-	d := time.Duration(dms) * time.Millisecond
-	if d <= 0 || d > *maxQueryDuration {
-		d = *maxQueryDuration
-	}
-	return d
-}
-
-var (
-	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
-	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
-	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
-	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
-	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
-	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
-	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
-	logsqlTailRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
-)

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +0,0 @@
-{
-  "files": {
-    "main.css": "./static/css/main.1041c3d4.css",
-    "main.js": "./static/js/main.8451d9b9.js",
-    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
-    "index.html": "./index.html"
-  },
-  "entrypoints": [
-    "static/css/main.1041c3d4.css",
-    "static/js/main.8451d9b9.js"
-  ]
-}

app/vlselect/vmui/index.html

@@ -1 +0,0 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8451d9b9.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/manifest.json

@@ -1,20 +0,0 @@
-{
-  "short_name": "Victoria Metrics UI",
-  "name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
-  "icons": [
-    {
-      "src": "favicon-32x32.png",
-      "sizes": "32x32",
-      "type": "image/png"
-    },
-    {
-      "src": "apple-touch-icon.png",
-      "type": "image/png",
-      "sizes": "192x192"
-    }
-  ],
-  "start_url": ".",
-  "display": "standalone",
-  "theme_color": "#000000",
-  "background_color": "#ffffff"
-}

app/vlselect/vmui/robots.txt

@@ -1,3 +0,0 @@
-# https://www.robotstxt.org/robotstxt.html
-User-agent: *
-Disallow:

app/vlstorage/main.go

@@ -1,207 +0,0 @@
-package vlstorage
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
-		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
-	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
-		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
-	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
-		"see https://docs.victoriametrics.com/victorialogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
-		"see https://docs.victoriametrics.com/victorialogs/#storage")
-	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
-		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
-	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields ; see also -logIngestedRows")
-	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
-		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
-	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
-		"the storage stops accepting new data")
-)
-
-// Init initializes vlstorage.
-//
-// Stop must be called when vlstorage is no longer needed
-func Init() {
-	if strg != nil {
-		logger.Panicf("BUG: Init() has been already called")
-	}
-
-	if retentionPeriod.Duration() < 24*time.Hour {
-		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
-	}
-	cfg := &logstorage.StorageConfig{
-		Retention:              retentionPeriod.Duration(),
-		MaxDiskSpaceUsageBytes: maxDiskSpaceUsageBytes.N,
-		FlushInterval:          *inmemoryDataFlushInterval,
-		FutureRetention:        futureRetention.Duration(),
-		LogNewStreams:          *logNewStreams,
-		LogIngestedRows:        *logIngestedRows,
-		MinFreeDiskSpaceBytes:  minFreeDiskSpaceBytes.N,
-	}
-	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
-	startTime := time.Now()
-	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
-
-	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
-	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
-		"smallPartSize: %d bytes; bigPartSize: %d bytes",
-		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
-		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
-
-	// register storage metrics
-	storageMetrics = metrics.NewSet()
-	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, strg)
-	})
-	metrics.RegisterSet(storageMetrics)
-}
-
-// Stop stops vlstorage.
-func Stop() {
-	metrics.UnregisterSet(storageMetrics, true)
-	storageMetrics = nil
-
-	strg.MustClose()
-	strg = nil
-}
-
-var strg *logstorage.Storage
-var storageMetrics *metrics.Set
-
-// CanWriteData returns non-nil error if it cannot write data to vlstorage.
-func CanWriteData() error {
-	if strg.IsReadOnly() {
-		return &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
-				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
-			StatusCode: http.StatusTooManyRequests,
-		}
-	}
-	return nil
-}
-
-// MustAddRows adds lr to vlstorage
-//
-// It is advised to call CanWriteData() before calling MustAddRows()
-func MustAddRows(lr *logstorage.LogRows) {
-	strg.MustAddRows(lr)
-}
-
-// RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
-	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
-}
-
-// GetFieldNames executes q and returns field names seen in results.
-func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return strg.GetFieldNames(ctx, tenantIDs, q)
-}
-
-// GetFieldValues executes q and returns unique values for the fieldName seen in results.
-//
-// If limit > 0, then up to limit unique values are returned.
-func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
-}
-
-// GetStreamFieldNames executes q and returns stream field names seen in results.
-func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
-}
-
-// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
-//
-// If limit > 0, then up to limit unique stream field values are returned.
-func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-}
-
-// GetStreams executes q and returns streams seen in query results.
-//
-// If limit > 0, then up to limit unique streams are returned.
-func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreams(ctx, tenantIDs, q, limit)
-}
-
-// GetStreamIDs executes q and returns streamIDs seen in query results.
-//
-// If limit > 0, then up to limit unique streamIDs are returned.
-func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
-}
-
-func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
-	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
-
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
-
-	isReadOnly := uint64(0)
-	if ss.IsReadOnly {
-		isReadOnly = 1
-	}
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), isReadOnly)
-
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/inmemory"}`, ss.InmemoryActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/small"}`, ss.SmallPartActiveMerges)
-	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/big"}`, ss.BigPartActiveMerges)
-
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/inmemory"}`, ss.InmemoryMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/small"}`, ss.SmallPartMergesTotal)
-	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/big"}`, ss.BigPartMergesTotal)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/inmemory"}`, ss.InmemoryRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/small"}`, ss.SmallPartRowsCount)
-	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/big"}`, ss.BigPartRowsCount)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/inmemory"}`, ss.InmemoryParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/small"}`, ss.SmallParts)
-	metrics.WriteGaugeUint64(w, `vl_storage_parts{type="storage/big"}`, ss.BigParts)
-
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/inmemory"}`, ss.InmemoryBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/small"}`, ss.SmallPartBlocks)
-	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/big"}`, ss.BigPartBlocks)
-
-	metrics.WriteGaugeUint64(w, `vl_partitions`, ss.PartitionsCount)
-	metrics.WriteCounterUint64(w, `vl_streams_created_total`, ss.StreamsCreatedTotal)
-
-	metrics.WriteGaugeUint64(w, `vl_indexdb_rows`, ss.IndexdbItemsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_parts`, ss.IndexdbPartsCount)
-	metrics.WriteGaugeUint64(w, `vl_indexdb_blocks`, ss.IndexdbBlocksCount)
-
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="indexdb"}`, ss.IndexdbSizeBytes)
-	metrics.WriteGaugeUint64(w, `vl_data_size_bytes{type="storage"}`, ss.CompressedInmemorySize+ss.CompressedSmallPartSize+ss.CompressedBigPartSize)
-
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/inmemory"}`, ss.CompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/small"}`, ss.CompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_compressed_data_size_bytes{type="storage/big"}`, ss.CompressedBigPartSize)
-
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/inmemory"}`, ss.UncompressedInmemorySize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/small"}`, ss.UncompressedSmallPartSize)
-	metrics.WriteGaugeUint64(w, `vl_uncompressed_data_size_bytes{type="storage/big"}`, ss.UncompressedBigPartSize)
-
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
-	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
-}

app/vmagent/Makefile

@@ -85,12 +85,6 @@ vmagent-linux-arm64:
 vmagent-linux-ppc64le:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmagent-linux-s390x:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vmagent-linux-loong64:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 vmagent-linux-386:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmagent/common/push_ctx.go

@@ -3,15 +3,13 @@ package common
 import (
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // PushCtx is a context used for populating WriteRequest.
 type PushCtx struct {
-	// WriteRequest contains the WriteRequest, which must be pushed later to remote storage.
-	//
-	// The actual labels and samples for the time series are stored in Labels and Samples fields.
 	WriteRequest prompbmarshal.WriteRequest
 
 	// Labels contains flat list of all the labels used in WriteRequest.
@@ -23,7 +21,13 @@ type PushCtx struct {
 
 // Reset resets ctx.
 func (ctx *PushCtx) Reset() {
-	ctx.WriteRequest.Reset()
+	tss := ctx.WriteRequest.Timeseries
+	for i := range tss {
+		ts := &tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
+	ctx.WriteRequest.Timeseries = ctx.WriteRequest.Timeseries[:0]
 
 	promrelabel.CleanLabels(ctx.Labels)
 	ctx.Labels = ctx.Labels[:0]
@@ -35,10 +39,15 @@ func (ctx *PushCtx) Reset() {
 //
 // Call PutPushCtx when the ctx is no longer needed.
 func GetPushCtx() *PushCtx {
-	if v := pushCtxPool.Get(); v != nil {
-		return v.(*PushCtx)
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*PushCtx)
+		}
+		return &PushCtx{}
 	}
-	return &PushCtx{}
 }
 
 // PutPushCtx returns ctx to the pool.
@@ -46,7 +55,12 @@ func GetPushCtx() *PushCtx {
 // ctx mustn't be used after returning to the pool.
 func PutPushCtx(ctx *PushCtx) {
 	ctx.Reset()
-	pushCtxPool.Put(ctx)
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
 }
 
 var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *PushCtx, cgroup.AvailableCPUs())

app/vmagent/csvimport/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -26,7 +25,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []parser.Row) error {
+	return parser.ParseStream(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
@@ -65,9 +64,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/datadog/request_handler.go

@@ -1,4 +1,4 @@
-package datadogv1
+package datadog
 
 import (
 	"net/http"
@@ -8,32 +8,32 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv1"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv1"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv1"}`)
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
 )
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
 	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(series []datadogv1.Series) error {
+	return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
 		return insertRows(at, series, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -62,7 +62,7 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbm
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := datadogutils.SplitTag(tag)
+			name, value := parser.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}
@@ -87,9 +87,7 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbm
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/datadogsketches/request_handler.go

@@ -1,95 +0,0 @@
-package datadogsketches
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
-		return insertRows(at, sketches, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for _, sketch := range sketches {
-		ms := sketch.ToSummary()
-		for _, m := range ms {
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: m.Name,
-			})
-			for _, label := range m.Labels {
-				labels = append(labels, prompbmarshal.Label{
-					Name:  label.Name,
-					Value: label.Value,
-				})
-			}
-			for _, tag := range sketch.Tags {
-				name, value := datadogutils.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
-				labels = append(labels, prompbmarshal.Label{
-					Name:  name,
-					Value: value,
-				})
-			}
-			labels = append(labels, extraLabels...)
-			samplesLen := len(samples)
-			for _, p := range m.Points {
-				samples = append(samples, prompbmarshal.Sample{
-					Timestamp: p.Timestamp,
-					Value:     p.Value,
-				})
-			}
-			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[samplesLen:],
-			})
-		}
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/datadogv2/request_handler.go

@@ -1,102 +0,0 @@
-package datadogv2
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv2"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv2"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv2"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/v2/series request.
-//
-// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ct := req.Header.Get("Content-Type")
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, ct, func(series []datadogv2.Series) error {
-		return insertRows(at, series, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range series {
-		ss := &series[i]
-		rowsTotal += len(ss.Points)
-		labelsLen := len(labels)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: ss.Metric,
-		})
-		for _, rs := range ss.Resources {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  rs.Type,
-				Value: rs.Name,
-			})
-		}
-		if ss.SourceTypeName != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "source_type_name",
-				Value: ss.SourceTypeName,
-			})
-		}
-		for _, tag := range ss.Tags {
-			name, value := datadogutils.SplitTag(tag)
-			if name == "host" {
-				name = "exported_host"
-			}
-			labels = append(labels, prompbmarshal.Label{
-				Name:  name,
-				Value: value,
-			})
-		}
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		for _, pt := range ss.Points {
-			samples = append(samples, prompbmarshal.Sample{
-				Timestamp: pt.Timestamp * 1000,
-				Value:     pt.Value,
-			})
-		}
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/graphite/request_handler.go

@@ -5,10 +5,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -21,12 +19,10 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
-		return insertRows(nil, rows)
-	})
+	return parser.ParseStream(r, insertRows)
 }
 
-func insertRows(at *auth.Token, rows []parser.Row) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -59,9 +55,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -10,18 +10,18 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metric name if InfluxDB line contains only a single field")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
 )
@@ -35,9 +35,9 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
-		return insertRows(at, db, rows, nil)
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+		return insertRows(nil, db, rows, nil)
 	})
 }
 
@@ -54,7 +54,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }
@@ -129,9 +129,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	if !remotewrite.TryPush(at, &ctx.ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)
@@ -159,15 +157,25 @@ func (ctx *pushCtx) reset() {
 }
 
 func getPushCtx() *pushCtx {
-	if v := pushCtxPool.Get(); v != nil {
-		return v.(*pushCtx)
+	select {
+	case ctx := <-pushCtxPoolCh:
+		return ctx
+	default:
+		if v := pushCtxPool.Get(); v != nil {
+			return v.(*pushCtx)
+		}
+		return &pushCtx{}
 	}
-	return &pushCtx{}
 }
 
 func putPushCtx(ctx *pushCtx) {
 	ctx.reset()
-	pushCtxPool.Put(ctx)
+	select {
+	case pushCtxPoolCh <- ctx:
+	default:
+		pushCtxPool.Put(ctx)
+	}
 }
 
 var pushCtxPool sync.Pool
+var pushCtxPoolCh = make(chan *pushCtx, cgroup.AvailableCPUs())

app/vmagent/main.go

@@ -8,19 +8,16 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
@@ -42,16 +39,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 )
 
 var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
@@ -63,19 +58,18 @@ var (
 		"See also -graphiteListenAddr.useProxyProtocol")
 	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
 		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
 	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
-		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
+	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	dryRun        = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
 
@@ -100,40 +94,33 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
 		return
 	}
 	if *dryRun {
-		if err := promscrape.CheckConfig(); err != nil {
-			logger.Fatalf("error when checking -promscrape.config: %s", err)
-		}
 		if err := remotewrite.CheckRelabelConfigs(); err != nil {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
-		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("all the configs are ok; exiting with 0 status code")
+		logger.Infof("all the configs are ok; exitting with 0 status code")
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8429"}
-	}
-	logger.Infof("starting vmagent at %q...", listenAddrs)
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
-	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, false)
+			return influx.InsertHandlerForReader(r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
@@ -148,23 +135,24 @@ func main() {
 		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
 	}
 
-	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
+	promscrape.Init(remotewrite.Push)
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
+	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
-	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	remotewrite.StopIngestionRateLimiter()
-	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -222,12 +210,12 @@ func getAuthTokenFromPath(path string) (*auth.Token, error) {
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
+		if r.Method != "GET" {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent/'>https://docs.victoriametrics.com/vmagent/</a></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent.html'>https://docs.victoriametrics.com/vmagent.html</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
@@ -266,10 +254,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		path = strings.TrimSuffix(path, "/")
 	}
 	switch path {
-	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
-		if common.HandleVMProtoServerHandshake(w, r) {
-			return true
-		}
+	case "/prometheus/api/v1/write", "/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -318,53 +303,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(nil, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "/newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "/newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(nil, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	case "/datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv2WriteErrors.Inc()
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -373,15 +315,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -436,7 +369,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -445,25 +378,22 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, *configAuthKey, "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		var bb bytesutil.ByteBuffer
 		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
-			return true
-		}
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
 	case "/ready":
-		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
 			http.Error(w, errMsg, http.StatusTooEarly)
 		} else {
@@ -515,7 +445,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
 	}
 	switch p.Suffix {
-	case "prometheus/", "prometheus", "prometheus/api/v1/write", "prometheus/api/v1/push":
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(at, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -564,52 +494,10 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(at, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(at, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	case "datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv2WriteErrors.Inc()
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -617,15 +505,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -676,29 +555,14 @@ var (
 
 	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
-	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-
-	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-
-	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-
-	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-
-	newrelicInventoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/inventory/deltas", protocol="newrelic"}`)
-	newrelicCheckRequest      = metrics.NewCounter(`vm_http_requests_total{path="/newrelic", protocol="newrelic"}`)
-
 	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
 
@@ -720,7 +584,7 @@ func usage() {
 	const s = `
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
 
-See the docs at https://docs.victoriametrics.com/vmagent/ .
+See the docs at https://docs.victoriametrics.com/vmagent.html .
 `
 	flagutil.Usage(s)
 }

app/vmagent/multiarch/Dockerfile

@@ -1,7 +1,7 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
 ARG certs_image
 ARG root_image
-FROM $certs_image AS certs
+FROM $certs_image as certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image

app/vmagent/native/request_handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -30,12 +30,12 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
+	return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
 		return insertRows(at, block, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -84,8 +84,6 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/newrelic/request_handler.go

@@ -1,88 +0,0 @@
-package newrelic
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="newrelic"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="newrelic"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="newrelic"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	isGzip := ce == "gzip"
-	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
-		return insertRows(at, rows, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	samplesCount := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		tags := r.Tags
-		srcSamples := r.Samples
-		for j := range srcSamples {
-			s := &srcSamples[j]
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: bytesutil.ToUnsafeString(s.Name),
-			})
-			for k := range tags {
-				t := &tags[k]
-				labels = append(labels, prompbmarshal.Label{
-					Name:  bytesutil.ToUnsafeString(t.Key),
-					Value: bytesutil.ToUnsafeString(t.Value),
-				})
-			}
-			samples = append(samples, prompbmarshal.Sample{
-				Value:     s.Value,
-				Timestamp: r.Timestamp,
-			})
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[len(samples)-1:],
-			})
-			labels = append(labels, extraLabels...)
-		}
-		samplesCount += len(srcSamples)
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(samplesCount)
-	}
-	rowsPerInsert.Update(float64(samplesCount))
-	return nil
-}

app/vmagent/opentelemetry/request_handler.go

@@ -1,77 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
-)
-
-// InsertHandler processes opentelemetry metrics.
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	var processBody func([]byte) ([]byte, error)
-	if req.Header.Get("Content-Type") == "application/json" {
-		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
-			processBody = firehose.ProcessRequestBody
-		} else {
-			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-		}
-	}
-	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
-		return insertRows(at, tss, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range tss {
-		ts := &tss[i]
-		rowsTotal += len(ts.Samples)
-		labelsLen := len(labels)
-		labels = append(labels, ts.Labels...)
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		samples = append(samples, ts.Samples...)
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/opentsdb/request_handler.go

@@ -7,7 +7,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,7 +19,7 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, insertRows)
+	return parser.ParseStream(r, insertRows)
 }
 
 func insertRows(rows []parser.Row) error {
@@ -56,9 +55,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(nil, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/opentsdbhttp/request_handler.go

@@ -9,7 +9,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -25,7 +24,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []parser.Row) error {
+	return parser.ParseStream(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
@@ -64,9 +63,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/prometheusimport/request_handler.go

@@ -10,7 +10,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -32,7 +31,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
+	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)
@@ -73,9 +72,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/prometheusimport/request_handler_test.go

@@ -23,14 +23,14 @@ var (
 func TestInsertHandler(t *testing.T) {
 	setUp()
 	defer tearDown()
-	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
+	req := httptest.NewRequest("POST", "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unxepected error %s", err)
+		t.Errorf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 

app/vmagent/promremotewrite/request_handler.go

@@ -6,10 +6,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -26,8 +27,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries) error {
+	return parser.ParseStream(req.Body, func(tss []prompb.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }
@@ -47,8 +47,8 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 		for i := range ts.Labels {
 			label := &ts.Labels[i]
 			labels = append(labels, prompbmarshal.Label{
-				Name:  label.Name,
-				Value: label.Value,
+				Name:  bytesutil.ToUnsafeString(label.Name),
+				Value: bytesutil.ToUnsafeString(label.Value),
 			})
 		}
 		labels = append(labels, extraLabels...)
@@ -68,9 +68,7 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []pr
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/remotewrite/client.go

@@ -2,7 +2,6 @@ package remotewrite
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,42 +10,31 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
-	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
-
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
-		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
-		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
+		"By default the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
+		"is sent after temporary unavailability of the remote storage")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
-	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
 	tlsInsecureSkipVerify = flagutil.NewArrayBool("remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
 	tlsCertFile           = flagutil.NewArrayString("remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
 		"to the corresponding -remoteWrite.url")
 	tlsKeyFile = flagutil.NewArrayString("remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
 	tlsCAFile  = flagutil.NewArrayString("remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
-		"By default, system CA is used")
+		"By default system CA is used")
 	tlsServerName = flagutil.NewArrayString("remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
-		"By default, the server name from -remoteWrite.url is used")
+		"By default the server name from -remoteWrite.url is used")
 
 	headers = flagutil.NewArrayString("remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
 		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
@@ -63,10 +51,8 @@ var (
 	oauth2ClientID         = flagutil.NewArrayString("remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecret     = flagutil.NewArrayString("remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
 	oauth2ClientSecretFile = flagutil.NewArrayString("remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
-	oauth2EndpointParams   = flagutil.NewArrayString("remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
-		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
-	oauth2TokenURL = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
-	oauth2Scopes   = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+	oauth2TokenURL         = flagutil.NewArrayString("remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	oauth2Scopes           = flagutil.NewArrayString("remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
 
 	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
 		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
@@ -83,18 +69,14 @@ var (
 type client struct {
 	sanitizedURL   string
 	remoteWriteURL string
-
-	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
-	useVMProto bool
-
-	fq *persistentqueue.FastQueue
-	hc *http.Client
+	fq             *persistentqueue.FastQueue
+	hc             *http.Client
 
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
 
-	rl *ratelimiter.RateLimiter
+	rl rateLimiter
 
 	bytesSent       *metrics.Counter
 	blocksSent      *metrics.Counter
@@ -113,15 +95,17 @@ type client struct {
 func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
 	authCfg, err := getAuthConfig(argIdx)
 	if err != nil {
-		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Panicf("FATAL: cannot initialize auth config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
+	tlsCfg := authCfg.NewTLSConfig()
 	awsCfg, err := getAWSAPIConfig(argIdx)
 	if err != nil {
-		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
+		logger.Fatalf("FATAL: cannot initialize AWS Config for remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
-		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
-		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
+		DialContext:         statDial,
+		TLSClientConfig:     tlsCfg,
+		TLSHandshakeTimeout: 10 * time.Second,
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
 		IdleConnTimeout:     time.Minute,
@@ -138,52 +122,33 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
-	hc := &http.Client{
-		Transport: authCfg.NewRoundTripper(tr),
-		Timeout:   sendTimeout.GetOptionalArg(argIdx),
-	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
 		remoteWriteURL: remoteWriteURL,
 		authCfg:        authCfg,
 		awsCfg:         awsCfg,
 		fq:             fq,
-		hc:             hc,
-		stopCh:         make(chan struct{}),
+		hc: &http.Client{
+			Transport: tr,
+			Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
+		},
+		stopCh: make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
-
-	useVMProto := forceVMProto.GetOptionalArg(argIdx)
-	usePromProto := forcePromProto.GetOptionalArg(argIdx)
-	if useVMProto && usePromProto {
-		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
-	}
-	if !useVMProto && !usePromProto {
-		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
-		doRequest := func(url string) (*http.Response, error) {
-			return c.doRequest(url, nil)
-		}
-		useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
-		if !useVMProto {
-			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
-				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", sanitizedURL)
-		}
-	}
-	c.useVMProto = useVMProto
-
 	return c
 }
 
 func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
-	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
-	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
+	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
-		c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
+		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
+	c.rl.limitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
+
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
 	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(rateLimit.GetOptionalArg(argIdx))
+		return float64(rateLimit.GetOptionalArgOrDefault(argIdx, 0))
 	})
 	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
 	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
@@ -235,16 +200,10 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
 	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
 	if clientSecretFile != "" || clientSecret != "" {
-		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
-		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
-		}
 		oauth2Cfg = &promauth.OAuth2Config{
 			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
 			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
-			EndpointParams:   endpointParams,
 			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
 			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
 		}
@@ -268,7 +227,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	}
 	authCfg, err := opts.NewConfig()
 	if err != nil {
-		return nil, fmt.Errorf("cannot populate auth config for remoteWrite idx: %d, err: %w", argIdx, err)
+		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
 	return authCfg, nil
 }
@@ -300,11 +259,6 @@ func (c *client) runWorker() {
 		if !ok {
 			return
 		}
-		if len(block) == 0 {
-			// skip empty data blocks from sending
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241
-			continue
-		}
 		go func() {
 			startTime := time.Now()
 			ch <- c.sendBlock(block)
@@ -317,7 +271,7 @@ func (c *client) runWorker() {
 				continue
 			}
 			// Return unsent block to the queue.
-			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			c.fq.MustWriteBlock(block)
 			return
 		case <-c.stopCh:
 			// c must be stopped. Wait for a while in the hope the block will be sent.
@@ -326,92 +280,55 @@ func (c *client) runWorker() {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
-					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+					c.fq.MustWriteBlock(block)
 				}
 			case <-time.After(graceDuration):
 				// Return unsent block to the queue.
-				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+				c.fq.MustWriteBlock(block)
 			}
 			return
 		}
 	}
 }
 
-func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
-	req, err := c.newRequest(url, body)
-	if err != nil {
-		return nil, err
+// sendBlockHTTP returns false only if c.stopCh is closed.
+// Otherwise it tries sending the block to remote storage indefinitely.
+func (c *client) sendBlockHTTP(block []byte) bool {
+	c.rl.register(len(block), c.stopCh)
+	retryDuration := time.Second
+	retriesCount := 0
+	c.bytesSent.Add(len(block))
+	c.blocksSent.Inc()
+	sigv4Hash := ""
+	if c.awsCfg != nil {
+		sigv4Hash = awsapi.HashHex(block)
 	}
-	resp, err := c.hc.Do(req)
-	if err == nil {
-		return resp, nil
-	}
-	if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
-		return nil, err
-	}
-	// It is likely connection became stale or timed out during the first request.
-	// Make another attempt in hope request will succeed.
-	// If not, the error should be handled by the caller as usual.
-	// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
-	req, err = c.newRequest(url, body)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	resp, err = c.hc.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("second attempt: %w", err)
-	}
-	return resp, nil
-}
 
-func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
-	reqBody := bytes.NewBuffer(body)
-	req, err := http.NewRequest(http.MethodPost, url, reqBody)
+again:
+	req, err := http.NewRequest("POST", c.remoteWriteURL, bytes.NewBuffer(block))
 	if err != nil {
-		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", url, err)
-	}
-	err = c.authCfg.SetHeaders(req, true)
-	if err != nil {
-		return nil, err
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.sanitizedURL, err)
 	}
+	c.authCfg.SetHeaders(req, true)
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
-	if c.useVMProto {
-		h.Set("Content-Encoding", "zstd")
-		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
-	} else {
-		h.Set("Content-Encoding", "snappy")
-		h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	}
+	h.Set("Content-Encoding", "snappy")
+	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
 	if c.awsCfg != nil {
-		sigv4Hash := awsapi.HashHex(body)
 		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
-			return nil, fmt.Errorf("cannot sign remoteWrite request with AWS sigv4: %w", err)
+			// there is no need in retry, request will be rejected by client.Do and retried by code below
+			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
 		}
 	}
-	return req, nil
-}
-
-// sendBlockHTTP sends the given block to c.remoteWriteURL.
-//
-// The function returns false only if c.stopCh is closed.
-// Otherwise it tries sending the block to remote storage indefinitely.
-func (c *client) sendBlockHTTP(block []byte) bool {
-	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
-	retryDuration := timeutil.AddJitterToDuration(time.Second)
-	retriesCount := 0
-
-again:
 	startTime := time.Now()
-	resp, err := c.doRequest(c.remoteWriteURL, block)
+	resp, err := c.hc.Do(req)
 	c.requestDuration.UpdateDuration(startTime)
 	if err != nil {
 		c.errorsCount.Inc()
 		retryDuration *= 2
-		if retryDuration > maxRetryDuration {
-			retryDuration = maxRetryDuration
+		if retryDuration > time.Minute {
+			retryDuration = time.Minute
 		}
 		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
@@ -430,8 +347,6 @@ again:
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
 		c.requestsOKCount.Inc()
-		c.bytesSent.Add(len(block))
-		c.blocksSent.Inc()
 		return true
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
@@ -457,8 +372,8 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
+	if retryDuration > time.Minute {
+		retryDuration = time.Minute
 	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
@@ -481,3 +396,45 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
+
+type rateLimiter struct {
+	perSecondLimit int64
+
+	// mu protects budget and deadline from concurrent access.
+	mu sync.Mutex
+
+	// The current budget. It is increased by perSecondLimit every second.
+	budget int64
+
+	// The next deadline for increasing the budget by perSecondLimit
+	deadline time.Time
+
+	limitReached *metrics.Counter
+}
+
+func (rl *rateLimiter) register(dataLen int, stopCh <-chan struct{}) {
+	limit := rl.perSecondLimit
+	if limit <= 0 {
+		return
+	}
+
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+
+	for rl.budget <= 0 {
+		if d := time.Until(rl.deadline); d > 0 {
+			rl.limitReached.Inc()
+			t := timerpool.Get(d)
+			select {
+			case <-stopCh:
+				timerpool.Put(t)
+				return
+			case <-t.C:
+				timerpool.Put(t)
+			}
+		}
+		rl.budget += limit
+		rl.deadline = time.Now().Add(time.Second)
+	}
+	rl.budget -= int64(dataLen)
+}

app/vmagent/remotewrite/pendingseries.go

@@ -7,16 +7,13 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
 )
@@ -26,9 +23,6 @@ var (
 		"This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url")
 	maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock")
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
-	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
-		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
-		"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
 )
 
 type pendingSeries struct {
@@ -39,10 +33,9 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(pushBlock func(block []byte), significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
-	ps.wr.fq = fq
-	ps.wr.isVMRemoteWrite = isVMRemoteWrite
+	ps.wr.pushBlock = pushBlock
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
@@ -59,11 +52,10 @@ func (ps *pendingSeries) MustStop() {
 	ps.periodicFlusherWG.Wait()
 }
 
-func (ps *pendingSeries) TryPush(tss []prompbmarshal.TimeSeries) bool {
+func (ps *pendingSeries) Push(tss []prompbmarshal.TimeSeries) {
 	ps.mu.Lock()
-	ok := ps.wr.tryPush(tss)
+	ps.wr.push(tss)
 	ps.mu.Unlock()
-	return ok
 }
 
 func (ps *pendingSeries) periodicFlusher() {
@@ -71,58 +63,56 @@ func (ps *pendingSeries) periodicFlusher() {
 	if flushSeconds <= 0 {
 		flushSeconds = 1
 	}
-	d := timeutil.AddJitterToDuration(*flushInterval)
-	ticker := time.NewTicker(d)
+	ticker := time.NewTicker(*flushInterval)
 	defer ticker.Stop()
-	for {
+	mustStop := false
+	for !mustStop {
 		select {
 		case <-ps.stopCh:
-			ps.mu.Lock()
-			ps.wr.mustFlushOnStop()
-			ps.mu.Unlock()
-			return
+			mustStop = true
 		case <-ticker.C:
-			if fasttime.UnixTimestamp()-ps.wr.lastFlushTime.Load() < uint64(flushSeconds) {
+			if fasttime.UnixTimestamp()-atomic.LoadUint64(&ps.wr.lastFlushTime) < uint64(flushSeconds) {
 				continue
 			}
 		}
 		ps.mu.Lock()
-		_ = ps.wr.tryFlush()
+		ps.wr.flush()
 		ps.mu.Unlock()
 	}
 }
 
 type writeRequest struct {
-	lastFlushTime atomic.Uint64
+	// Move lastFlushTime to the top of the struct in order to guarantee atomic access on 32-bit architectures.
+	lastFlushTime uint64
 
-	// The queue to send blocks to.
-	fq *persistentqueue.FastQueue
+	// pushBlock is called when whe write request is ready to be sent.
+	pushBlock func(block []byte)
 
-	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	isVMRemoteWrite bool
-
-	// How many significant figures must be left before sending the writeRequest to fq.
+	// How many significant figures must be left before sending the writeRequest to pushBlock.
 	significantFigures int
 
-	// How many decimal digits after point must be left before sending the writeRequest to fq.
+	// How many decimal digits after point must be left before sending the writeRequest to pushBlock.
 	roundDigits int
 
 	wr prompbmarshal.WriteRequest
 
-	tss     []prompbmarshal.TimeSeries
+	tss []prompbmarshal.TimeSeries
+
 	labels  []prompbmarshal.Label
 	samples []prompbmarshal.Sample
-
-	// buf holds labels data
-	buf []byte
+	buf     []byte
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
+	// Do not reset pushBlock, significantFigures and roundDigits, since they are re-used.
 
 	wr.wr.Timeseries = nil
 
-	clear(wr.tss)
+	for i := range wr.tss {
+		ts := &wr.tss[i]
+		ts.Labels = nil
+		ts.Samples = nil
+	}
 	wr.tss = wr.tss[:0]
 
 	promrelabel.CleanLabels(wr.labels)
@@ -132,40 +122,23 @@ func (wr *writeRequest) reset() {
 	wr.buf = wr.buf[:0]
 }
 
-// mustFlushOnStop force pushes wr data into wr.fq
-//
-// This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
-func (wr *writeRequest) mustFlushOnStop() {
+func (wr *writeRequest) flush() {
 	wr.wr.Timeseries = wr.tss
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
-		logger.Panicf("BUG: final flush must always return true")
-	}
+	wr.adjustSampleValues()
+	atomic.StoreUint64(&wr.lastFlushTime, fasttime.UnixTimestamp())
+	pushWriteRequest(&wr.wr, wr.pushBlock)
 	wr.reset()
 }
 
-func (wr *writeRequest) mustWriteBlock(block []byte) bool {
-	wr.fq.MustWriteBlockIgnoreDisabledPQ(block)
-	return true
-}
-
-func (wr *writeRequest) tryFlush() bool {
-	wr.wr.Timeseries = wr.tss
-	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
-		return false
-	}
-	wr.reset()
-	return true
-}
-
-func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roundDigits int) {
-	if n := significantFigures; n > 0 {
+func (wr *writeRequest) adjustSampleValues() {
+	samples := wr.samples
+	if n := wr.significantFigures; n > 0 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToSignificantFigures(s.Value, n)
 		}
 	}
-	if n := roundDigits; n < 100 {
+	if n := wr.roundDigits; n < 100 {
 		for i := range samples {
 			s := &samples[i]
 			s.Value = decimal.RoundToDecimalDigits(s.Value, n)
@@ -173,27 +146,21 @@ func adjustSampleValues(samples []prompbmarshal.Sample, significantFigures, roun
 	}
 }
 
-func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
+func (wr *writeRequest) push(src []prompbmarshal.TimeSeries) {
 	tssDst := wr.tss
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
 	maxLabelsPerBlock := 10 * maxSamplesPerBlock
 	for i := range src {
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
+		wr.copyTimeSeries(&tssDst[len(tssDst)-1], &src[i])
 		if len(wr.samples) >= maxSamplesPerBlock || len(wr.labels) >= maxLabelsPerBlock {
 			wr.tss = tssDst
-			if !wr.tryFlush() {
-				return false
-			}
+			wr.flush()
 			tssDst = wr.tss
 		}
-		tsSrc := &src[i]
-		adjustSampleValues(tsSrc.Samples, wr.significantFigures, wr.roundDigits)
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{})
-		wr.copyTimeSeries(&tssDst[len(tssDst)-1], tsSrc)
 	}
-
 	wr.tss = tssDst
-	return true
 }
 
 func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
@@ -221,45 +188,27 @@ func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
 	wr.buf = buf
 }
 
-// marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.
-var marshalConcurrencyCh = make(chan struct{}, cgroup.AvailableCPUs())
-
-func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block []byte) bool, isVMRemoteWrite bool) bool {
+func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byte)) {
 	if len(wr.Timeseries) == 0 {
 		// Nothing to push
-		return true
+		return
 	}
-
-	marshalConcurrencyCh <- struct{}{}
-
 	bb := writeRequestBufPool.Get()
-	bb.B = wr.MarshalProtobuf(bb.B[:0])
+	bb.B = prompbmarshal.MarshalWriteRequest(bb.B[:0], wr)
 	if len(bb.B) <= maxUnpackedBlockSize.IntN() {
-		zb := compressBufPool.Get()
-		if isVMRemoteWrite {
-			zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel)
-		} else {
-			zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
-		}
+		zb := snappyBufPool.Get()
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
-
 		if len(zb.B) <= persistentqueue.MaxBlockSize {
-			zbLen := len(zb.B)
-			ok := tryPushBlock(zb.B)
-			compressBufPool.Put(zb)
-			if ok {
-				blockSizeRows.Update(float64(len(wr.Timeseries)))
-				blockSizeBytes.Update(float64(zbLen))
-			}
-			return ok
+			pushBlock(zb.B)
+			blockSizeRows.Update(float64(len(wr.Timeseries)))
+			blockSizeBytes.Update(float64(len(zb.B)))
+			snappyBufPool.Put(zb)
+			return
 		}
-		compressBufPool.Put(zb)
+		snappyBufPool.Put(zb)
 	} else {
 		writeRequestBufPool.Put(bb)
-
-		<-marshalConcurrencyCh
 	}
 
 	// Too big block. Recursively split it into smaller parts if possible.
@@ -268,36 +217,23 @@ func tryPushWriteRequest(wr *prompbmarshal.WriteRequest, tryPushBlock func(block
 		samples := wr.Timeseries[0].Samples
 		if len(samples) == 1 {
 			logger.Warnf("dropping a sample for metric with too long labels exceeding -remoteWrite.maxBlockSize=%d bytes", maxUnpackedBlockSize.N)
-			return true
+			return
 		}
 		n := len(samples) / 2
 		wr.Timeseries[0].Samples = samples[:n]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			return false
-		}
+		pushWriteRequest(wr, pushBlock)
 		wr.Timeseries[0].Samples = samples[n:]
-		if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-			wr.Timeseries[0].Samples = samples
-			return false
-		}
+		pushWriteRequest(wr, pushBlock)
 		wr.Timeseries[0].Samples = samples
-		return true
+		return
 	}
 	timeseries := wr.Timeseries
 	n := len(timeseries) / 2
 	wr.Timeseries = timeseries[:n]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
+	pushWriteRequest(wr, pushBlock)
 	wr.Timeseries = timeseries[n:]
-	if !tryPushWriteRequest(wr, tryPushBlock, isVMRemoteWrite) {
-		wr.Timeseries = timeseries
-		return false
-	}
+	pushWriteRequest(wr, pushBlock)
 	wr.Timeseries = timeseries
-	return true
 }
 
 var (
@@ -305,7 +241,5 @@ var (
 	blockSizeRows  = metrics.NewHistogram(`vmagent_remotewrite_block_size_rows`)
 )
 
-var (
-	writeRequestBufPool bytesutil.ByteBufferPool
-	compressBufPool     bytesutil.ByteBufferPool
-)
+var writeRequestBufPool bytesutil.ByteBufferPool
+var snappyBufPool bytesutil.ByteBufferPool

app/vmagent/remotewrite/pendingseries_test.go

@@ -2,51 +2,40 @@ package remotewrite
 
 import (
 	"fmt"
-	"math"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/golang/snappy"
 )
 
 func TestPushWriteRequest(t *testing.T) {
-	rowsCounts := []int{1, 10, 100, 1e3, 1e4}
-	expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
-	expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
-	for i, rowsCount := range rowsCounts {
-		expectedBlockLenProm := expectedBlockLensProm[i]
-		expectedBlockLenVM := expectedBlockLensVM[i]
+	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
 		t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
-			testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
+			testPushWriteRequest(t, rowsCount)
 		})
 	}
 }
 
-func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
-	f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
-		t.Helper()
-		wr := newTestWriteRequest(rowsCount, 20)
-		pushBlockLen := 0
-		pushBlock := func(block []byte) bool {
-			if pushBlockLen > 0 {
-				panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
-			}
-			pushBlockLen = len(block)
-			return true
-		}
-		if !tryPushWriteRequest(wr, pushBlock, isVMRemoteWrite) {
-			t.Fatalf("cannot push data to remote storage")
-		}
-		if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
-			t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
-				rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
+func testPushWriteRequest(t *testing.T, rowsCount int) {
+	wr := newTestWriteRequest(rowsCount, 10)
+	pushBlockLen := 0
+	pushBlock := func(block []byte) {
+		if pushBlockLen > 0 {
+			panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
 		}
+		pushBlockLen = len(block)
+	}
+	pushWriteRequest(wr, pushBlock)
+	b := prompbmarshal.MarshalWriteRequest(nil, wr)
+	zb := snappy.Encode(nil, b)
+	maxPushBlockLen := len(zb)
+	minPushBlockLen := maxPushBlockLen / 2
+	if pushBlockLen < minPushBlockLen {
+		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be at least %d bytes", pushBlockLen, minPushBlockLen)
+	}
+	if pushBlockLen > maxPushBlockLen {
+		t.Fatalf("unexpected block len after pushWriteRequest; got %d bytes; must be smaller or equal to %d bytes", pushBlockLen, maxPushBlockLen)
 	}
-
-	// Check Prometheus remote write
-	f(false, expectedBlockLenProm, 3)
-
-	// Check VictoriaMetrics remote write
-	f(true, expectedBlockLenVM, 15)
 }
 
 func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {

app/vmagent/remotewrite/pendingseries_timing_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/golang/snappy"
 	"github.com/klauspost/compress/s2"
 )
@@ -21,7 +22,7 @@ func benchmarkCompressWriteRequest(b *testing.B, compressFunc func(dst, src []by
 	for _, rowsCount := range []int{1, 10, 100, 1e3, 1e4} {
 		b.Run(fmt.Sprintf("rows_%d", rowsCount), func(b *testing.B) {
 			wr := newTestWriteRequest(rowsCount, 10)
-			data := wr.MarshalProtobuf(nil)
+			data := prompbmarshal.MarshalWriteRequest(nil, wr)
 			b.ReportAllocs()
 			b.SetBytes(int64(rowsCount))
 			b.RunParallel(func(pb *testing.PB) {

app/vmagent/remotewrite/relabel.go

@@ -3,7 +3,6 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -19,10 +18,10 @@ var (
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
 		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
 		"The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent/#relabeling")
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
 	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
 		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/vmagent/#relabeling")
+		"See https://docs.victoriametrics.com/vmagent.html#relabeling")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -46,11 +45,11 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		}
 		rcs.global = global
 	}
-	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
-		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
+	if len(*relabelConfigPaths) > (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)) {
+		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url or -remoteWrite.multitenantURL args: %d",
+			len(*relabelConfigPaths), (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	}
-	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, (len(*remoteWriteURLs) + len(*remoteWriteMultitenantURLs)))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
 			// Skip empty relabel config.
@@ -88,27 +87,45 @@ func initLabelsGlobal() {
 	}
 }
 
-func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
-	if pcs.Len() == 0 && !*usePromCompatibleNaming {
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
+	if len(extraLabels) == 0 && pcs.Len() == 0 && !*usePromCompatibleNaming {
 		// Nothing to change.
 		return tss
 	}
-	rctx.reset()
 	tssDst := tss[:0]
 	labels := rctx.labels[:0]
 	for i := range tss {
 		ts := &tss[i]
 		labelsLen := len(labels)
 		labels = append(labels, ts.Labels...)
+		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
+		for j := range extraLabels {
+			extraLabel := &extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, *extraLabel)
+			}
+		}
+		if *usePromCompatibleNaming {
+			// Replace unsupported Prometheus chars in label names and metric names with underscores.
+			tmpLabels := labels[labelsLen:]
+			for j := range tmpLabels {
+				label := &tmpLabels[j]
+				if label.Name == "__name__" {
+					label.Value = promrelabel.SanitizeMetricName(label.Value)
+				} else {
+					label.Name = promrelabel.SanitizeLabelName(label.Name)
+				}
+			}
+		}
 		labels = pcs.Apply(labels, labelsLen)
 		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue
 		}
-		if *usePromCompatibleNaming {
-			fixPromCompatibleNaming(labels[labelsLen:])
-		}
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: ts.Samples,
@@ -118,58 +135,6 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *pro
 	return tssDst
 }
 
-func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) {
-	if len(extraLabels) == 0 {
-		return
-	}
-	rctx.reset()
-	labels := rctx.labels[:0]
-	for i := range tss {
-		ts := &tss[i]
-		labelsLen := len(labels)
-		labels = append(labels, ts.Labels...)
-		for j := range extraLabels {
-			extraLabel := extraLabels[j]
-			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
-			if tmp != nil {
-				tmp.Value = extraLabel.Value
-			} else {
-				labels = append(labels, extraLabel)
-			}
-		}
-		ts.Labels = labels[labelsLen:]
-	}
-	rctx.labels = labels
-}
-
-func (rctx *relabelCtx) tenantToLabels(tss []prompbmarshal.TimeSeries, accountID, projectID uint32) {
-	rctx.reset()
-	accountIDStr := strconv.FormatUint(uint64(accountID), 10)
-	projectIDStr := strconv.FormatUint(uint64(projectID), 10)
-	labels := rctx.labels[:0]
-	for i := range tss {
-		ts := &tss[i]
-		labelsLen := len(labels)
-		for _, label := range ts.Labels {
-			labelName := label.Name
-			if labelName == "vm_account_id" || labelName == "vm_project_id" {
-				continue
-			}
-			labels = append(labels, label)
-		}
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "vm_account_id",
-			Value: accountIDStr,
-		})
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "vm_project_id",
-			Value: projectIDStr,
-		})
-		ts.Labels = labels[labelsLen:]
-	}
-	rctx.labels = labels
-}
-
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
 	labels []prompbmarshal.Label
@@ -181,7 +146,7 @@ func (rctx *relabelCtx) reset() {
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		return &relabelCtx{}
 	},
 }
@@ -191,18 +156,6 @@ func getRelabelCtx() *relabelCtx {
 }
 
 func putRelabelCtx(rctx *relabelCtx) {
-	rctx.reset()
+	rctx.labels = rctx.labels[:0]
 	relabelCtxPool.Put(rctx)
 }
-
-func fixPromCompatibleNaming(labels []prompbmarshal.Label) {
-	// Replace unsupported Prometheus chars in label names and metric names with underscores.
-	for i := range labels {
-		label := &labels[i]
-		if label.Name == "__name__" {
-			label.Value = promrelabel.SanitizeMetricName(label.Value)
-		} else {
-			label.Name = promrelabel.SanitizeLabelName(label.Name)
-		}
-	}
-}

app/vmagent/remotewrite/relabel_test.go

@@ -10,16 +10,18 @@ import (
 )
 
 func TestApplyRelabeling(t *testing.T) {
-	f := func(pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
+	f := func(extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
 		rctx := &relabelCtx{}
 		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		gotTss := rctx.applyRelabeling(tss, pcs)
+		gotTss := rctx.applyRelabeling(tss, extraLabels, pcs)
 		if !reflect.DeepEqual(gotTss, expTss) {
 			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
 		}
 	}
 
-	f(nil, "up", "up")
+	f(nil, nil, "up", "up")
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, "up", `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, `up{foo="baz"}`, `up{foo="bar"}`)
 
 	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
 - target_label: "foo"
@@ -30,33 +32,11 @@ func TestApplyRelabeling(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	f(pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
+	f(nil, pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
 
 	oldVal := *usePromCompatibleNaming
 	*usePromCompatibleNaming = true
-	f(nil, `foo.bar`, `foo_bar`)
-	*usePromCompatibleNaming = oldVal
-}
-
-func TestAppendExtraLabels(t *testing.T) {
-	f := func(extraLabels []prompbmarshal.Label, sTss, sExpTss string) {
-		t.Helper()
-		rctx := &relabelCtx{}
-		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		rctx.appendExtraLabels(tss, extraLabels)
-		if !reflect.DeepEqual(tss, expTss) {
-			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, tss)
-		}
-	}
-
-	f(nil, "up", "up")
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, "up", `up{foo="bar"}`)
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, `up{foo="baz"}`, `up{foo="bar"}`)
-	f([]prompbmarshal.Label{{Name: "baz", Value: "qux"}}, `up{foo="baz"}`, `up{foo="baz",baz="qux"}`)
-
-	oldVal := *usePromCompatibleNaming
-	*usePromCompatibleNaming = true
-	f([]prompbmarshal.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo.bar="baz"}`)
+	f(nil, nil, `foo.bar`, `foo_bar`)
 	*usePromCompatibleNaming = oldVal
 }
 

app/vmagent/remotewrite/remotewrite_test.go

@@ -1,169 +0,0 @@
-package remotewrite
-
-import (
-	"fmt"
-	"math"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-func TestGetLabelsHash_Distribution(t *testing.T) {
-	f := func(bucketsCount int) {
-		t.Helper()
-
-		// Distribute itemsCount hashes returned by getLabelsHash() across bucketsCount buckets.
-		itemsCount := 1_000 * bucketsCount
-		m := make([]int, bucketsCount)
-		var labels []prompbmarshal.Label
-		for i := 0; i < itemsCount; i++ {
-			labels = append(labels[:0], prompbmarshal.Label{
-				Name:  "__name__",
-				Value: fmt.Sprintf("some_name_%d", i),
-			})
-			for j := 0; j < 10; j++ {
-				labels = append(labels, prompbmarshal.Label{
-					Name:  fmt.Sprintf("label_%d", j),
-					Value: fmt.Sprintf("value_%d_%d", i, j),
-				})
-			}
-			h := getLabelsHash(labels)
-			m[h%uint64(bucketsCount)]++
-		}
-
-		// Verify that the distribution is even
-		expectedItemsPerBucket := itemsCount / bucketsCount
-		for _, n := range m {
-			if math.Abs(1-float64(n)/float64(expectedItemsPerBucket)) > 0.04 {
-				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want around %d", bucketsCount, n, expectedItemsPerBucket)
-			}
-		}
-	}
-
-	f(2)
-	f(3)
-	f(4)
-	f(5)
-	f(10)
-}
-
-func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
-		t.Helper()
-		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
-		if err != nil {
-			t.Fatalf("cannot load relabel configs: %s", err)
-		}
-		rcs := &relabelConfigs{
-			perURL: []*promrelabel.ParsedConfigs{
-				perURLRelabel,
-			},
-		}
-		allRelabelConfigs.Store(rcs)
-
-		pss := make([]*pendingSeries, 1)
-		pss[0] = newPendingSeries(nil, true, 0, 100)
-		rwctx := &remoteWriteCtx{
-			idx:                    0,
-			streamAggrKeepInput:    keepInput,
-			streamAggrDropInput:    dropInput,
-			pss:                    pss,
-			rowsPushedAfterRelabel: metrics.GetOrCreateCounter(`foo`),
-			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
-		}
-		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
-		}
-
-		if streamAggrConfig != "" {
-			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
-			if err != nil {
-				t.Fatalf("cannot load streamaggr configs: %s", err)
-			}
-			defer sas.MustStop()
-			rwctx.sas.Store(sas)
-		}
-
-		offsetMsecs := time.Now().UnixMilli()
-		inputTss := prompbmarshal.MustParsePromMetrics(input, offsetMsecs)
-		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
-
-		// copy inputTss to make sure it is not mutated during TryPush call
-		copy(expectedTss, inputTss)
-		if !rwctx.TryPush(inputTss, false) {
-			t.Fatalf("cannot push samples to rwctx")
-		}
-
-		if !reflect.DeepEqual(expectedTss, inputTss) {
-			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
-		}
-	}
-
-	f(`
-- interval: 1m
-  outputs: [sum_samples]
-- interval: 2m
-  outputs: [count_series]
-`, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, 0, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-	f(``, ``, time.Hour, false, false, `
-metric{env="dev"} 10
-metric{env="foo"} 20
-metric{env="dev"} 15
-metric{env="foo"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, true, false, `
-metric{env="test"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, false, true, `
-metric{env="foo"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, time.Hour, true, true, `
-metric{env="dev"} 10
-metric{env="test"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-}

app/vmagent/remotewrite/statconn.go

@@ -0,0 +1,92 @@
+package remotewrite
+
+import (
+	"context"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func getStdDialer() *net.Dialer {
+	stdDialerOnce.Do(func() {
+		stdDialer = &net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: netutil.TCP6Enabled(),
+		}
+	})
+	return stdDialer
+}
+
+var (
+	stdDialer     *net.Dialer
+	stdDialerOnce sync.Once
+)
+
+func statDial(ctx context.Context, networkUnused, addr string) (conn net.Conn, err error) {
+	network := netutil.GetTCPNetwork()
+	d := getStdDialer()
+	conn, err = d.DialContext(ctx, network, addr)
+	dialsTotal.Inc()
+	if err != nil {
+		dialErrors.Inc()
+		return nil, err
+	}
+	conns.Inc()
+	sc := &statConn{
+		Conn: conn,
+	}
+	return sc, nil
+}
+
+var (
+	dialsTotal = metrics.NewCounter(`vmagent_remotewrite_dials_total`)
+	dialErrors = metrics.NewCounter(`vmagent_remotewrite_dial_errors_total`)
+	conns      = metrics.NewCounter(`vmagent_remotewrite_conns`)
+)
+
+type statConn struct {
+	closed uint64
+	net.Conn
+}
+
+func (sc *statConn) Read(p []byte) (int, error) {
+	n, err := sc.Conn.Read(p)
+	connReadsTotal.Inc()
+	if err != nil {
+		connReadErrors.Inc()
+	}
+	connBytesRead.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Write(p []byte) (int, error) {
+	n, err := sc.Conn.Write(p)
+	connWritesTotal.Inc()
+	if err != nil {
+		connWriteErrors.Inc()
+	}
+	connBytesWritten.Add(n)
+	return n, err
+}
+
+func (sc *statConn) Close() error {
+	err := sc.Conn.Close()
+	if atomic.AddUint64(&sc.closed, 1) == 1 {
+		conns.Dec()
+	}
+	return err
+}
+
+var (
+	connReadsTotal   = metrics.NewCounter(`vmagent_remotewrite_conn_reads_total`)
+	connWritesTotal  = metrics.NewCounter(`vmagent_remotewrite_conn_writes_total`)
+	connReadErrors   = metrics.NewCounter(`vmagent_remotewrite_conn_read_errors_total`)
+	connWriteErrors  = metrics.NewCounter(`vmagent_remotewrite_conn_write_errors_total`)
+	connBytesRead    = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_read_total`)
+	connBytesWritten = metrics.NewCounter(`vmagent_remotewrite_conn_bytes_written_total`)
+)

app/vmagent/remotewrite/streamaggr.go

@@ -1,240 +0,0 @@
-package remotewrite
-
-import (
-	"flag"
-	"fmt"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	// Global config
-	streamAggrGlobalConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
-		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
-	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation "+
-		"with -streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrGlobalDedupInterval = flagutil.NewDuration("streamAggr.dedupInterval", "0s", "Input samples are de-duplicated with this interval on "+
-		"aggregator before optional aggregation with -streamAggr.config . "+
-		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
-	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
-		"current aggregation interval for aggregator. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrGlobalIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start for "+
-		"aggregator. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from "+
-		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
-	streamAggrGlobalDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples for aggregator "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
-
-	// Per URL config
-	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
-		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
-	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
-	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
-		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
-		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
-		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
-	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
-		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
-)
-
-// CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
-func CheckStreamAggrConfigs() error {
-	// Check global config
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		return err
-	}
-	sas.MustStop()
-
-	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
-		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
-	}
-
-	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-	for idx := range *streamAggrConfig {
-		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
-		if err != nil {
-			return err
-		}
-		sas.MustStop()
-	}
-	return nil
-}
-
-func reloadStreamAggrConfigs() {
-	reloadStreamAggrConfigGlobal()
-	for _, rwctx := range rwctxsGlobal {
-		rwctx.reloadStreamAggrConfig()
-	}
-}
-
-func reloadStreamAggrConfigGlobal() {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
-		return
-	}
-
-	sas := sasGlobal.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := sasGlobal.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
-}
-
-func initStreamAggrConfigGlobal() {
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		sasGlobal.Store(sas)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrGlobalDedupInterval.Duration()
-		if dedupInterval > 0 {
-			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
-		}
-	}
-}
-
-func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
-	idx := rwctx.idx
-
-	sas, err := rwctx.newStreamAggrConfig()
-	if err != nil {
-		logger.Fatalf("cannot initialize stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-		if dedupInterval > 0 {
-			alias := fmt.Sprintf("dedup-%d", idx+1)
-			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
-		}
-	}
-}
-
-func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
-	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if path == "" {
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := rwctx.newStreamAggrConfig()
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
-		return
-	}
-
-	sas := rwctx.sas.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := rwctx.sas.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
-}
-
-func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return nil, nil
-	}
-
-	opts := &streamaggr.Options{
-		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
-		DropInputLabels:      *streamAggrGlobalDropInputLabels,
-		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
-		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
-	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
-	}
-	return sas, nil
-}
-
-func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
-	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
-}
-
-func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path := streamAggrConfig.GetOptionalArg(idx)
-	if path == "" {
-		return nil, nil
-	}
-
-	alias := fmt.Sprintf("%d:secret-url", idx+1)
-	if *showRemoteWriteURL {
-		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
-	}
-	opts := &streamaggr.Options{
-		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
-		DropInputLabels:      *streamAggrDropInputLabels,
-		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
-	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
-	}
-	return sas, nil
-}