app/vmselect/netstorage: fix a typo, which leads to incorrect query results in VictoriaMetrics cluster

The typo has been introduced in the commit 1a254ea20c

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3067

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,46 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+It would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
+to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+and verify whether the bug is reproducible there.
+It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/#troubleshooting).
+
+**To Reproduce**
+Steps to reproduce the behavior.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Logs**
+Check if any warnings or errors were logged by VictoriaMetrics components
+or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+For VictoriaMetrics health-state issues please provide full-length screenshots 
+of Grafana dashboards if possible:
+* [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/dashboards/10229)
+* [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
+
+See how to setup monitoring here:
+* [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
+* [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
+
+**Version**
+The line returned when passing `--version` command line flag to the binary. For example:
+```
+$ ./victoria-metrics-prod --version
+victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
+```
+
+**Used command-line flags**
+Please provide the command-line flags used for running VictoriaMetrics and its components. 

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,86 +0,0 @@
-name: Bug report
-description: Create a report to help us improve
-labels: [bug]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
-        to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-        and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html) first.
-  - type: textarea
-    id: describe-the-bug
-    attributes:
-      label: Describe the bug
-      description: |
-        A clear and concise description of what the bug is.
-      placeholder: |
-        When I do `A` VictoriaMetrics does `B`. I expect it to do `C`.
-    validations:
-      required: true
-  - type: textarea
-    id: to-reproduce
-    attributes:
-      label: To Reproduce
-      description: |
-        Steps to reproduce the behavior.
-        If reproducing an issue requires some specific configuration file, please paste it here.
-      placeholder: |
-        Steps to reproduce the behavior.
-    validations:
-      required: true
-  - type: textarea
-    id: version
-    attributes:
-      label: Version
-      description: |
-        The line returned when passing `--version` command line flag to the binary. For example:
-        ```
-        $ ./victoria-metrics-prod --version
-        victoria-metrics-20190730-121249-heads-single-node-0-g671d9e55
-        ```
-    validations:
-      required: true
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs
-      description: |
-        Check if any warnings or errors were logged by VictoriaMetrics components
-        or components in communication with VictoriaMetrics (e.g. Prometheus, Grafana).
-    validations:
-      required: false
-  - type: textarea
-    id: screenshots
-    attributes:
-      label: Screenshots
-      description: |
-        If applicable, add screenshots to help explain your problem.
-  
-        For VictoriaMetrics health-state issues please provide full-length screenshots
-        of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
-        
-        See how to setup monitoring here:
-          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring)
-    validations:
-      required: false
-  - type: textarea
-    id: flags
-    attributes:
-      label: Used command-line flags
-      description: |
-        Please provide the command-line flags used for running VictoriaMetrics and its components.
-    validations:
-      required: false
-  - type: textarea
-    id: additional-info
-    attributes:
-      label: Additional information
-      placeholder: |
-        Additional information that doesn't fit elsewhere
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/configuration.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Ask on Slack
-    url: https://slack.victoriametrics.com/
-    about: You can ask for help here!

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,43 +0,0 @@
-name: Feature request
-description: Suggest an idea for this project
-labels: [enhancement]
-body:
-  - type: textarea
-    id: describe-the-problem
-    attributes:
-      label: Is your feature request related to a problem? Please describe
-      description: |
-        A clear and concise description of what the problem is.
-      placeholder: |
-        Ex. I'm always frustrated when [...]
-    validations:
-      required: false
-  - type: textarea
-    id: describe-the-solution
-    attributes:
-      label: Describe the solution you'd like
-      description: |
-        A clear and concise description of what you want to happen.
-    validations:
-      required: true
-  - type: textarea
-    id: alternative-solutions
-    attributes:
-      label: Describe alternatives you've considered
-      description: |
-        A clear and concise description of any alternative solutions or features you've considered.
-      placeholder: |
-        I have tried to do `A`, but that doesn't solve a problem completely.
-        I have tried to do `A` and `B`, but implementing this would be better.
-    validations:
-      required: false
-  - type: textarea
-    id: feature-additional-info
-    attributes:
-      label: Additional information
-      description: |
-        Additional information which you consider helpful for implementing this feature.
-      placeholder: |
-        Add any other context or screenshots about the feature request here.
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/question.yml

@@ -1,32 +0,0 @@
-name: Question
-description: Ask a question regarding VictoriaMetrics or its components
-labels: [question]
-body:
-  - type: textarea
-    id: describe-the-component
-    attributes:
-      label: Is your question request related to a specific component?
-      placeholder: |
-        VictoriaMetrics, vmagent, vmalert, vmui, etc...
-    validations:
-      required: false
-  - type: textarea
-    id: describe-the-question
-    attributes:
-      label: Describe the question in detail
-      description: |
-        A clear and concise description of the issue and the question.
-    validations:
-      required: true
-  - type: checkboxes
-    id: troubleshooting
-    attributes:
-      label: Troubleshooting docs
-      description: I am familiar with the following troubleshooting docs
-      options:
-        - label: General - https://docs.victoriametrics.com/Troubleshooting.html
-          required: false
-        - label: vmagent - https://docs.victoriametrics.com/vmagent.html#troubleshooting
-          required: false
-        - label: vmalert - https://docs.victoriametrics.com/vmalert.html#troubleshooting
-          required: false

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -1,35 +0,0 @@
-### Describe Your Changes
-
-Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications.
-
-### Checklist
-
-The following checks are mandatory:
-
-- [ ] I have read the [Contributing Guidelines](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/CONTRIBUTING.md)
-- [ ] All commits are signed and include `Signed-off-by` line. Use `git commit -s` to include `Signed-off-by` your commits. See this [doc](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) about how to sign your commits.
-- [ ] Tests are passing locally. Use `make test` to run all tests locally.
-- [ ] Linting is passing locally. Use `make check-all` to run all linters locally.
-
-Further checks are optional for External Contributions:
-
-- [ ] Include a link to the GitHub issue in the commit message, if issue exists.
-- [ ] Mention the change in the [Changelog](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/CHANGELOG.md). Explain what has changed and why. If there is a related issue or documentation change - link them as well.
-
-  Tips for writing a good changelog message::
-
-    * Write a human-readable changelog message that describes the problem and solution.
-    * Include a link to the issue or pull request in your changelog message.
-    * Use specific language identifying the fix, such as an error message, metric name, or flag name.
-    * Provide a link to the relevant documentation for any new features you add or modify.
-
-- [ ] After your pull request is merged, please add a message to the issue with instructions for how to test the fix or try the feature you added. Here is an [example](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048#issuecomment-1546453726)
-- [ ] Do not close the original issue before the change is released. Please note, in some cases Github can automatically close the issue once PR is merged. Re-open the issue in such case.
-- [ ] If the change somehow affects public interfaces (a new flag was added or updated, or some behavior has changed) - add the corresponding change to documentation.
-
-
-Examples of good changelog messages:
-
-1. FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol) when [sending / receiving data to / from Kafka](https://docs.victoriametrics.com/vmagent.html#kafka-integration). This protocol allows saving egress network bandwidth costs when sending data from `vmagent` to `Kafka` located in another datacenter or availability zone. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1225).
-
-2. BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation.html): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmgent.html) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).

.github/dependabot.yml

@@ -8,12 +8,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    open-pull-requests-limit: 0
   - package-ecosystem: "bundler"
     directory: "/docs"
     schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 0
+      interval: "daily"
   - package-ecosystem: "gomod"
     directory: "/app/vmui/packages/vmui/web"
     schedule:

.github/workflows/check-licenses.yml

@@ -14,25 +14,13 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
+      - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.18
+        id: go
       - name: Code checkout
         uses: actions/checkout@master
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
-
       - name: Check License
-        run: make check-licenses
+        run: |
+          make check-licenses

.github/workflows/codeql-analysis-js.yml

@@ -1,46 +0,0 @@
-name: "CodeQL - JS"
-
-on:
-  push:
-    branches: [master, cluster]
-    paths:
-      - "**.js"
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [master, cluster]
-    paths:
-      - "**.js"
-  schedule:
-    - cron: "30 18 * * 2"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-  
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["javascript"]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: ${{ matrix.language }}
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: "javascript"

.github/workflows/codeql-analysis.yml

@@ -13,26 +13,12 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
+    branches: [ master, cluster ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [master, cluster]
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "**.txt"
-      - "**.js"
+    branches: [ master, cluster ]
   schedule:
-    - cron: "30 18 * * 2"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+    - cron: '30 18 * * 2'
 
 jobs:
   analyze:
@@ -46,58 +32,39 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ["go"]
+        language: [ 'go', 'javascript' ]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Learn more about CodeQL language support at https://git.io/codeql-language-support
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+    - name: Checkout repository
+      uses: actions/checkout@v3
 
-      - name: Set up Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-        if: ${{ matrix.language == 'go' }}
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
-        if: ${{ matrix.language == 'go' }}
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
 
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
 
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
 
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
+    #- run: |
+    #   make bootstrap
+    #   make release
 
-      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-      #    and modify them (or add more) to build your code if your project
-      #    uses a compiled language
-
-      #- run: |
-      #   make bootstrap
-      #   make release
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -1,120 +1,47 @@
 name: main
 on:
   push:
-    branches:
-      - master
-      - cluster
     paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - 'docs/**'
+      - '**.md'
   pull_request:
-    branches:
-      - master
-      - cluster
     paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
+      - 'docs/**'
+      - '**.md'
 permissions:
   contents: read
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 jobs:
-  lint:
-    name: lint
+  build:
+    name: Build
     runs-on: ubuntu-latest
     steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
       - name: Setup Go
+        uses: actions/setup-go@main
+        with:
+          go-version: 1.18
         id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-all-
-
-      - name: Run check-all
+      - name: Code checkout
+        uses: actions/checkout@master
+      - name: Dependencies
         run: |
+          make install-golint
+          make install-errcheck
+          make install-golangci-lint
+      - name: Build
+        env:
+          GO111MODULE: on
+        run: |
+          export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14
           make check-all
           git diff --exit-code
-
-  build:
-    needs: lint
-    name: build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-crossbuild-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-crossbuild-
-
-      - name: Build
-        run: make crossbuild
-
-  test:
-    needs: lint
-    strategy:
-      matrix:
-        scenario: ["test-full", "test-pure", "test-full-386"]
-    name: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-          cache: false
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            ~/go/bin
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: run tests
-        run: make ${{ matrix.scenario}}
-
+          make test-full
+          make test-pure
+          make test-full-386
+          make victoria-metrics-crossbuild
+          make vmuitils-crossbuild
       - name: Publish coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt
+

.github/workflows/sync-docs.yml

@@ -1,51 +0,0 @@
-name: publish-docs
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'docs/**'
-  workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-        with:
-          path: main
-      - name: Checkout private code
-        uses: actions/checkout@v4
-        with:
-          repository: VictoriaMetrics/vmdocs
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: docs
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v5
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-          workdir: docs
-      - name: Set short git commit SHA
-        id: vars
-        run: |
-          calculatedSha=$(git rev-parse --short ${{ github.sha }})
-          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
-        working-directory: main
-      - name: update code and commit
-        run: |
-          rm -rf content
-          cp -r ../main/docs content
-          make clean-after-copy
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          git add .
-          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
-          git push
-        working-directory: docs

.github/workflows/wiki.yml

@@ -0,0 +1,33 @@
+name: wiki
+on:
+  push:
+    paths:
+      - 'docs/*'
+    branches:
+      - master
+permissions:
+  contents: read
+
+jobs:
+  build:
+    permissions:
+      contents: write  # for Git to git push
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: publish
+        shell: bash
+        env:
+          TOKEN: ${{secrets.CI_TOKEN}}
+        run: |
+          git clone https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git wiki
+          cp -r docs/* wiki
+          cd wiki
+          git config --local user.email "info@victoriametrics.com"
+          git config --local user.name "Vika"
+          git add .
+          git commit -m "update wiki pages"
+          remote_repo="https://vika:${TOKEN}@github.com/VictoriaMetrics/VictoriaMetrics.wiki.git"
+          git push "${remote_repo}"
+          cd ..
+          rm -rf wiki

.gitignore

@@ -8,7 +8,6 @@
 *.test
 *.swp
 /gocache-for-docker
-/victoria-logs-data
 /victoria-metrics-data
 /vmagent-remotewrite-data
 /vmstorage-data
@@ -20,6 +19,4 @@
 .DS_store
 Gemfile.lock
 /_site
-_site
-*.tmp
-/docs/.jekyll-metadata
+_site

.golangci.yml

@@ -1,19 +0,0 @@
-run:
-  timeout: 2m
-
-linters:
-  enable:
-  - revive
-
-issues:
-  exclude-rules:
-  - linters:
-    - staticcheck
-    text: "SA(4003|1019|5011):"
-  include:
-  - EXC0012
-  - EXC0014
-
-linters-settings:
-  errcheck:
-    exclude: ./errcheck_excludes.txt

.wwhrd.yml

@@ -3,4 +3,3 @@ allowlist:
   - MIT
   - BSD-3-Clause
   - BSD-2-Clause
-  - ISC

CONTRIBUTING.md

@@ -14,8 +14,3 @@ We are open to third-party pull requests provided they follow [KISS design princ
 - Avoid automated decisions, which may hurt cluster availability, consistency or performance.
 
 Adhering `KISS` principle simplifies the resulting code and architecture, so it can be reviewed, understood and verified by many people.
-
-Before sending a pull request please check the following:
-- [ ] All commits are signed and include `Signed-off-by` line. Use `git commit -s` to include `Signed-off-by` your commits. See this [doc](https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work) about how to sign your commits.
-- [ ] Tests are passing locally. Use `make test` to run all tests locally.
-- [ ] Linting is passing locally. Use `make check-all` to run all linters locally.

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2024 VictoriaMetrics, Inc.
+   Copyright 2019-2022 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,11 +1,8 @@
 PKG_PREFIX := github.com/VictoriaMetrics/VictoriaMetrics
 
-MAKE_CONCURRENCY ?= $(shell getconf _NPROCESSORS_ONLN)
-MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
-	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
-LATEST_TAG ?= latest
+	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -c 10-17)))
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -17,18 +14,14 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include docs/Makefile
 include deployment/*/Makefile
-include dashboards/Makefile
 include snap/local/Makefile
 include package/release/Makefile
 
 all: \
 	victoria-metrics-prod \
-	victoria-logs-prod \
 	vmagent-prod \
 	vmalert-prod \
-	vmalert-tool-prod \
 	vmauth-prod \
 	vmbackup-prod \
 	vmrestore-prod \
@@ -37,11 +30,10 @@ all: \
 clean:
 	rm -rf bin/*
 
-publish: \
+publish: docker-scan \
 	publish-victoria-metrics \
 	publish-vmagent \
 	publish-vmalert \
-	publish-vmalert-tool \
 	publish-vmauth \
 	publish-vmbackup \
 	publish-vmrestore \
@@ -49,10 +41,8 @@ publish: \
 
 package: \
 	package-victoria-metrics \
-	package-victoria-logs \
 	package-vmagent \
 	package-vmalert \
-	package-vmalert-tool \
 	package-vmauth \
 	package-vmbackup \
 	package-vmrestore \
@@ -61,7 +51,6 @@ package: \
 vmutils: \
 	vmagent \
 	vmalert \
-	vmalert-tool \
 	vmauth \
 	vmbackup \
 	vmrestore \
@@ -70,7 +59,6 @@ vmutils: \
 vmutils-pure: \
 	vmagent-pure \
 	vmalert-pure \
-	vmalert-tool-pure \
 	vmauth-pure \
 	vmbackup-pure \
 	vmrestore-pure \
@@ -79,7 +67,6 @@ vmutils-pure: \
 vmutils-linux-amd64: \
 	vmagent-linux-amd64 \
 	vmalert-linux-amd64 \
-	vmalert-tool-linux-amd64 \
 	vmauth-linux-amd64 \
 	vmbackup-linux-amd64 \
 	vmrestore-linux-amd64 \
@@ -88,7 +75,6 @@ vmutils-linux-amd64: \
 vmutils-linux-arm64: \
 	vmagent-linux-arm64 \
 	vmalert-linux-arm64 \
-	vmalert-tool-linux-arm64 \
 	vmauth-linux-arm64 \
 	vmbackup-linux-arm64 \
 	vmrestore-linux-arm64 \
@@ -97,7 +83,6 @@ vmutils-linux-arm64: \
 vmutils-linux-arm: \
 	vmagent-linux-arm \
 	vmalert-linux-arm \
-	vmalert-tool-linux-arm \
 	vmauth-linux-arm \
 	vmbackup-linux-arm \
 	vmrestore-linux-arm \
@@ -106,7 +91,6 @@ vmutils-linux-arm: \
 vmutils-linux-386: \
 	vmagent-linux-386 \
 	vmalert-linux-386 \
-	vmalert-tool-linux-386 \
 	vmauth-linux-386 \
 	vmbackup-linux-386 \
 	vmrestore-linux-386 \
@@ -115,7 +99,6 @@ vmutils-linux-386: \
 vmutils-linux-ppc64le: \
 	vmagent-linux-ppc64le \
 	vmalert-linux-ppc64le \
-	vmalert-tool-linux-ppc64le \
 	vmauth-linux-ppc64le \
 	vmbackup-linux-ppc64le \
 	vmrestore-linux-ppc64le \
@@ -124,7 +107,6 @@ vmutils-linux-ppc64le: \
 vmutils-darwin-amd64: \
 	vmagent-darwin-amd64 \
 	vmalert-darwin-amd64 \
-	vmalert-tool-darwin-amd64 \
 	vmauth-darwin-amd64 \
 	vmbackup-darwin-amd64 \
 	vmrestore-darwin-amd64 \
@@ -133,7 +115,6 @@ vmutils-darwin-amd64: \
 vmutils-darwin-arm64: \
 	vmagent-darwin-arm64 \
 	vmalert-darwin-arm64 \
-	vmalert-tool-darwin-arm64 \
 	vmauth-darwin-arm64 \
 	vmbackup-darwin-arm64 \
 	vmrestore-darwin-arm64 \
@@ -142,7 +123,6 @@ vmutils-darwin-arm64: \
 vmutils-freebsd-amd64: \
 	vmagent-freebsd-amd64 \
 	vmalert-freebsd-amd64 \
-	vmalert-tool-freebsd-amd64 \
 	vmauth-freebsd-amd64 \
 	vmbackup-freebsd-amd64 \
 	vmrestore-freebsd-amd64 \
@@ -151,7 +131,6 @@ vmutils-freebsd-amd64: \
 vmutils-openbsd-amd64: \
 	vmagent-openbsd-amd64 \
 	vmalert-openbsd-amd64 \
-	vmalert-tool-openbsd-amd64 \
 	vmauth-openbsd-amd64 \
 	vmbackup-openbsd-amd64 \
 	vmrestore-openbsd-amd64 \
@@ -160,32 +139,25 @@ vmutils-openbsd-amd64: \
 vmutils-windows-amd64: \
 	vmagent-windows-amd64 \
 	vmalert-windows-amd64 \
-	vmalert-tool-windows-amd64 \
 	vmauth-windows-amd64 \
-	vmbackup-windows-amd64 \
-	vmrestore-windows-amd64 \
 	vmctl-windows-amd64
 
-crossbuild:
-	$(MAKE_PARALLEL) victoria-metrics-crossbuild vmutils-crossbuild
-
 victoria-metrics-crossbuild: \
-	victoria-metrics-linux-386 \
 	victoria-metrics-linux-amd64 \
 	victoria-metrics-linux-arm64 \
 	victoria-metrics-linux-arm \
+	victoria-metrics-linux-386 \
 	victoria-metrics-linux-ppc64le \
 	victoria-metrics-darwin-amd64 \
 	victoria-metrics-darwin-arm64 \
 	victoria-metrics-freebsd-amd64 \
-	victoria-metrics-openbsd-amd64 \
-	victoria-metrics-windows-amd64
+	victoria-metrics-openbsd-amd64
 
 vmutils-crossbuild: \
-	vmutils-linux-386 \
 	vmutils-linux-amd64 \
 	vmutils-linux-arm64 \
 	vmutils-linux-arm \
+	vmutils-linux-386 \
 	vmutils-linux-ppc64le \
 	vmutils-darwin-amd64 \
 	vmutils-darwin-arm64 \
@@ -194,30 +166,26 @@ vmutils-crossbuild: \
 	vmutils-windows-amd64
 
 publish-release:
-	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
+	git checkout $(TAG) && $(MAKE) release publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release publish && \
+		git checkout $(TAG) && git push $(TAG) && git push $(TAG)-cluster && \
+		$(MAKE) github-create-release && \
+		$(MAKE) github-upload-assets
 
-release:
-	$(MAKE_PARALLEL) \
-		release-victoria-metrics \
-		release-vmutils
+release: \
+	release-victoria-metrics \
+	release-vmutils
 
 release-victoria-metrics: \
-	release-victoria-metrics-linux-386 \
 	release-victoria-metrics-linux-amd64 \
 	release-victoria-metrics-linux-arm \
 	release-victoria-metrics-linux-arm64 \
 	release-victoria-metrics-darwin-amd64 \
 	release-victoria-metrics-darwin-arm64 \
 	release-victoria-metrics-freebsd-amd64 \
-	release-victoria-metrics-openbsd-amd64 \
-	release-victoria-metrics-windows-amd64
-
-release-victoria-metrics-linux-386:
-	GOOS=linux GOARCH=386 $(MAKE) release-victoria-metrics-goos-goarch
+	release-victoria-metrics-openbsd-amd64
 
 release-victoria-metrics-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
@@ -240,9 +208,6 @@ release-victoria-metrics-freebsd-amd64:
 release-victoria-metrics-openbsd-amd64:
 	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
-release-victoria-metrics-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-metrics-windows-goarch
-
 release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
 		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
@@ -252,75 +217,7 @@ release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
 	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
 
-release-victoria-metrics-windows-goarch: victoria-metrics-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-metrics-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-metrics-windows-$(GOARCH)-prod.exe \
-			> victoria-metrics-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		victoria-metrics-windows-$(GOARCH)-prod.exe
-
-release-victoria-logs:
-	$(MAKE_PARALLEL) release-victoria-logs-linux-386 \
-		release-victoria-logs-linux-amd64 \
-		release-victoria-logs-linux-arm \
-		release-victoria-logs-linux-arm64 \
-		release-victoria-logs-darwin-amd64 \
-		release-victoria-logs-darwin-arm64 \
-		release-victoria-logs-freebsd-amd64 \
-		release-victoria-logs-openbsd-amd64 \
-		release-victoria-logs-windows-amd64
-
-release-victoria-logs-linux-386:
-	GOOS=linux GOARCH=386 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-linux-amd64:
-	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-linux-arm:
-	GOOS=linux GOARCH=arm $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-linux-arm64:
-	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-darwin-arm64:
-	GOOS=darwin GOARCH=arm64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-freebsd-amd64:
-	GOOS=freebsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-openbsd-amd64:
-	GOOS=openbsd GOARCH=amd64 $(MAKE) release-victoria-logs-goos-goarch
-
-release-victoria-logs-windows-amd64:
-	GOARCH=amd64 $(MAKE) release-victoria-logs-windows-goarch
-
-release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
-	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-logs-$(GOOS)-$(GOARCH)-prod \
-		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
-			victoria-logs-$(GOOS)-$(GOARCH)-prod \
-			| sed s/-$(GOOS)-$(GOARCH)-prod/-prod/ > victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf victoria-logs-$(GOOS)-$(GOARCH)-prod
-
-release-victoria-logs-windows-goarch: victoria-logs-windows-$(GOARCH)-prod
-	cd bin && \
-		zip victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-logs-windows-$(GOARCH)-prod.exe \
-		&& sha256sum victoria-logs-windows-$(GOARCH)-$(PKG_TAG).zip \
-			victoria-logs-windows-$(GOARCH)-prod.exe \
-			> victoria-logs-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
-	cd bin && rm -rf \
-		victoria-logs-windows-$(GOARCH)-prod.exe
-
 release-vmutils: \
-	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
 	release-vmutils-linux-arm64 \
 	release-vmutils-linux-arm \
@@ -330,9 +227,6 @@ release-vmutils: \
 	release-vmutils-openbsd-amd64 \
 	release-vmutils-windows-amd64
 
-release-vmutils-linux-386:
-	GOOS=linux GOARCH=386 $(MAKE) release-vmutils-goos-goarch
-	
 release-vmutils-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
@@ -360,7 +254,6 @@ release-vmutils-windows-amd64:
 release-vmutils-goos-goarch: \
 	vmagent-$(GOOS)-$(GOARCH)-prod \
 	vmalert-$(GOOS)-$(GOARCH)-prod \
-	vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 	vmauth-$(GOOS)-$(GOARCH)-prod \
 	vmbackup-$(GOOS)-$(GOARCH)-prod \
 	vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -369,7 +262,6 @@ release-vmutils-goos-goarch: \
 		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 			vmauth-$(GOOS)-$(GOARCH)-prod \
 			vmbackup-$(GOOS)-$(GOARCH)-prod \
 			vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -377,7 +269,6 @@ release-vmutils-goos-goarch: \
 		&& sha256sum vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
-			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 			vmauth-$(GOOS)-$(GOARCH)-prod \
 			vmbackup-$(GOOS)-$(GOARCH)-prod \
 			vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -386,7 +277,6 @@ release-vmutils-goos-goarch: \
 	cd bin && rm -rf \
 		vmagent-$(GOOS)-$(GOARCH)-prod \
 		vmalert-$(GOOS)-$(GOARCH)-prod \
-		vmalert-tool-$(GOOS)-$(GOARCH)-prod \
 		vmauth-$(GOOS)-$(GOARCH)-prod \
 		vmbackup-$(GOOS)-$(GOARCH)-prod \
 		vmrestore-$(GOOS)-$(GOARCH)-prod \
@@ -395,38 +285,27 @@ release-vmutils-goos-goarch: \
 release-vmutils-windows-goarch: \
 	vmagent-windows-$(GOARCH)-prod \
 	vmalert-windows-$(GOARCH)-prod \
-	vmalert-tool-windows-$(GOARCH)-prod \
 	vmauth-windows-$(GOARCH)-prod \
-	vmbackup-windows-$(GOARCH)-prod \
-	vmrestore-windows-$(GOARCH)-prod \
 	vmctl-windows-$(GOARCH)-prod
 	cd bin && \
 		zip vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 		&& sha256sum vmutils-windows-$(GOARCH)-$(PKG_TAG).zip \
 			vmagent-windows-$(GOARCH)-prod.exe \
 			vmalert-windows-$(GOARCH)-prod.exe \
-			vmalert-tool-windows-$(GOARCH)-prod.exe \
 			vmauth-windows-$(GOARCH)-prod.exe \
-			vmbackup-windows-$(GOARCH)-prod.exe \
-			vmrestore-windows-$(GOARCH)-prod.exe \
 			vmctl-windows-$(GOARCH)-prod.exe \
 			> vmutils-windows-$(GOARCH)-$(PKG_TAG)_checksums.txt
 	cd bin && rm -rf \
 		vmagent-windows-$(GOARCH)-prod.exe \
 		vmalert-windows-$(GOARCH)-prod.exe \
-		vmalert-tool-windows-$(GOARCH)-prod.exe \
 		vmauth-windows-$(GOARCH)-prod.exe \
-		vmbackup-windows-$(GOARCH)-prod.exe \
-		vmrestore-windows-$(GOARCH)-prod.exe \
 		vmctl-windows-$(GOARCH)-prod.exe
 
+
 pprof-cpu:
 	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
 
@@ -438,7 +317,29 @@ vet:
 	go vet ./lib/...
 	go vet ./app/...
 
-check-all: fmt vet golangci-lint govulncheck
+lint: install-golint
+	golint lib/...
+	golint app/...
+
+install-golint:
+	which golint || GO111MODULE=off go get golang.org/x/lint/golint
+
+errcheck: install-errcheck
+	errcheck -exclude=errcheck_excludes.txt ./lib/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vminsert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmselect/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmstorage/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmagent/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmalert/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmauth/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmbackup/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmrestore/...
+	errcheck -exclude=errcheck_excludes.txt ./app/vmctl/...
+
+install-errcheck:
+	which errcheck || GO111MODULE=off go get github.com/kisielk/errcheck
+
+check-all: fmt vet lint errcheck golangci-lint
 
 test:
 	go test ./lib/... ./app/...
@@ -466,7 +367,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.21
+	go mod tidy -compat=1.18
 	go mod vendor
 
 app-local:
@@ -485,53 +386,36 @@ quicktemplate-gen: install-qtc
 	qtc
 
 install-qtc:
-	which qtc || go install github.com/valyala/quicktemplate/qtc@latest
+	which qtc || GO111MODULE=off go get github.com/valyala/quicktemplate/qtc
 
 
 golangci-lint: install-golangci-lint
-	golangci-lint run
+	golangci-lint run --exclude '(SA4003|SA1019|SA5011):' -D errcheck -D structcheck --timeout 2m
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.57.1
-
-govulncheck: install-govulncheck
-	govulncheck ./...
-
-install-govulncheck:
-	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
 
 install-wwhrd:
-	which wwhrd || go install github.com/frapposelli/wwhrd@latest
+	which wwhrd || GO111MODULE=off go get github.com/frapposelli/wwhrd
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
 
 copy-docs:
-# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
-# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
-	echo "---" > ${DST}
-	@if [ ${ORDER} -ne 0 ]; then \
-		echo "sort: ${ORDER}" >> ${DST}; \
-		echo "weight: ${ORDER}" >> ${DST}; \
-		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
-	fi
-
-	echo "title: ${TITLE}" >> ${DST}
-	@if [ ${OLD_URL} ]; then \
-		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
-	fi
-	echo "---" >> ${DST}
+	echo "---\nsort: ${ORDER}\n---\n" > ${DST}
 	cat ${SRC} >> ${DST}
-	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
-	rm -rf docs/*.tmp
 
-# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
-# For ORDER=0 it adds no order tag/weight tag.
-# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
-# Images starting with <img src="docs/ are replaced with <img src="
-# Cluster docs are supposed to be ordered as 2nd.
-# The rest of docs is ordered manually.
+# Copies docs for all components and adds the order tag.
+# Cluster docs are supposed to be ordered as 9th.
+# For The rest of docs is ordered manually.t
 docs-sync:
-	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
-	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs
+	cp README.md docs/README.md
+	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md ORDER=1 $(MAKE) copy-docs
+	SRC=app/vmagent/README.md DST=docs/vmagent.md ORDER=3 $(MAKE) copy-docs
+	SRC=app/vmalert/README.md DST=docs/vmalert.md ORDER=4 $(MAKE) copy-docs
+	SRC=app/vmauth/README.md DST=docs/vmauth.md ORDER=5 $(MAKE) copy-docs
+	SRC=app/vmbackup/README.md DST=docs/vmbackup.md ORDER=6 $(MAKE) copy-docs
+	SRC=app/vmrestore/README.md DST=docs/vmrestore.md ORDER=7 $(MAKE) copy-docs
+	SRC=app/vmctl/README.md DST=docs/vmctl.md ORDER=8 $(MAKE) copy-docs
+	SRC=app/vmgateway/README.md DST=docs/vmgateway.md ORDER=9 $(MAKE) copy-docs
+	SRC=app/vmbackupmanager/README.md DST=docs/vmbackupmanager.md ORDER=10 $(MAKE) copy-docs

SECURITY.md

@@ -1,18 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-The following versions of VictoriaMetrics receive regular security fixes:
-
-| Version | Supported          |
-|---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
-| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| other releases  | :x:                |
-
-See [this page](https://victoriametrics.com/security/) for more details.
-
-## Reporting a Vulnerability
-
-Please report any security issues to security@victoriametrics.com

app/victoria-logs/Makefile

@@ -1,103 +0,0 @@
-# All these commands must run from repository root.
-
-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) app-local
-
-victoria-logs-race:
-	APP_NAME=victoria-logs RACE=-race $(MAKE) app-local
-
-victoria-logs-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker
-
-victoria-logs-pure-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-pure
-
-victoria-logs-linux-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-amd64
-
-victoria-logs-linux-arm-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm
-
-victoria-logs-linux-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-arm64
-
-victoria-logs-linux-ppc64le-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-ppc64le
-
-victoria-logs-linux-386-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-linux-386
-
-victoria-logs-darwin-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-amd64
-
-victoria-logs-darwin-arm64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-darwin-arm64
-
-victoria-logs-freebsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-freebsd-amd64
-
-victoria-logs-openbsd-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-openbsd-amd64
-
-victoria-logs-windows-amd64-prod:
-	APP_NAME=victoria-logs $(MAKE) app-via-docker-windows-amd64
-
-package-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker
-
-package-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-pure
-
-package-victoria-logs-amd64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-amd64
-
-package-victoria-logs-arm:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm
-
-package-victoria-logs-arm64:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-arm64
-
-package-victoria-logs-ppc64le:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-ppc64le
-
-package-victoria-logs-386:
-	APP_NAME=victoria-logs $(MAKE) package-via-docker-386
-
-publish-victoria-logs:
-	APP_NAME=victoria-logs $(MAKE) publish-via-docker
-
-victoria-logs-linux-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-ppc64le:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-s390x:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-victoria-logs-linux-386:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-darwin-arm64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-freebsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-openbsd-amd64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-victoria-logs-windows-amd64:
-	GOARCH=amd64 APP_NAME=victoria-logs $(MAKE) app-local-windows-goarch
-
-victoria-logs-pure:
-	APP_NAME=victoria-logs $(MAKE) app-local-pure

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image
-FROM $base_image
-
-EXPOSE 9428
-
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary
-COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -1,105 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-)
-
-var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.Init()
-
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":9428"}
-	}
-	logger.Infof("starting VictoriaLogs at %q...", listenAddrs)
-	startTime := time.Now()
-
-	vlstorage.Init()
-	vlselect.Init()
-	vlinsert.Init()
-
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
-	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/VictoriaLogs/", time.Since(startTime).Seconds())
-
-	pushmetrics.Init()
-	sig := procutil.WaitForSigterm()
-	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
-
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
-	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
-
-	vlinsert.Stop()
-	vlselect.Stop()
-	vlstorage.Stop()
-
-	fs.MustStopDirRemover()
-
-	logger.Infof("the VictoriaLogs has been stopped in %.3f seconds", time.Since(startTime).Seconds())
-}
-
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
-			return false
-		}
-		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, "<h2>Single-node VictoriaLogs</h2></br>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/VictoriaLogs/'>https://docs.victoriametrics.com/VictoriaLogs/</a></br>")
-		fmt.Fprintf(w, "Useful endpoints:</br>")
-		httpserver.WriteAPIHelp(w, [][2]string{
-			{"select/vmui", "Web UI for VictoriaLogs"},
-			{"metrics", "available service metrics"},
-			{"flags", "command-line flags"},
-		})
-		return true
-	}
-	if vlinsert.RequestHandler(w, r) {
-		return true
-	}
-	if vlselect.RequestHandler(w, r) {
-		return true
-	}
-	return false
-}
-
-func usage() {
-	const s = `
-victoria-logs is a log management and analytics service.
-
-See the docs at https://docs.victoriametrics.com/VictoriaLogs/
-`
-	flagutil.Usage(s)
-}

app/victoria-logs/multiarch/Dockerfile

@@ -1,12 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
-FROM $certs_image as certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-EXPOSE 9428
-ENTRYPOINT ["/victoria-logs-prod"]
-ARG TARGETARCH
-COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/Makefile

@@ -39,9 +39,6 @@ victoria-metrics-freebsd-amd64-prod:
 victoria-metrics-openbsd-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-openbsd-amd64
 
-victoria-metrics-windows-amd64-prod:
-	APP_NAME=victoria-metrics $(MAKE) app-via-docker-windows-amd64
-
 package-victoria-metrics:
 	APP_NAME=victoria-metrics $(MAKE) package-via-docker
 
@@ -85,9 +82,6 @@ victoria-metrics-linux-arm64:
 victoria-metrics-linux-ppc64le:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-victoria-metrics-linux-s390x:
-	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
 victoria-metrics-linux-386:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 
@@ -103,9 +97,6 @@ victoria-metrics-freebsd-amd64:
 victoria-metrics-openbsd-amd64:
 	APP_NAME=victoria-metrics CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
 
-victoria-metrics-windows-amd64:
-	GOARCH=amd64 APP_NAME=victoria-metrics $(MAKE) app-local-windows-goarch
-
 victoria-metrics-pure:
 	APP_NAME=victoria-metrics $(MAKE) app-local-pure
 

app/victoria-metrics/main.go

@@ -8,8 +8,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert"
-	vminsertcommon "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
-	vminsertrelabel "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
@@ -26,19 +24,11 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP addresses to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
-		"equal to -dedup.minScrapeInterval > 0. See also -streamAggr.dedupInterval and https://docs.victoriametrics.com/#deduplication")
-	dryRun = flag.Bool("dryRun", false, "Whether to check config files without running VictoriaMetrics. The following config files are checked: "+
-		"-promscrape.config, -relabelConfig and -streamAggr.config. Unknown config entries aren't allowed in -promscrape.config by default. "+
-		"This can be changed with -promscrape.config.strictParse=false command-line flag")
-	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
-		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
+		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
+	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
+		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
 )
 
 func main() {
@@ -48,6 +38,7 @@ func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		*dryRun = true
@@ -56,47 +47,33 @@ func main() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		if err := vminsertrelabel.CheckRelabelConfig(); err != nil {
-			logger.Fatalf("error when checking -relabelConfig: %s", err)
-		}
-		if err := vminsertcommon.CheckStreamAggrConfig(); err != nil {
-			logger.Fatalf("error when checking -streamAggr.config: %s", err)
-		}
-		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8428"}
-	}
-	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
+	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
-	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-
 	startSelfScraper()
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
-	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	pushmetrics.Stop()
 
 	stopSelfScraper()
 
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
+	logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
 	startTime = time.Now()
-	if err := httpserver.Stop(listenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		logger.Fatalf("cannot stop the webservice: %s", err)
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	vminsert.Stop()
+	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	vmstorage.Stop()
 	vmselect.Stop()
@@ -108,7 +85,7 @@ func main() {
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
+		if r.Method != "GET" {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
@@ -119,8 +96,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"vmui", "Web UI"},
 			{"targets", "status for discovered active targets"},
 			{"service-discovery", "labels before and after relabeling for discovered targets"},
-			{"metric-relabel-debug", "debug metric relabeling"},
-			{"expand-with-exprs", "WITH expressions' tutorial"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -128,7 +103,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"api/v1/status/tsdb", "tsdb status page"},
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
-			{"-/reload", "reload configuration"},
 		})
 		return true
 	}

app/victoria-metrics/main_test.go

@@ -7,13 +7,11 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -39,13 +37,11 @@ const (
 )
 
 const (
-	testReadHTTPPath           = "http://127.0.0.1" + testHTTPListenAddr
-	testWriteHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr + "/write"
-	testOpenTSDBWriteHTTPPath  = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
-	testPromWriteHTTPPath      = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
-	testImportCSVWriteHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/import/csv"
-
-	testHealthHTTPPath = "http://127.0.0.1" + testHTTPListenAddr + "/health"
+	testReadHTTPPath          = "http://127.0.0.1" + testHTTPListenAddr
+	testWriteHTTPPath         = "http://127.0.0.1" + testHTTPListenAddr + "/write"
+	testOpenTSDBWriteHTTPPath = "http://127.0.0.1" + testOpenTSDBHTTPListenAddr + "/api/put"
+	testPromWriteHTTPPath     = "http://127.0.0.1" + testHTTPListenAddr + "/api/v1/write"
+	testHealthHTTPPath        = "http://127.0.0.1" + testHTTPListenAddr + "/health"
 )
 
 const (
@@ -58,15 +54,15 @@ var (
 )
 
 type test struct {
-	Name                     string   `json:"name"`
-	Data                     []string `json:"data"`
-	InsertQuery              string   `json:"insert_query"`
-	Query                    []string `json:"query"`
-	ResultMetrics            []Metric `json:"result_metrics"`
-	ResultSeries             Series   `json:"result_series"`
-	ResultQuery              Query    `json:"result_query"`
-	Issue                    string   `json:"issue"`
-	ExpectedResultLinesCount int      `json:"expected_result_lines_count"`
+	Name             string     `json:"name"`
+	Data             []string   `json:"data"`
+	InsertQuery      string     `json:"insert_query"`
+	Query            []string   `json:"query"`
+	ResultMetrics    []Metric   `json:"result_metrics"`
+	ResultSeries     Series     `json:"result_series"`
+	ResultQuery      Query      `json:"result_query"`
+	ResultQueryRange QueryRange `json:"result_query_range"`
+	Issue            string     `json:"issue"`
 }
 
 type Metric struct {
@@ -84,90 +80,42 @@ type Series struct {
 	Status string              `json:"status"`
 	Data   []map[string]string `json:"data"`
 }
-
 type Query struct {
-	Status string `json:"status"`
-	Data   struct {
-		ResultType string          `json:"resultType"`
-		Result     json.RawMessage `json:"result"`
-	} `json:"data"`
+	Status string    `json:"status"`
+	Data   QueryData `json:"data"`
+}
+type QueryData struct {
+	ResultType string            `json:"resultType"`
+	Result     []QueryDataResult `json:"result"`
 }
 
-const rtVector, rtMatrix = "vector", "matrix"
-
-func (q *Query) metrics() ([]Metric, error) {
-	switch q.Data.ResultType {
-	case rtVector:
-		var r QueryInstant
-		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
-			return nil, err
-		}
-		return r.metrics()
-	case rtMatrix:
-		var r QueryRange
-		if err := json.Unmarshal(q.Data.Result, &r.Result); err != nil {
-			return nil, err
-		}
-		return r.metrics()
-	default:
-		return nil, fmt.Errorf("unknown result type %q", q.Data.ResultType)
-	}
+type QueryDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Value  []interface{}     `json:"value"`
 }
 
-type QueryInstant struct {
-	Result []struct {
-		Labels map[string]string `json:"metric"`
-		TV     [2]interface{}    `json:"value"`
-	} `json:"result"`
-}
-
-func (q QueryInstant) metrics() ([]Metric, error) {
-	result := make([]Metric, len(q.Result))
-	for i, res := range q.Result {
-		f, err := strconv.ParseFloat(res.TV[1].(string), 64)
-		if err != nil {
-			return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, res.TV[1], err)
-		}
-		var m Metric
-		m.Metric = res.Labels
-		m.Timestamps = append(m.Timestamps, int64(res.TV[0].(float64)))
-		m.Values = append(m.Values, f)
-		result[i] = m
-	}
-	return result, nil
+func (r *QueryDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
 }
 
 type QueryRange struct {
-	Result []struct {
-		Metric map[string]string `json:"metric"`
-		Values [][]interface{}   `json:"values"`
-	} `json:"result"`
+	Status string         `json:"status"`
+	Data   QueryRangeData `json:"data"`
+}
+type QueryRangeData struct {
+	ResultType string                 `json:"resultType"`
+	Result     []QueryRangeDataResult `json:"result"`
 }
 
-func (q QueryRange) metrics() ([]Metric, error) {
-	var result []Metric
-	for i, res := range q.Result {
-		var m Metric
-		for _, tv := range res.Values {
-			f, err := strconv.ParseFloat(tv[1].(string), 64)
-			if err != nil {
-				return nil, fmt.Errorf("metric %v, unable to parse float64 from %s: %w", res, tv[1], err)
-			}
-			m.Values = append(m.Values, f)
-			m.Timestamps = append(m.Timestamps, int64(tv[0].(float64)))
-		}
-		if len(m.Values) < 1 || len(m.Timestamps) < 1 {
-			return nil, fmt.Errorf("metric %v contains no values", res)
-		}
-		m.Metric = q.Result[i].Metric
-		result = append(result, m)
-	}
-	return result, nil
+type QueryRangeDataResult struct {
+	Metric map[string]string `json:"metric"`
+	Values [][]interface{}   `json:"values"`
 }
 
-func (q *Query) UnmarshalJSON(b []byte) error {
-	type plain Query
-	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(q))
+func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error {
+	type plain QueryRangeDataResult
+	return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r))
 }
 
 func TestMain(m *testing.M) {
@@ -181,16 +129,16 @@ func setUp() {
 	storagePath = filepath.Join(os.TempDir(), testStorageSuffix)
 	processFlags()
 	logger.Init()
-	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
+	vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddrs, useProxyProtocol, requestHandler)
+	go httpserver.Serve(*httpListenAddr, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
 			return false
 		}
-		_ = resp.Body.Close()
+		resp.Body.Close()
 		return resp.StatusCode == 200
 	}
 	if err := waitFor(testStorageInitTimeout, readyStorageCheckFunc); err != nil {
@@ -230,7 +178,7 @@ func waitFor(timeout time.Duration, f func() bool) error {
 }
 
 func tearDown() {
-	if err := httpserver.Stop(*httpListenAddrs); err != nil {
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
 		log.Printf("cannot stop the webservice: %s", err)
 	}
 	vminsert.Stop()
@@ -241,18 +189,16 @@ func tearDown() {
 
 func TestWriteRead(t *testing.T) {
 	t.Run("write", testWrite)
-	time.Sleep(500 * time.Millisecond)
-	vmstorage.Storage.DebugFlush()
-	time.Sleep(1500 * time.Millisecond)
+	time.Sleep(1 * time.Second)
+	vmstorage.Stop()
+	// open storage after stop in write
+	vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
 	t.Run("read", testRead)
 }
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
 		for _, test := range readIn("prometheus", t, insertionTime) {
-			if test.Data == nil {
-				continue
-			}
 			s := newSuite(t)
 			r := testutil.WriteRequest{}
 			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
@@ -265,14 +211,6 @@ func testWrite(t *testing.T) {
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
-	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", t, insertionTime) {
-			if test.Data == nil {
-				continue
-			}
-			httpWrite(t, testImportCSVWriteHTTPPath, test.InsertQuery, bytes.NewBuffer([]byte(strings.Join(test.Data, "\n"))))
-		}
-	})
 
 	t.Run("influxdb", func(t *testing.T) {
 		for _, x := range readIn("influxdb", t, insertionTime) {
@@ -314,7 +252,7 @@ func testWrite(t *testing.T) {
 }
 
 func testRead(t *testing.T) {
-	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
+	for _, engine := range []string{"prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
 			for _, x := range readIn(engine, t, insertionTime) {
 				test := x
@@ -323,14 +261,9 @@ func testRead(t *testing.T) {
 					for _, q := range test.Query {
 						q = testutil.PopulateTimeTplString(q, insertionTime)
 						if test.Issue != "" {
-							test.Issue = "\nRegression in " + test.Issue
+							test.Issue = "Regression in " + test.Issue
 						}
-						switch {
-						case strings.HasPrefix(q, "/api/v1/export/csv"):
-							data := strings.Split(string(httpReadData(t, testReadHTTPPath, q)), "\n")
-							if len(data) == test.ExpectedResultLinesCount {
-								t.Fatalf("not expected number of csv lines want=%d\ngot=%d test=%s.%s\n\response=%q", len(data), test.ExpectedResultLinesCount, q, test.Issue, strings.Join(data, "\n"))
-							}
+						switch true {
 						case strings.HasPrefix(q, "/api/v1/export"):
 							if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil {
 								t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue)
@@ -341,19 +274,17 @@ func testRead(t *testing.T) {
 							if err := checkSeriesResult(s, test.ResultSeries); err != nil {
 								t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue)
 							}
+						case strings.HasPrefix(q, "/api/v1/query_range"):
+							queryResult := QueryRange{}
+							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
+							if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil {
+								t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue)
+							}
 						case strings.HasPrefix(q, "/api/v1/query"):
 							queryResult := Query{}
 							httpReadStruct(t, testReadHTTPPath, q, &queryResult)
-							gotMetrics, err := queryResult.metrics()
-							if err != nil {
-								t.Fatalf("failed to parse query response: %s", err)
-							}
-							expMetrics, err := test.ResultQuery.metrics()
-							if err != nil {
-								t.Fatalf("failed to parse expected response: %s", err)
-							}
-							if err := checkMetricsResult(gotMetrics, expMetrics); err != nil {
-								t.Fatalf("%q fails with error %s.%s", q, err, test.Issue)
+							if err := checkQueryResult(queryResult, test.ResultQuery); err != nil {
+								t.Fatalf("Query. %s fails with error %s.%s", q, err, test.Issue)
 							}
 						default:
 							t.Fatalf("unsupported read query %s", q)
@@ -369,7 +300,7 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 	t.Helper()
 	s := newSuite(t)
 	var tt []test
-	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -406,9 +337,7 @@ func tcpWrite(t *testing.T, address string, data string) {
 	s := newSuite(t)
 	conn, err := net.Dial("tcp", address)
 	s.noError(err)
-	defer func() {
-		_ = conn.Close()
-	}()
+	defer conn.Close()
 	n, err := conn.Write([]byte(data))
 	s.noError(err)
 	s.equalInt(n, len(data))
@@ -419,9 +348,7 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer func() {
-		_ = resp.Body.Close()
-	}()
+	defer resp.Body.Close()
 	s.equalInt(resp.StatusCode, 200)
 	var rows []Metric
 	for dec := json.NewDecoder(resp.Body); dec.More(); {
@@ -431,33 +358,16 @@ func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	}
 	return rows
 }
-
 func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
 	s := newSuite(t)
 	resp, err := http.Get(address + query)
 	s.noError(err)
-	defer func() {
-		_ = resp.Body.Close()
-	}()
+	defer resp.Body.Close()
 	s.equalInt(resp.StatusCode, 200)
 	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
-func httpReadData(t *testing.T, address, query string) []byte {
-	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	s.equalInt(resp.StatusCode, 200)
-	data, err := io.ReadAll(resp.Body)
-	s.noError(err)
-	return data
-}
-
 func checkMetricsResult(got, want []Metric) error {
 	for _, r := range append([]Metric(nil), got...) {
 		want = removeIfFoundMetrics(r, want)
@@ -503,6 +413,60 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
+func checkQueryResult(got, want Query) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Value[0], item.Value[0]) && reflect.DeepEqual(r.Value[1], item.Value[1]) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
+func checkQueryRangeResult(got, want QueryRange) error {
+	if got.Status != want.Status {
+		return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status)
+	}
+	if got.Data.ResultType != want.Data.ResultType {
+		return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType)
+	}
+	wantData := append([]QueryRangeDataResult(nil), want.Data.Result...)
+	for _, r := range got.Data.Result {
+		wantData = removeIfFoundQueryRangeData(r, wantData)
+	}
+	if len(wantData) > 0 {
+		return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result)
+	}
+	return nil
+}
+
+func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult {
+	for i, item := range contains {
+		if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) {
+			contains[i] = contains[len(contains)-1]
+			return contains[:len(contains)-1]
+		}
+	}
+	return contains
+}
+
 type suite struct{ t *testing.T }
 
 func newSuite(t *testing.T) *suite { return &suite{t: t} }
@@ -530,73 +494,3 @@ func (s *suite) greaterThan(a, b int) {
 		s.t.FailNow()
 	}
 }
-
-func TestImportJSONLines(t *testing.T) {
-	f := func(labelsCount, labelLen int) {
-		t.Helper()
-
-		reqURL := fmt.Sprintf("http://localhost%s/api/v1/import", testHTTPListenAddr)
-		line := generateJSONLine(labelsCount, labelLen)
-		req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(line))
-		if err != nil {
-			t.Fatalf("cannot create request: %s", err)
-		}
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			t.Fatalf("cannot perform request for labelsCount=%d, labelLen=%d: %s", labelsCount, labelLen, err)
-		}
-		if resp.StatusCode != 204 {
-			t.Fatalf("unexpected statusCode for labelsCount=%d, labelLen=%d; got %d; want 204", labelsCount, labelLen, resp.StatusCode)
-		}
-	}
-
-	// labels with various lengths
-	for i := 0; i < 500; i++ {
-		f(10, i*5)
-	}
-
-	// Too many labels
-	f(1000, 100)
-
-	// Too long labels
-	f(1, 100_000)
-	f(10, 100_000)
-	f(10, 10_000)
-}
-
-func generateJSONLine(labelsCount, labelLen int) string {
-	m := make(map[string]string, labelsCount)
-	m["__name__"] = generateSizedRandomString(labelLen)
-	for j := 1; j < labelsCount; j++ {
-		labelName := generateSizedRandomString(labelLen)
-		labelValue := generateSizedRandomString(labelLen)
-		m[labelName] = labelValue
-	}
-
-	type jsonLine struct {
-		Metric     map[string]string `json:"metric"`
-		Values     []float64         `json:"values"`
-		Timestamps []int64           `json:"timestamps"`
-	}
-	line := &jsonLine{
-		Metric:     m,
-		Values:     []float64{1.34},
-		Timestamps: []int64{time.Now().UnixNano() / 1e6},
-	}
-	data, err := json.Marshal(&line)
-	if err != nil {
-		panic(fmt.Errorf("cannot marshal JSON: %w", err))
-	}
-	data = append(data, '\n')
-	return string(data)
-}
-
-const alphabetSample = `qwertyuiopasdfghjklzxcvbnm`
-
-func generateSizedRandomString(size int) string {
-	dst := make([]byte, size)
-	for i := range dst {
-		dst[i] = alphabetSample[rand.Intn(len(alphabetSample))]
-	}
-	return string(dst)
-}

app/victoria-metrics/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
+RUN apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/victoria-metrics/self_scraper.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -50,8 +49,16 @@ func selfScraper(scrapeInterval time.Duration) {
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
-	f := func(currentTime time.Time, sendStaleMarkers bool) {
-		currentTimestamp := currentTime.UnixNano() / 1e6
+	var currentTimestamp int64
+	for {
+		select {
+		case <-selfScraperStopCh:
+			t.Stop()
+			logger.Infof("stopped self-scraping `/metrics` page")
+			return
+		case currentTime := <-t.C:
+			currentTimestamp = currentTime.UnixNano() / 1e6
+		}
 		bb.Reset()
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
@@ -76,26 +83,9 @@ func selfScraper(scrapeInterval time.Duration) {
 			mr := &mrs[len(mrs)-1]
 			mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
 			mr.Timestamp = currentTimestamp
-			if sendStaleMarkers {
-				mr.Value = decimal.StaleNaN
-			} else {
-				mr.Value = r.Value
-			}
-		}
-		if err := vmstorage.AddRows(mrs); err != nil {
-			logger.Errorf("cannot store self-scraped metrics: %s", err)
-		}
-	}
-	for {
-		select {
-		case <-selfScraperStopCh:
-			f(time.Now(), true)
-			t.Stop()
-			logger.Infof("stopped self-scraping `/metrics` page")
-			return
-		case currentTime := <-t.C:
-			f(currentTime, false)
+			mr.Value = r.Value
 		}
+		vmstorage.AddRows(mrs)
 	}
 }
 
@@ -106,7 +96,7 @@ func addLabel(dst []prompb.Label, key, value string) []prompb.Label {
 		dst = append(dst, prompb.Label{})
 	}
 	lb := &dst[len(dst)-1]
-	lb.Name = key
-	lb.Value = value
+	lb.Name = bytesutil.ToUnsafeBytes(key)
+	lb.Value = bytesutil.ToUnsafeBytes(value)
 	return dst
 }

app/victoria-metrics/test/parser_test.go

@@ -12,7 +12,7 @@ func TestPopulateTimeTplString(t *testing.T) {
 	}
 	f := func(s, resultExpected string) {
 		t.Helper()
-		result := PopulateTimeTplString(s, now.UTC())
+		result := PopulateTimeTplString(s, now)
 		if result != resultExpected {
 			t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
 		}

app/victoria-metrics/testdata/csv/basic.json

@@ -1,14 +0,0 @@
-{
-  "name": "csv export",
-  "data": [
-    "rfc3339,4,{TIME_MS}",
-    "rfc3339milli,6,{TIME_MS}",
-    "ts,8,{TIME_MS}",
-    "tsms,10,{TIME_MS},"
-  ],
-  "insert_query": "?format=1:label:tfmt,2:metric:test_csv,3:time:unix_ms",
-  "query": [
-    "/api/v1/export/csv?format=__name__,tfmt,__value__,__timestamp__:rfc3339&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
-  ],
-  "expected_result_lines_count": 4
-}

app/victoria-metrics/testdata/csv/with_extra_labels.json

@@ -1,14 +0,0 @@
-{
-  "name": "csv export with extra_labels",
-  "data": [
-    "location-1,4,{TIME_MS}",
-    "location-2,6,{TIME_MS}",
-    "location-3,8,{TIME_MS}",
-    "location-4,10,{TIME_MS},"
-  ],
-  "insert_query": "?format=1:label:location,2:metric:test_csv_labels,3:time:unix_ms&extra_label=location=location-1",
-  "query": [
-    "/api/v1/export/csv?format=__name__,location,__value__,__timestamp__:unix_ms&match[]={__name__=\"test_csv\"}&step=30s&start={TIME_MS-180s}"
-  ],
-  "expected_result_lines_count": 4
-}

app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json

@@ -7,7 +7,7 @@
     "not_nan_not_inf;item=y 3 {TIME_S-1m}",
     "not_nan_not_inf;item=y 1 {TIME_S-2m}"],
   "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-3m}&end={TIME_S}&step=60"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/empty-label-match.json

@@ -6,7 +6,7 @@
     "empty_label_match;foo=bar 2 {TIME_S-1m}",
     "empty_label_match;foo=baz 3 {TIME_S-1m}"],
   "query": ["/api/v1/query_range?query=empty_label_match{foo=~'bar|'}&start={TIME_S-1m}&end={TIME_S}&step=60"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/max_lookback_set.json

@@ -8,7 +8,7 @@
     "max_lookback_set 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_set&start={TIME_S-150s}&end={TIME_S}&step=10s&max_lookback=1s"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_set"},"values":[

app/victoria-metrics/testdata/graphite/max_lookback_unset.json

@@ -8,7 +8,7 @@
     "max_lookback_unset 4 {TIME_S-150s}"
   ],
   "query": ["/api/v1/query_range?query=max_lookback_unset&start={TIME_S-150s}&end={TIME_S}&step=10s"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[{"metric":{"__name__":"max_lookback_unset"},"values":[

app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json

@@ -8,7 +8,7 @@
     "not_nan_as_missing_data;item=y 3 {TIME_S-1m}"
   ],
   "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"],
-  "result_query": {
+  "result_query_range": {
     "status":"success",
     "data":{"resultType":"matrix",
       "result":[

app/victoria-metrics/testdata/graphite/subquery-aggregation.json

@@ -6,7 +6,7 @@
     "forms_daily_count;item=x 2 {TIME_S-2m}",
     "forms_daily_count;item=y 3 {TIME_S-1m}",
     "forms_daily_count;item=y 4 {TIME_S-2m}"],
-  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}&latency_offset=1ms"],
+  "query": ["/api/v1/query?query=min%20by%20(item)%20(min_over_time(forms_daily_count[10m:1m]))&time={TIME_S-1m}"],
   "result_query": {
     "status":"success",
     "data":{"resultType":"vector","result":[{"metric":{"item":"x"},"value":["{TIME_S-1m}","2"]},{"metric":{"item":"y"},"value":["{TIME_S-1m}","4"]}]}

app/victoria-metrics/testdata/prometheus/instant-matrix.json

@@ -1,12 +0,0 @@
-{
-  "name": "instant query with look-behind window",
-  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
-  "query": ["/api/v1/query?query=foo[5m]"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"matrix",
-        "result":[{"metric":{"__name__":"foo"},"values":[["{TIME_S-60s}", "1"]]}]
-      }
-    }
-}

app/victoria-metrics/testdata/prometheus/instant-scalar.json

@@ -1,11 +0,0 @@
-{
-  "name": "instant scalar query",
-  "query": ["/api/v1/query?query=42&time={TIME_S}"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"vector",
-        "result":[{"metric":{},"value":["{TIME_S}", "42"]}]
-      }
-    }
-}

app/victoria-metrics/testdata/prometheus/issue-5553-too-big-lookback.json

@@ -1,13 +0,0 @@
-{
-  "name": "too big look-behind window",
-  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
-  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"foo\"},{\"name\":\"issue\",\"value\":\"5553\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}]}]"],
-  "query": ["/api/v1/query?query=foo{issue=\"5553\"}[100y]"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"matrix",
-        "result":[{"metric":{"__name__":"foo", "issue": "5553"},"values":[["{TIME_S-60s}", "1"]]}]
-      }
-    }
-}

app/victoria-metrics/testdata/prometheus/query-range.json

@@ -1,18 +0,0 @@
-{
-  "name": "query range",
-  "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553",
-  "data": ["[{\"labels\":[{\"name\":\"__name__\",\"value\":\"bar\"}],\"samples\":[{\"value\":1,\"timestamp\":\"{TIME_MS-60s}\"}, {\"value\":2,\"timestamp\":\"{TIME_MS-120s}\"}, {\"value\":1,\"timestamp\":\"{TIME_MS-180s}\"}]}]"],
-  "query": ["/api/v1/query_range?query=bar&step=30s&start={TIME_MS-180s}"],
-  "result_query": {
-      "status": "success",
-      "data":{
-        "resultType":"matrix",
-        "result":[
-          {
-            "metric":{"__name__":"bar"},
-            "values":[["{TIME_S-180s}", "1"],["{TIME_S-150s}", "1"],["{TIME_S-120s}", "2"],["{TIME_S-90s}", "2"], ["{TIME_S-60s}", "1"], ["{TIME_S-30s}", "1"], ["{TIME_S}", "1"]]
-          }
-        ]
-      }
-    }
-}

app/vlinsert/elasticsearch/bulk_response.qtpl

@@ -1,20 +0,0 @@
-{% stripspace %}
-
-{% func BulkResponse(n int, tookMs int64) %}
-{
-	"took":{%dl tookMs %},
-	"errors":false,
-	"items":[
-		{% for i := 0; i < n; i++ %}
-		{
-			"create":{
-				"status":201
-			}
-		}
-		{% if i+1 < n %},{% endif %}
-		{% endfor %}
-	]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlinsert/elasticsearch/bulk_response.qtpl.go

@@ -1,69 +0,0 @@
-// Code generated by qtc from "bulk_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-package elasticsearch
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-func StreamBulkResponse(qw422016 *qt422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:3
-	qw422016.N().S(`{"took":`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().DL(tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:5
-	qw422016.N().S(`,"errors":false,"items":[`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-	for i := 0; i < n; i++ {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:8
-		qw422016.N().S(`{"create":{"status":201}}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		if i+1 < n {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-			qw422016.N().S(`,`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:14
-		}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	}
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:15
-	qw422016.N().S(`]}`)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func WriteBulkResponse(qq422016 qtio422016.Writer, n int, tookMs int64) {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	StreamBulkResponse(qw422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}
-
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-func BulkResponse(n int, tookMs int64) string {
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	WriteBulkResponse(qb422016, n, tookMs)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-	return qs422016
-//line app/vlinsert/elasticsearch/bulk_response.qtpl:18
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -1,281 +0,0 @@
-package elasticsearch
-
-import (
-	"bufio"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-)
-
-var (
-	elasticsearchVersion = flag.String("elasticsearch.version", "8.9.0", "Elasticsearch version to report to client")
-)
-
-// RequestHandler processes Elasticsearch insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	// This header is needed for Logstash
-	w.Header().Set("X-Elastic-Product", "Elasticsearch")
-
-	if strings.HasPrefix(path, "/_ilm/policy") {
-		// Return fake response for Elasticsearch ilm request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_index_template") {
-		// Return fake response for Elasticsearch index template request.
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_ingest") {
-		// Return fake response for Elasticsearch ingest pipeline request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	if strings.HasPrefix(path, "/_nodes") {
-		// Return fake response for Elasticsearch nodes discovery request.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
-	switch path {
-	case "/":
-		switch r.Method {
-		case http.MethodGet:
-			// Return fake response for Elasticsearch ping request.
-			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
-			fmt.Fprintf(w, `{
-			"version": {
-				"number": %q
-			}
-		}`, *elasticsearchVersion)
-		case http.MethodHead:
-			// Return empty response for Logstash ping request.
-		}
-
-		return true
-	case "/_license":
-		// Return fake response for Elasticsearch license request.
-		fmt.Fprintf(w, `{
-			"license": {
-				"uid": "cbff45e7-c553-41f7-ae4f-9205eabd80xx",
-				"type": "oss",
-				"status": "active",
-				"expiry_date_in_millis" : 4000000000000
-			}
-		}`)
-		return true
-	case "/_bulk":
-		startTime := time.Now()
-		bulkRequestsTotal.Inc()
-
-		cp, err := insertutils.GetCommonParams(r)
-		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		if err := vlstorage.CanWriteData(); err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-		processLogMessage := cp.GetProcessLogMessageFunc(lr)
-		isGzip := r.Header.Get("Content-Encoding") == "gzip"
-		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
-		vlstorage.MustAddRows(lr)
-		logstorage.PutLogRows(lr)
-		if err != nil {
-			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
-			return true
-		}
-
-		tookMs := time.Since(startTime).Milliseconds()
-		bw := bufferedwriter.Get(w)
-		defer bufferedwriter.Put(bw)
-		WriteBulkResponse(bw, n, tookMs)
-		_ = bw.Flush()
-
-		// update bulkRequestDuration only for successfully parsed requests
-		// There is no need in updating bulkRequestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		bulkRequestDuration.UpdateDuration(startTime)
-
-		return true
-	default:
-		return false
-	}
-}
-
-var (
-	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
-	rowsIngestedTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
-	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
-)
-
-func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (int, error) {
-	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
-
-	if isGzip {
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
-		}
-		defer common.PutGzipReader(zr)
-		r = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(r)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lb := lineBufferPool.Get()
-	defer lineBufferPool.Put(lb)
-
-	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
-	sc := bufio.NewScanner(wcr)
-	sc.Buffer(lb.B, len(lb.B))
-
-	n := 0
-	nCheckpoint := 0
-	for {
-		ok, err := readBulkLine(sc, timeField, msgField, processLogMessage)
-		wcr.DecConcurrency()
-		if err != nil || !ok {
-			rowsIngestedTotal.Add(n - nCheckpoint)
-			return n, err
-		}
-		n++
-		if batchSize := n - nCheckpoint; n >= 1000 {
-			rowsIngestedTotal.Add(batchSize)
-			nCheckpoint = n
-		}
-	}
-}
-
-var lineBufferPool bytesutil.ByteBufferPool
-
-func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
-	processLogMessage func(timestamp int64, fields []logstorage.Field),
-) (bool, error) {
-	var line []byte
-
-	// Read the command, must be "create" or "index"
-	for len(line) == 0 {
-		if !sc.Scan() {
-			if err := sc.Err(); err != nil {
-				if errors.Is(err, bufio.ErrTooLong) {
-					return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
-						insertutils.MaxLineSizeBytes.IntN())
-				}
-				return false, err
-			}
-			return false, nil
-		}
-		line = sc.Bytes()
-	}
-	lineStr := bytesutil.ToUnsafeString(line)
-	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
-		return false, fmt.Errorf(`unexpected command %q; expecting "create" or "index"`, line)
-	}
-
-	// Decode log message
-	if !sc.Scan() {
-		if err := sc.Err(); err != nil {
-			if errors.Is(err, bufio.ErrTooLong) {
-				return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
-			}
-			return false, err
-		}
-		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
-	}
-	line = sc.Bytes()
-	p := logjson.GetParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
-	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
-	logjson.PutParser(p)
-
-	return true, nil
-}
-
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		timestamp, err := parseElasticsearchTimestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
-	}
-	return 0, nil
-}
-
-func parseElasticsearchTimestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case - zero or empty timestamp must be substituted
-		// with the current time by the caller.
-		return 0, nil
-	}
-	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in milliseconds
-		n, err := strconv.ParseInt(s, 10, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
-		}
-		if n > int64(math.MaxInt64)/1e6 {
-			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
-		}
-		if n < int64(math.MinInt64)/1e6 {
-			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
-		}
-		n *= 1e6
-		return n, nil
-	}
-	if len(s) == len("YYYY-MM-DD") {
-		t, err := time.Parse("2006-01-02", s)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse date %q: %w", s, err)
-		}
-		return t.UnixNano(), nil
-	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
-	}
-	return t.UnixNano(), nil
-}

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -1,129 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestReadBulkRequestFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			t.Fatalf("unexpected call to processLogMessage with timestamp=%d, fields=%s", timestamp, fields)
-		}
-
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, "_time", "_msg", processLogMessage)
-		if err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if rows != 0 {
-			t.Fatalf("unexpected non-zero rows=%d", rows)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`{"create":{}}`)
-	f(`{"creat":{}}
-{}`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadBulkRequestSuccess(t *testing.T) {
-	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
-
-		// Read the request without compression
-		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest(r, false, timeField, msgField, processLogMessage)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-
-		// Read the request with compression
-		timestamps = nil
-		result = ""
-		compressedData := compressData(data)
-		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest(r, true, timeField, msgField, processLogMessage)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Verify an empty data
-	f("", "_time", "_msg", 0, nil, "")
-	f("\n", "_time", "_msg", 0, nil, "")
-	f("\n\n", "_time", "_msg", 0, nil, "")
-
-	// Verify non-empty data
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	rowsExpected := 3
-	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
-	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
-}
-
-func compressData(s string) string {
-	var bb bytes.Buffer
-	zw := gzip.NewWriter(&bb)
-	if _, err := zw.Write([]byte(s)); err != nil {
-		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
-	}
-	if err := zw.Close(); err != nil {
-		panic(fmt.Errorf("unexpected error when closing gzip writer: %w", err))
-	}
-	return bb.String()
-}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -1,50 +0,0 @@
-package elasticsearch
-
-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("gzip:off", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, false)
-	})
-	b.Run("gzip:on", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, true)
-	})
-}
-
-func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
-	data := `{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	if isGzip {
-		data = compressData(data)
-	}
-	dataBytes := bytesutil.ToUnsafeBytes(data)
-
-	timeField := "@timestamp"
-	msgField := "message"
-	processLogMessage := func(_ int64, _ []logstorage.Field) {}
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		for pb.Next() {
-			r.Reset(dataBytes)
-			_, err := readBulkRequest(r, isGzip, timeField, msgField, processLogMessage)
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}

app/vlinsert/insertutils/common_params.go

@@ -1,100 +0,0 @@
-package insertutils
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// CommonParams contains common HTTP parameters used by log ingestion APIs.
-//
-// See https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/#http-parameters
-type CommonParams struct {
-	TenantID     logstorage.TenantID
-	TimeField    string
-	MsgField     string
-	StreamFields []string
-	IgnoreFields []string
-
-	Debug           bool
-	DebugRequestURI string
-	DebugRemoteAddr string
-}
-
-// GetCommonParams returns CommonParams from r.
-func GetCommonParams(r *http.Request) (*CommonParams, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// Extract time field name from _time_field query arg
-	var timeField = "_time"
-	if tf := r.FormValue("_time_field"); tf != "" {
-		timeField = tf
-	}
-
-	// Extract message field name from _msg_field query arg
-	var msgField = ""
-	if msgf := r.FormValue("_msg_field"); msgf != "" {
-		msgField = msgf
-	}
-
-	streamFields := httputils.GetArray(r, "_stream_fields")
-	ignoreFields := httputils.GetArray(r, "ignore_fields")
-
-	debug := httputils.GetBool(r, "debug")
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-	if debug {
-		debugRequestURI = httpserver.GetRequestURI(r)
-		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
-	}
-
-	cp := &CommonParams{
-		TenantID:        tenantID,
-		TimeField:       timeField,
-		MsgField:        msgField,
-		StreamFields:    streamFields,
-		IgnoreFields:    ignoreFields,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
-	}
-	return cp, nil
-}
-
-// GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
-func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
-	return func(timestamp int64, fields []logstorage.Field) {
-		if len(fields) > *MaxFieldsPerLine {
-			rf := logstorage.RowFormatter(fields)
-			logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
-			rowsDroppedTotalTooManyFields.Inc()
-			return
-		}
-
-		lr.MustAdd(cp.TenantID, timestamp, fields)
-		if cp.Debug {
-			s := lr.GetRowString(0)
-			lr.ResetKeepSettings()
-			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
-			rowsDroppedTotalDebug.Inc()
-			return
-		}
-		if lr.NeedFlush() {
-			vlstorage.MustAddRows(lr)
-			lr.ResetKeepSettings()
-		}
-	}
-}
-
-var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/flags.go

@@ -1,15 +0,0 @@
-package insertutils
-
-import (
-	"flag"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-)
-
-var (
-	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
-
-	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
-)

app/vlinsert/jsonline/jsonline.go

@@ -1,161 +0,0 @@
-package jsonline
-
-import (
-	"bufio"
-	"errors"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// RequestHandler processes jsonline insert requests
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	startTime := time.Now()
-	w.Header().Add("Content-Type", "application/json")
-
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return true
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			logger.Errorf("cannot read gzipped _bulk request: %s", err)
-			return true
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	defer writeconcurrencylimiter.PutReader(wcr)
-
-	lb := lineBufferPool.Get()
-	defer lineBufferPool.Put(lb)
-
-	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
-	sc := bufio.NewScanner(wcr)
-	sc.Buffer(lb.B, len(lb.B))
-
-	n := 0
-	for {
-		ok, err := readLine(sc, cp.TimeField, cp.MsgField, processLogMessage)
-		wcr.DecConcurrency()
-		if err != nil {
-			logger.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
-			break
-		}
-		if !ok {
-			break
-		}
-		n++
-		rowsIngestedTotal.Inc()
-	}
-
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-
-	// update jsonlineRequestDuration only for successfully parsed requests.
-	// There is no need in updating jsonlineRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	jsonlineRequestDuration.UpdateDuration(startTime)
-
-	return true
-}
-
-func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage func(timestamp int64, fields []logstorage.Field)) (bool, error) {
-	var line []byte
-	for len(line) == 0 {
-		if !sc.Scan() {
-			if err := sc.Err(); err != nil {
-				if errors.Is(err, bufio.ErrTooLong) {
-					return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
-				}
-				return false, err
-			}
-			return false, nil
-		}
-		line = sc.Bytes()
-	}
-
-	p := logjson.GetParser()
-	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
-	}
-	ts, err := extractTimestampFromFields(timeField, p.Fields)
-	if err != nil {
-		return false, fmt.Errorf("cannot parse timestamp: %w", err)
-	}
-	if ts == 0 {
-		ts = time.Now().UnixNano()
-	}
-	p.RenameField(msgField, "_msg")
-	processLogMessage(ts, p.Fields)
-	logjson.PutParser(p)
-
-	return true, nil
-}
-
-func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
-	for i := range fields {
-		f := &fields[i]
-		if f.Name != timeField {
-			continue
-		}
-		timestamp, err := parseISO8601Timestamp(f.Value)
-		if err != nil {
-			return 0, err
-		}
-		f.Value = ""
-		return timestamp, nil
-	}
-	return 0, nil
-}
-
-func parseISO8601Timestamp(s string) (int64, error) {
-	if s == "0" || s == "" {
-		// Special case for returning the current timestamp.
-		// It must be automatically converted to the current timestamp by the caller.
-		return 0, nil
-	}
-	t, err := time.Parse(time.RFC3339, s)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse timestamp %q: %w", s, err)
-	}
-	return t.UnixNano(), nil
-}
-
-var lineBufferPool bytesutil.ByteBufferPool
-
-var (
-	requestsTotal           = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	rowsIngestedTotal       = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
-	jsonlineRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
-)

app/vlinsert/jsonline/jsonline_test.go

@@ -1,70 +0,0 @@
-package jsonline
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"reflect"
-	"strings"
-	"testing"
-)
-
-func TestReadBulkRequestSuccess(t *testing.T) {
-	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		var timestamps []int64
-		var result string
-		processLogMessage := func(timestamp int64, fields []logstorage.Field) {
-			timestamps = append(timestamps, timestamp)
-
-			a := make([]string, len(fields))
-			for i, f := range fields {
-				a[i] = fmt.Sprintf("%q:%q", f.Name, f.Value)
-			}
-			s := "{" + strings.Join(a, ",") + "}\n"
-			result += s
-		}
-
-		// Read the request without compression
-		r := bytes.NewBufferString(data)
-		sc := bufio.NewScanner(r)
-		rows := 0
-		for {
-			ok, err := readLine(sc, timeField, msgField, processLogMessage)
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
-			if !ok {
-				break
-			}
-			rows++
-		}
-		if rows != rowsExpected {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
-		}
-
-		if !reflect.DeepEqual(timestamps, timestampsExpected) {
-			t.Fatalf("unexpected timestamps;\ngot\n%d\nwant\n%d", timestamps, timestampsExpected)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Verify non-empty data
-	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
-`
-	timeField := "@timestamp"
-	msgField := "message"
-	rowsExpected := 3
-	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-`
-	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
-}

app/vlinsert/loki/loki.go

@@ -1,58 +0,0 @@
-package loki
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// RequestHandler processes Loki insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/api/v1/push":
-		return handleInsert(r, w)
-	case "/ready":
-		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("ready"))
-		return true
-	default:
-		return false
-	}
-}
-
-// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-func handleInsert(r *http.Request, w http.ResponseWriter) bool {
-	contentType := r.Header.Get("Content-Type")
-	switch contentType {
-	case "application/json":
-		return handleJSON(r, w)
-	default:
-		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		return handleProtobuf(r, w)
-	}
-}
-
-func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// If parsed tenant is (0,0) it is likely to be default tenant
-	// Try parsing tenant from Loki headers
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		org := r.Header.Get("X-Scope-OrgID")
-		if org != "" {
-			tenantID, err := logstorage.GetTenantIDFromString(org)
-			if err != nil {
-				return nil, err
-			}
-			cp.TenantID = tenantID
-		}
-
-	}
-
-	return cp, nil
-}

app/vlinsert/loki/loki_json.go

@@ -1,205 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-)
-
-var parserPool fastjson.ParserPool
-
-func handleJSON(r *http.Request, w http.ResponseWriter) bool {
-	startTime := time.Now()
-	lokiRequestsJSONTotal.Inc()
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return true
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
-	}
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseJSONRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
-		return true
-	}
-
-	rowsIngestedJSONTotal.Add(n)
-
-	// update lokiRequestJSONDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestJSONDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestJSONDuration.UpdateDuration(startTime)
-
-	return true
-}
-
-var (
-	lokiRequestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	rowsIngestedJSONTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
-	lokiRequestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
-)
-
-func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-
-	streamsV := v.Get("streams")
-	if streamsV == nil {
-		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
-	}
-	streams, err := streamsV.Array()
-	if err != nil {
-		return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
-	}
-
-	currentTimestamp := time.Now().UnixNano()
-	var commonFields []logstorage.Field
-	rowsIngested := 0
-	for _, stream := range streams {
-		// populate common labels from `stream` dict
-		commonFields = commonFields[:0]
-		labelsV := stream.Get("stream")
-		var labels *fastjson.Object
-		if labelsV != nil {
-			o, err := labelsV.Object()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
-			}
-			labels = o
-		}
-		labels.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
-			vStr, errLocal := v.StringBytes()
-			if errLocal != nil {
-				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-				return
-			}
-			commonFields = append(commonFields, logstorage.Field{
-				Name:  bytesutil.ToUnsafeString(k),
-				Value: bytesutil.ToUnsafeString(vStr),
-			})
-		})
-		if err != nil {
-			return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
-		}
-
-		// populate messages from `values` array
-		linesV := stream.Get("values")
-		if linesV == nil {
-			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
-		}
-		lines, err := linesV.Array()
-		if err != nil {
-			return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
-		}
-
-		fields := commonFields
-		for _, line := range lines {
-			lineA, err := line.Array()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
-			}
-			if len(lineA) != 2 {
-				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
-			}
-
-			// parse timestamp
-			timestamp, err := lineA[0].StringBytes()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
-			}
-			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
-			if err != nil {
-				return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
-			}
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-
-			// parse log message
-			msg, err := lineA[1].StringBytes()
-			if err != nil {
-				return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
-			}
-
-			fields = append(fields[:len(commonFields)], logstorage.Field{
-				Name:  "_msg",
-				Value: bytesutil.ToUnsafeString(msg),
-			})
-			processLogMessage(ts, fields)
-		}
-		rowsIngested += len(lines)
-	}
-
-	return rowsIngested, nil
-}
-
-func parseLokiTimestamp(s string) (int64, error) {
-	if s == "" {
-		// Special case - an empty timestamp must be substituted with the current time by the caller.
-		return 0, nil
-	}
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		// Fall back to parsing floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, err
-		}
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		n = int64(f)
-	}
-	if n < 0 {
-		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
-	}
-	return n, nil
-}

app/vlinsert/loki/loki_json_test.go

@@ -1,130 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestParseJSONRequestFailure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		n, err := parseJSONRequest([]byte(s), func(_ int64, _ []logstorage.Field) {
-			t.Fatalf("unexpected call to parseJSONRequest callback!")
-		})
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if n != 0 {
-			t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
-		}
-	}
-	f(``)
-
-	// Invalid json
-	f(`{}`)
-	f(`[]`)
-	f(`"foo"`)
-	f(`123`)
-
-	// invalid type for `streams` item
-	f(`{"streams":123}`)
-
-	// Missing `values` item
-	f(`{"streams":[{}]}`)
-
-	// Invalid type for `values` item
-	f(`{"streams":[{"values":"foobar"}]}`)
-
-	// Invalid type for `stream` item
-	f(`{"streams":[{"stream":[],"values":[]}]}`)
-
-	// Invalid type for `values` individual item
-	f(`{"streams":[{"values":[123]}]}`)
-
-	// Invalid length of `values` individual item
-	f(`{"streams":[{"values":[[]]}]}`)
-	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
-
-	// Invalid type for timestamp inside `values` individual item
-	f(`{"streams":[{"values":[[123,"456"]}]}`)
-
-	// Invalid type for log message
-	f(`{"streams":[{"values":[["123",1234]]}]}`)
-}
-
-func TestParseJSONRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
-		t.Helper()
-		var lines []string
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
-}

app/vlinsert/loki/loki_json_timing_test.go

@@ -1,78 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func BenchmarkParseJSONRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseJSONRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		data := getJSONBody(streams, rows, labels)
-		for pb.Next() {
-			_, err := parseJSONRequest(data, func(_ int64, _ []logstorage.Field) {})
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getJSONBody(streams, rows, labels int) []byte {
-	body := append([]byte{}, `{"streams":[`...)
-	now := time.Now().UnixNano()
-	valuePrefix := fmt.Sprintf(`["%d","value_`, now)
-
-	for i := 0; i < streams; i++ {
-		body = append(body, `{"stream":{`...)
-
-		for j := 0; j < labels; j++ {
-			body = append(body, `"label_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `":"value_`...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, '"')
-			if j < labels-1 {
-				body = append(body, ',')
-			}
-
-		}
-		body = append(body, `}, "values":[`...)
-
-		for j := 0; j < rows; j++ {
-			body = append(body, valuePrefix...)
-			body = strconv.AppendInt(body, int64(j), 10)
-			body = append(body, `"]`...)
-			if j < rows-1 {
-				body = append(body, ',')
-			}
-		}
-
-		body = append(body, `]}`...)
-		if i < streams-1 {
-			body = append(body, ',')
-		}
-
-	}
-
-	body = append(body, `]}`...)
-
-	return body
-}

app/vlinsert/loki/loki_protobuf.go

@@ -1,189 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
-)
-
-var (
-	bytesBufPool bytesutil.ByteBufferPool
-	pushReqsPool sync.Pool
-)
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) bool {
-	startTime := time.Now()
-	lokiRequestsProtobufTotal.Inc()
-	wcr := writeconcurrencylimiter.GetReader(r.Body)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
-	}
-
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return true
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
-	processLogMessage := cp.GetProcessLogMessageFunc(lr)
-	n, err := parseProtobufRequest(data, processLogMessage)
-	vlstorage.MustAddRows(lr)
-	logstorage.PutLogRows(lr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
-		return true
-	}
-
-	rowsIngestedProtobufTotal.Add(n)
-
-	// update lokiRequestProtobufDuration only for successfully parsed requests
-	// There is no need in updating lokiRequestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	lokiRequestProtobufDuration.UpdateDuration(startTime)
-
-	return true
-}
-
-var (
-	lokiRequestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	rowsIngestedProtobufTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
-	lokiRequestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
-)
-
-func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
-	bb := bytesBufPool.Get()
-	defer bytesBufPool.Put(bb)
-
-	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
-	if err != nil {
-		return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
-	}
-	bb.B = buf
-
-	req := getPushRequest()
-	defer putPushRequest(req)
-
-	err = req.Unmarshal(bb.B)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse request body: %w", err)
-	}
-
-	var commonFields []logstorage.Field
-	rowsIngested := 0
-	streams := req.Streams
-	currentTimestamp := time.Now().UnixNano()
-	for i := range streams {
-		stream := &streams[i]
-		// st.Labels contains labels for the stream.
-		// Labels are same for all entries in the stream.
-		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
-		if err != nil {
-			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
-		}
-		fields := commonFields
-
-		entries := stream.Entries
-		for j := range entries {
-			entry := &entries[j]
-			fields = append(fields[:len(commonFields)], logstorage.Field{
-				Name:  "_msg",
-				Value: entry.Line,
-			})
-			ts := entry.Timestamp.UnixNano()
-			if ts == 0 {
-				ts = currentTimestamp
-			}
-			processLogMessage(ts, fields)
-		}
-		rowsIngested += len(stream.Entries)
-	}
-	return rowsIngested, nil
-}
-
-// parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
-//
-// See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
-func parsePromLabels(dst []logstorage.Field, s string) ([]logstorage.Field, error) {
-	// Make sure s is wrapped into `{...}`
-	s = strings.TrimSpace(s)
-	if len(s) < 2 {
-		return nil, fmt.Errorf("too short string to parse: %q", s)
-	}
-	if s[0] != '{' {
-		return nil, fmt.Errorf("missing `{` at the beginning of %q", s)
-	}
-	if s[len(s)-1] != '}' {
-		return nil, fmt.Errorf("missing `}` at the end of %q", s)
-	}
-	s = s[1 : len(s)-1]
-
-	for len(s) > 0 {
-		// Parse label name
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			return nil, fmt.Errorf("cannot find `=` char for label value at %s", s)
-		}
-		name := s[:n]
-		s = s[n+1:]
-
-		// Parse label value
-		qs, err := strconv.QuotedPrefix(s)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse value for label %q at %s: %w", name, s, err)
-		}
-		s = s[len(qs):]
-		value, err := strconv.Unquote(qs)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unquote value %q for label %q: %w", qs, name, err)
-		}
-
-		// Append the found field to dst.
-		dst = append(dst, logstorage.Field{
-			Name:  name,
-			Value: value,
-		})
-
-		// Check whether there are other labels remaining
-		if len(s) == 0 {
-			break
-		}
-		if !strings.HasPrefix(s, ",") {
-			return nil, fmt.Errorf("missing `,` char at %s", s)
-		}
-		s = s[1:]
-		s = strings.TrimPrefix(s, " ")
-	}
-	return dst, nil
-}
-
-func getPushRequest() *PushRequest {
-	v := pushReqsPool.Get()
-	if v == nil {
-		return &PushRequest{}
-	}
-	return v.(*PushRequest)
-}
-
-func putPushRequest(req *PushRequest) {
-	req.Reset()
-	pushReqsPool.Put(req)
-}

app/vlinsert/loki/loki_protobuf_test.go

@@ -1,171 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/golang/snappy"
-)
-
-func TestParseProtobufRequestSuccess(t *testing.T) {
-	f := func(s string, resultExpected string) {
-		t.Helper()
-		var pr PushRequest
-		n, err := parseJSONRequest([]byte(s), func(timestamp int64, fields []logstorage.Field) {
-			msg := ""
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					msg = f.Value
-				}
-			}
-			var a []string
-			for _, f := range fields {
-				if f.Name == "_msg" {
-					continue
-				}
-				item := fmt.Sprintf("%s=%q", f.Name, f.Value)
-				a = append(a, item)
-			}
-			labels := "{" + strings.Join(a, ", ") + "}"
-			pr.Streams = append(pr.Streams, Stream{
-				Labels: labels,
-				Entries: []Entry{
-					{
-						Timestamp: time.Unix(0, timestamp),
-						Line:      msg,
-					},
-				},
-			})
-		})
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if n != len(pr.Streams) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(pr.Streams), n)
-		}
-
-		data, err := pr.Marshal()
-		if err != nil {
-			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
-		}
-		encodedData := snappy.Encode(nil, data)
-
-		var lines []string
-		n, err = parseProtobufRequest(encodedData, func(timestamp int64, fields []logstorage.Field) {
-			var a []string
-			for _, f := range fields {
-				a = append(a, f.String())
-			}
-			line := fmt.Sprintf("_time:%d %s", timestamp, strings.Join(a, " "))
-			lines = append(lines, line)
-		})
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if n != len(lines) {
-			t.Fatalf("unexpected number of lines parsed; got %d; want %d", n, len(lines))
-		}
-		result := strings.Join(lines, "\n")
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	// Empty streams
-	f(`{"streams":[]}`, ``)
-	f(`{"streams":[{"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{},"values":[]}]}`, ``)
-	f(`{"streams":[{"stream":{"foo":"bar"},"values":[]}]}`, ``)
-
-	// Empty stream labels
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-	f(`{"streams":[{"stream":{},"values":[["1577836800000000001", "foo bar"]]}]}`, `_time:1577836800000000001 "_msg":"foo bar"`)
-
-	// Non-empty stream labels
-	f(`{"streams":[{"stream":{
-	"label1": "value1",
-	"label2": "value2"
-},"values":[
-	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
-	["147.78369e9", "foobar"]
-]}]}`, `_time:1577836800000000001 "label1":"value1" "label2":"value2" "_msg":"foo bar"
-_time:1477836900005000002 "label1":"value1" "label2":"value2" "_msg":"abc"
-_time:147783690000 "label1":"value1" "label2":"value2" "_msg":"foobar"`)
-
-	// Multiple streams
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "foo bar"],
-				["1577836900005000002", "abc"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000002", "yx"]
-			]
-		}
-	]
-}`, `_time:1577836800000000001 "foo":"bar" "a":"b" "_msg":"foo bar"
-_time:1577836900005000002 "foo":"bar" "a":"b" "_msg":"abc"
-_time:1877836900005000002 "x":"y" "_msg":"yx"`)
-}
-
-func TestParsePromLabelsSuccess(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		var a []string
-		for _, f := range fields {
-			a = append(a, fmt.Sprintf("%s=%q", f.Name, f.Value))
-		}
-		result := "{" + strings.Join(a, ", ") + "}"
-		if result != s {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, s)
-		}
-	}
-
-	f("{}")
-	f(`{foo="bar"}`)
-	f(`{foo="bar", baz="x", y="z"}`)
-	f(`{foo="ba\"r\\z\n", a="", b="\"\\"}`)
-}
-
-func TestParsePromLabelsFailure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-		fields, err := parsePromLabels(nil, s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(fields) > 0 {
-			t.Fatalf("unexpected non-empty fields: %s", fields)
-		}
-	}
-
-	f("")
-	f("{")
-	f(`{foo}`)
-	f(`{foo=bar}`)
-	f(`{foo="bar}`)
-	f(`{foo="ba\",r}`)
-	f(`{foo="bar" baz="aa"}`)
-	f(`foobar`)
-	f(`foo{bar="baz"}`)
-}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -1,66 +0,0 @@
-package loki
-
-import (
-	"fmt"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/golang/snappy"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, streams := range []int{5, 10} {
-		for _, rows := range []int{100, 1000} {
-			for _, labels := range []int{10, 50} {
-				b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, streams, rows, labels)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			_, err := parseProtobufRequest(body, func(_ int64, _ []logstorage.Field) {})
-			if err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(streams, rows, labels int) []byte {
-	var pr PushRequest
-
-	for i := 0; i < streams; i++ {
-		var st Stream
-
-		st.Labels = `{`
-		for j := 0; j < labels; j++ {
-			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
-			if j < labels-1 {
-				st.Labels += `,`
-			}
-		}
-		st.Labels += `}`
-
-		for j := 0; j < rows; j++ {
-			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
-		}
-
-		pr.Streams = append(pr.Streams, st)
-	}
-
-	body, _ := pr.Marshal()
-	encodedBody := snappy.Encode(nil, body)
-
-	return encodedBody
-}

app/vlinsert/loki/push_request.proto

@@ -1,38 +0,0 @@
-syntax = "proto3";
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package logproto;
-
-import "gogoproto/gogo.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
-
-message PushRequest {
-  repeated StreamAdapter streams = 1 [
-    (gogoproto.jsontag) = "streams",
-    (gogoproto.customtype) = "Stream"
-  ];
-}
-
-message StreamAdapter {
-  string labels = 1 [(gogoproto.jsontag) = "labels"];
-  repeated EntryAdapter entries = 2 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "entries"
-  ];
-  // hash contains the original hash of the stream.
-  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
-}
-
-message EntryAdapter {
-  google.protobuf.Timestamp timestamp = 1 [
-    (gogoproto.stdtime) = true,
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "ts"
-  ];
-  string line = 2 [(gogoproto.jsontag) = "line"];
-}

app/vlinsert/loki/timestamp.go

@@ -1,110 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"errors"
-	"strconv"
-	"time"
-
-	"github.com/gogo/protobuf/types"
-)
-
-const (
-	// Seconds field of the earliest valid Timestamp.
-	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	minValidSeconds = -62135596800
-	// Seconds field just after the latest valid Timestamp.
-	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	maxValidSeconds = 253402300800
-)
-
-// validateTimestamp determines whether a Timestamp is valid.
-// A valid timestamp represents a time in the range
-// [0001-01-01, 10000-01-01) and has a Nanos field
-// in the range [0, 1e9).
-//
-// If the Timestamp is valid, validateTimestamp returns nil.
-// Otherwise, it returns an error that describes
-// the problem.
-//
-// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
-func validateTimestamp(ts *types.Timestamp) error {
-	if ts == nil {
-		return errors.New("timestamp: nil Timestamp")
-	}
-	if ts.Seconds < minValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
-	}
-	if ts.Seconds >= maxValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
-	}
-	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
-		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
-	}
-	return nil
-}
-
-// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
-// but avoids the escape incurred by using fmt.Sprintf, eliminating
-// unnecessary heap allocations.
-func formatTimestamp(ts *types.Timestamp) string {
-	if ts == nil {
-		return "nil"
-	}
-
-	seconds := strconv.FormatInt(ts.Seconds, 10)
-	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
-	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
-}
-
-func sizeOfStdTime(t time.Time) int {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0
-	}
-	return ts.Size()
-}
-
-func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0, err
-	}
-	return ts.MarshalTo(data)
-}
-
-func stdTimeUnmarshal(t *time.Time, data []byte) error {
-	ts := &types.Timestamp{}
-	if err := ts.Unmarshal(data); err != nil {
-		return err
-	}
-	tt, err := timestampFromProto(ts)
-	if err != nil {
-		return err
-	}
-	*t = tt
-	return nil
-}
-
-func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
-	// Don't return the zero value on error, because corresponds to a valid
-	// timestamp. Instead return whatever time.Unix gives us.
-	var t time.Time
-	if ts == nil {
-		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
-	} else {
-		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	}
-	return t, validateTimestamp(ts)
-}
-
-func timestampProto(t time.Time) (types.Timestamp, error) {
-	ts := types.Timestamp{
-		Seconds: t.Unix(),
-		Nanos:   int32(t.Nanosecond()),
-	}
-	return ts, validateTimestamp(&ts)
-}

app/vlinsert/loki/types.go

@@ -1,481 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"fmt"
-	"io"
-	"time"
-)
-
-// Stream contains a unique labels set as a string and a set of entries for it.
-// We are not using the proto generated version but this custom one so that we
-// can improve serialization see benchmark.
-type Stream struct {
-	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
-	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
-	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
-}
-
-// Entry is a log entry with a timestamp.
-type Entry struct {
-	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
-	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Stream) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Hash != 0 {
-		i = encodeVarintPush(dAtA, i, m.Hash)
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Entries) > 0 {
-		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintPush(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x12
-		}
-	}
-	if len(m.Labels) > 0 {
-		i -= len(m.Labels)
-		copy(dAtA[i:], m.Labels)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Entry) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if len(m.Line) > 0 {
-		i -= len(m.Line)
-		copy(dAtA[i:], m.Line)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
-		i--
-		dAtA[i] = 0x12
-	}
-	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
-	if err7 != nil {
-		return 0, err7
-	}
-	i -= n7
-	i = encodeVarintPush(dAtA, i, uint64(n7))
-	i--
-	dAtA[i] = 0xa
-	return len(dAtA) - i, nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Stream) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Labels = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Entries = append(m.Entries, Entry{})
-			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
-			}
-			m.Hash = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Hash |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Entry) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Line = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Size returns the size of the serialized Stream.
-func (m *Stream) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Labels)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	if len(m.Entries) > 0 {
-		for _, e := range m.Entries {
-			l = e.Size()
-			n += 1 + l + sovPush(uint64(l))
-		}
-	}
-	if m.Hash != 0 {
-		n += 1 + sovPush(m.Hash)
-	}
-	return n
-}
-
-// Size returns the size of the serialized Entry
-func (m *Entry) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = sizeOfStdTime(m.Timestamp)
-	n += 1 + l + sovPush(uint64(l))
-	l = len(m.Line)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	return n
-}
-
-// Equal returns true if the two Streams are equal.
-func (m *Stream) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Stream)
-	if !ok {
-		that2, ok := that.(Stream)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if m.Labels != that1.Labels {
-		return false
-	}
-	if len(m.Entries) != len(that1.Entries) {
-		return false
-	}
-	for i := range m.Entries {
-		if !m.Entries[i].Equal(that1.Entries[i]) {
-			return false
-		}
-	}
-	return m.Hash == that1.Hash
-}
-
-// Equal returns true if the two Entries are equal.
-func (m *Entry) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Entry)
-	if !ok {
-		that2, ok := that.(Entry)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if !m.Timestamp.Equal(that1.Timestamp) {
-		return false
-	}
-	if m.Line != that1.Line {
-		return false
-	}
-	return true
-}

app/vlinsert/main.go

@@ -1,43 +0,0 @@
-package vlinsert
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-)
-
-// Init initializes vlinsert
-func Init() {
-}
-
-// Stop stops vlinsert
-func Stop() {
-}
-
-// RequestHandler handles insert requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if !strings.HasPrefix(path, "/insert/") {
-		// Skip requests, which do not start with /insert/, since these aren't our requests.
-		return false
-	}
-	path = strings.TrimPrefix(path, "/insert")
-	path = strings.ReplaceAll(path, "//", "/")
-
-	if path == "/jsonline" {
-		return jsonline.RequestHandler(w, r)
-	}
-	switch {
-	case strings.HasPrefix(path, "/elasticsearch/"):
-		path = strings.TrimPrefix(path, "/elasticsearch")
-		return elasticsearch.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/loki/"):
-		path = strings.TrimPrefix(path, "/loki")
-		return loki.RequestHandler(path, w, r)
-	default:
-		return false
-	}
-}

app/vlselect/logsql/logsql.go

@@ -1,87 +0,0 @@
-package logsql
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	maxSortBufferSize = flagutil.NewBytes("select.maxSortBufferSize", 1024*1024, "Query results from /select/logsql/query are automatically sorted by _time "+
-		"if their summary size doesn't exceed this value; otherwise, query results are streamed in the response without sorting; "+
-		"too big value for this flag may result in high memory usage since the sorting is performed in memory")
-)
-
-// ProcessQueryRequest handles /select/logsql/query request
-func ProcessQueryRequest(w http.ResponseWriter, r *http.Request, stopCh <-chan struct{}, cancel func()) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse query [%s]: %s", qStr, err)
-		return
-	}
-	w.Header().Set("Content-Type", "application/stream+json; charset=utf-8")
-
-	sw := getSortWriter()
-	sw.Init(w, maxSortBufferSize.IntN(), limit)
-	tenantIDs := []logstorage.TenantID{tenantID}
-	vlstorage.RunQuery(tenantIDs, q, stopCh, func(columns []logstorage.BlockColumn) {
-		if len(columns) == 0 {
-			return
-		}
-		rowsCount := len(columns[0].Values)
-
-		// skip entries with empty _stream column
-		// _stream is empty in case indexdb entry was not flushed to the storage yet
-		// skipping such entries makes the result more consistent
-		streamCol := 0
-
-		// fast path
-		// _stream column is a built-in column and it is always supposed to be at the same position
-		if len(columns) >= 2 && columns[1].Name == "_stream" {
-			streamCol = 1
-		} else {
-			for i := 1; i < len(columns); i++ {
-				if columns[i].Name == "_stream" {
-					streamCol = i
-					break
-				}
-			}
-		}
-
-		bb := blockResultPool.Get()
-		for rowIdx := 0; rowIdx < rowsCount; rowIdx++ {
-			if columns[streamCol].Values[rowIdx] == "" {
-				continue
-			}
-			WriteJSONRow(bb, columns, rowIdx)
-		}
-
-		if !sw.TryWrite(bb.B) {
-			cancel()
-		}
-
-		blockResultPool.Put(bb)
-	})
-	sw.FinalFlush()
-	putSortWriter(sw)
-}
-
-var blockResultPool bytesutil.ByteBufferPool

app/vlselect/logsql/query_response.qtpl

@@ -1,41 +0,0 @@
-{% import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-) %}
-
-{% stripspace %}
-
-// JSONRow creates JSON row from the given fields.
-{% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-{
-	{% code c := &columns[0] %}
-	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% code columns = columns[1:] %}
-	{% for colIdx := range columns %}
-		{% code c := &columns[colIdx] %}
-		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
-	{% endfor %}
-}{% newline %}
-{% endfunc %}
-
-// JSONRows prints formatted rows
-{% func JSONRows(rows [][]logstorage.Field) %}
-	{% if len(rows) == 0 %}
-		{% return %}
-	{% endif %}
-	{% for _, fields := range rows %}
-	{
-		{% if len(fields) > 0 %}
-			{% code
-				f := fields[0]
-				fields = fields[1:]
-			%}
-			{%q= f.Name %}:{%q= f.Value %}
-			{% for _, f := range fields %}
-				,{%q= f.Name %}:{%q= f.Value %}
-			{% endfor %}
-		{% endif %}
-	}{% newline %}
-	{% endfor %}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/query_response.qtpl.go

@@ -1,166 +0,0 @@
-// Code generated by qtc from "query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/query_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/query_response.qtpl:1
-import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// JSONRow creates JSON row from the given fields.
-
-//line app/vlselect/logsql/query_response.qtpl:8
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/query_response.qtpl:8
-func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:8
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:10
-	c := &columns[0]
-
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:11
-	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:12
-	columns = columns[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:13
-	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:14
-		c := &columns[colIdx]
-
-//line app/vlselect/logsql/query_response.qtpl:14
-		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:15
-		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:16
-	}
-//line app/vlselect/logsql/query_response.qtpl:16
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:17
-	qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-//line app/vlselect/logsql/query_response.qtpl:18
-func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:18
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-//line app/vlselect/logsql/query_response.qtpl:18
-func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:18
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:18
-	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:18
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:18
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:18
-}
-
-// JSONRows prints formatted rows
-
-//line app/vlselect/logsql/query_response.qtpl:21
-func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:22
-	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:23
-		return
-//line app/vlselect/logsql/query_response.qtpl:24
-	}
-//line app/vlselect/logsql/query_response.qtpl:25
-	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:25
-		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:27
-		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:29
-			f := fields[0]
-			fields = fields[1:]
-
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:32
-			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:33
-			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:33
-				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:34
-				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:35
-			}
-//line app/vlselect/logsql/query_response.qtpl:36
-		}
-//line app/vlselect/logsql/query_response.qtpl:36
-		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:37
-		qw422016.N().S(`
-`)
-//line app/vlselect/logsql/query_response.qtpl:38
-	}
-//line app/vlselect/logsql/query_response.qtpl:39
-}
-
-//line app/vlselect/logsql/query_response.qtpl:39
-func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:39
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-}
-
-//line app/vlselect/logsql/query_response.qtpl:39
-func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:39
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:39
-	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:39
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:39
-	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:39
-}

app/vlselect/logsql/sort_writer.go

@@ -1,290 +0,0 @@
-package logsql
-
-import (
-	"bytes"
-	"io"
-	"sort"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logjson"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func getSortWriter() *sortWriter {
-	v := sortWriterPool.Get()
-	if v == nil {
-		return &sortWriter{}
-	}
-	return v.(*sortWriter)
-}
-
-func putSortWriter(sw *sortWriter) {
-	sw.reset()
-	sortWriterPool.Put(sw)
-}
-
-var sortWriterPool sync.Pool
-
-// sortWriter expects JSON line stream to be written to it.
-//
-// It buffers the incoming data until its size reaches maxBufLen.
-// Then it streams the buffered data and all the incoming data to w.
-//
-// The FinalFlush() must be called when all the data is written.
-// If the buf isn't empty at FinalFlush() call, then the buffered data
-// is sorted by _time field.
-type sortWriter struct {
-	mu sync.Mutex
-	w  io.Writer
-
-	maxLines     int
-	linesWritten int
-
-	maxBufLen  int
-	buf        []byte
-	bufFlushed bool
-
-	hasErr bool
-}
-
-func (sw *sortWriter) reset() {
-	sw.w = nil
-
-	sw.maxLines = 0
-	sw.linesWritten = 0
-
-	sw.maxBufLen = 0
-	sw.buf = sw.buf[:0]
-	sw.bufFlushed = false
-	sw.hasErr = false
-}
-
-// Init initializes sw.
-//
-// If maxLines is set to positive value, then sw accepts up to maxLines
-// and then rejects all the other lines by returning false from TryWrite.
-func (sw *sortWriter) Init(w io.Writer, maxBufLen, maxLines int) {
-	sw.reset()
-
-	sw.w = w
-	sw.maxBufLen = maxBufLen
-	sw.maxLines = maxLines
-}
-
-// TryWrite writes p to sw.
-//
-// True is returned on successful write, false otherwise.
-//
-// Unsuccessful write may occur on underlying write error or when maxLines lines are already written to sw.
-func (sw *sortWriter) TryWrite(p []byte) bool {
-	sw.mu.Lock()
-	defer sw.mu.Unlock()
-
-	if sw.hasErr {
-		return false
-	}
-
-	if sw.bufFlushed {
-		if !sw.writeToUnderlyingWriterLocked(p) {
-			sw.hasErr = true
-			return false
-		}
-		return true
-	}
-
-	if len(sw.buf)+len(p) < sw.maxBufLen {
-		sw.buf = append(sw.buf, p...)
-		return true
-	}
-
-	sw.bufFlushed = true
-	if !sw.writeToUnderlyingWriterLocked(sw.buf) {
-		sw.hasErr = true
-		return false
-	}
-	sw.buf = sw.buf[:0]
-
-	if !sw.writeToUnderlyingWriterLocked(p) {
-		sw.hasErr = true
-		return false
-	}
-	return true
-}
-
-func (sw *sortWriter) writeToUnderlyingWriterLocked(p []byte) bool {
-	if len(p) == 0 {
-		return true
-	}
-	if sw.maxLines > 0 {
-		if sw.linesWritten >= sw.maxLines {
-			return false
-		}
-		var linesLeft int
-		p, linesLeft = trimLines(p, sw.maxLines-sw.linesWritten)
-		sw.linesWritten += linesLeft
-	}
-	if _, err := sw.w.Write(p); err != nil {
-		return false
-	}
-	return true
-}
-
-func trimLines(p []byte, maxLines int) ([]byte, int) {
-	if maxLines <= 0 {
-		return nil, 0
-	}
-	n := bytes.Count(p, newline)
-	if n < maxLines {
-		return p, n
-	}
-	for n >= maxLines {
-		idx := bytes.LastIndexByte(p, '\n')
-		p = p[:idx]
-		n--
-	}
-	return p[:len(p)+1], maxLines
-}
-
-var newline = []byte("\n")
-
-func (sw *sortWriter) FinalFlush() {
-	if sw.hasErr || sw.bufFlushed {
-		return
-	}
-
-	rs := getRowsSorter()
-	rs.parseRows(sw.buf)
-	rs.sort()
-
-	rows := rs.rows
-	if sw.maxLines > 0 && len(rows) > sw.maxLines {
-		rows = rows[:sw.maxLines]
-	}
-	WriteJSONRows(sw.w, rows)
-
-	putRowsSorter(rs)
-}
-
-func getRowsSorter() *rowsSorter {
-	v := rowsSorterPool.Get()
-	if v == nil {
-		return &rowsSorter{}
-	}
-	return v.(*rowsSorter)
-}
-
-func putRowsSorter(rs *rowsSorter) {
-	rs.reset()
-	rowsSorterPool.Put(rs)
-}
-
-var rowsSorterPool sync.Pool
-
-type rowsSorter struct {
-	buf       []byte
-	fieldsBuf []logstorage.Field
-	rows      [][]logstorage.Field
-	times     []string
-}
-
-func (rs *rowsSorter) reset() {
-	rs.buf = rs.buf[:0]
-
-	fieldsBuf := rs.fieldsBuf
-	for i := range fieldsBuf {
-		fieldsBuf[i].Reset()
-	}
-	rs.fieldsBuf = fieldsBuf[:0]
-
-	rows := rs.rows
-	for i := range rows {
-		rows[i] = nil
-	}
-	rs.rows = rows[:0]
-
-	times := rs.times
-	for i := range times {
-		times[i] = ""
-	}
-	rs.times = times[:0]
-}
-
-func (rs *rowsSorter) parseRows(src []byte) {
-	rs.reset()
-
-	buf := rs.buf
-	fieldsBuf := rs.fieldsBuf
-	rows := rs.rows
-	times := rs.times
-
-	p := logjson.GetParser()
-	for len(src) > 0 {
-		var line []byte
-		n := bytes.IndexByte(src, '\n')
-		if n < 0 {
-			line = src
-			src = nil
-		} else {
-			line = src[:n]
-			src = src[n+1:]
-		}
-		if len(line) == 0 {
-			continue
-		}
-
-		if err := p.ParseLogMessage(line); err != nil {
-			logger.Panicf("BUG: unexpected invalid JSON line: %s", err)
-		}
-
-		timeValue := ""
-		fieldsBufLen := len(fieldsBuf)
-		for _, f := range p.Fields {
-			bufLen := len(buf)
-			buf = append(buf, f.Name...)
-			name := bytesutil.ToUnsafeString(buf[bufLen:])
-
-			bufLen = len(buf)
-			buf = append(buf, f.Value...)
-			value := bytesutil.ToUnsafeString(buf[bufLen:])
-
-			fieldsBuf = append(fieldsBuf, logstorage.Field{
-				Name:  name,
-				Value: value,
-			})
-
-			if name == "_time" {
-				timeValue = value
-			}
-		}
-		rows = append(rows, fieldsBuf[fieldsBufLen:])
-		times = append(times, timeValue)
-	}
-	logjson.PutParser(p)
-
-	rs.buf = buf
-	rs.fieldsBuf = fieldsBuf
-	rs.rows = rows
-	rs.times = times
-}
-
-func (rs *rowsSorter) Len() int {
-	return len(rs.rows)
-}
-
-func (rs *rowsSorter) Less(i, j int) bool {
-	times := rs.times
-	return times[i] < times[j]
-}
-
-func (rs *rowsSorter) Swap(i, j int) {
-	times := rs.times
-	rows := rs.rows
-	times[i], times[j] = times[j], times[i]
-	rows[i], rows[j] = rows[j], rows[i]
-}
-
-func (rs *rowsSorter) sort() {
-	sort.Sort(rs)
-}

app/vlselect/logsql/sort_writer_test.go

@@ -1,46 +0,0 @@
-package logsql
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-)
-
-func TestSortWriter(t *testing.T) {
-	f := func(maxBufLen, maxLines int, data string, expectedResult string) {
-		t.Helper()
-
-		var bb bytes.Buffer
-		sw := getSortWriter()
-		sw.Init(&bb, maxBufLen, maxLines)
-		for _, s := range strings.Split(data, "\n") {
-			if !sw.TryWrite([]byte(s + "\n")) {
-				break
-			}
-		}
-		sw.FinalFlush()
-		putSortWriter(sw)
-
-		result := bb.String()
-		if result != expectedResult {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, expectedResult)
-		}
-	}
-
-	f(100, 0, "", "")
-	f(100, 0, "{}", "{}\n")
-
-	data := `{"_time":"def","_msg":"xxx"}
-{"_time":"abc","_msg":"foo"}`
-	resultExpected := `{"_time":"abc","_msg":"foo"}
-{"_time":"def","_msg":"xxx"}
-`
-	f(100, 0, data, resultExpected)
-	f(10, 0, data, data+"\n")
-
-	// Test with the maxLines
-	f(100, 1, data, `{"_time":"abc","_msg":"foo"}`+"\n")
-	f(10, 1, data, `{"_time":"def","_msg":"xxx"}`+"\n")
-	f(10, 2, data, data+"\n")
-	f(100, 2, data, resultExpected)
-}

app/vlselect/main.go

@@ -1,174 +0,0 @@
-package vlselect
-
-import (
-	"context"
-	"embed"
-	"flag"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration")
-	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
-		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution")
-)
-
-func getDefaultMaxConcurrentRequests() int {
-	n := cgroup.AvailableCPUs()
-	if n <= 4 {
-		n *= 2
-	}
-	if n > 16 {
-		// A single request can saturate all the CPU cores, so there is no sense
-		// in allowing higher number of concurrent requests - they will just contend
-		// for unavailable CPU time.
-		n = 16
-	}
-	return n
-}
-
-// Init initializes vlselect
-func Init() {
-	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
-}
-
-// Stop stops vlselect
-func Stop() {
-}
-
-var concurrencyLimitCh chan struct{}
-
-var (
-	concurrencyLimitReached = metrics.NewCounter(`vl_concurrent_select_limit_reached_total`)
-	concurrencyLimitTimeout = metrics.NewCounter(`vl_concurrent_select_limit_timeout_total`)
-
-	_ = metrics.NewGauge(`vl_concurrent_select_capacity`, func() float64 {
-		return float64(cap(concurrencyLimitCh))
-	})
-	_ = metrics.NewGauge(`vl_concurrent_select_current`, func() float64 {
-		return float64(len(concurrencyLimitCh))
-	})
-)
-
-//go:embed vmui
-var vmuiFiles embed.FS
-
-var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
-
-// RequestHandler handles select requests for VictoriaLogs
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if !strings.HasPrefix(path, "/select/") {
-		// Skip requests, which do not start with /select/, since these aren't our requests.
-		return false
-	}
-	path = strings.TrimPrefix(path, "/select")
-	path = strings.ReplaceAll(path, "//", "/")
-
-	if path == "/vmui" {
-		// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
-		// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
-		// is hidden behind vmauth or similar proxy.
-		_ = r.ParseForm()
-		newURL := "vmui/?" + r.Form.Encode()
-		httpserver.Redirect(w, newURL)
-		return true
-	}
-	if strings.HasPrefix(path, "/vmui/") {
-		if strings.HasPrefix(path, "/vmui/static/") {
-			// Allow clients caching static contents for long period of time, since it shouldn't change over time.
-			// Path to static contents (such as js and css) must be changed whenever its contents is changed.
-			// See https://developer.chrome.com/docs/lighthouse/performance/uses-long-cache-ttl/
-			w.Header().Set("Cache-Control", "max-age=31536000")
-		}
-		r.URL.Path = path
-		vmuiFileServer.ServeHTTP(w, r)
-		return true
-	}
-
-	// Limit the number of concurrent queries, which can consume big amounts of CPU.
-	startTime := time.Now()
-	ctx := r.Context()
-	stopCh := ctx.Done()
-	select {
-	case concurrencyLimitCh <- struct{}{}:
-		defer func() { <-concurrencyLimitCh }()
-	default:
-		// Sleep for a while until giving up. This should resolve short bursts in requests.
-		concurrencyLimitReached.Inc()
-		d := getMaxQueryDuration(r)
-		if d > *maxQueueDuration {
-			d = *maxQueueDuration
-		}
-		t := timerpool.Get(d)
-		select {
-		case concurrencyLimitCh <- struct{}{}:
-			timerpool.Put(t)
-			defer func() { <-concurrencyLimitCh }()
-		case <-stopCh:
-			timerpool.Put(t)
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
-				time.Since(startTime).Seconds(), remoteAddr, requestURI)
-			return true
-		case <-t.C:
-			timerpool.Put(t)
-			concurrencyLimitTimeout.Inc()
-			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+
-					"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
-					"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests",
-					d.Seconds(), *maxConcurrentRequests, maxQueueDuration),
-				StatusCode: http.StatusServiceUnavailable,
-			}
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-	}
-
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	defer cancel()
-	stopCh = ctxWithCancel.Done()
-
-	switch {
-	case path == "/logsql/query":
-		logsqlQueryRequests.Inc()
-		httpserver.EnableCORS(w, r)
-		logsql.ProcessQueryRequest(w, r, stopCh, cancel)
-		return true
-	default:
-		return false
-	}
-}
-
-// getMaxQueryDuration returns the maximum duration for query from r.
-func getMaxQueryDuration(r *http.Request) time.Duration {
-	dms, err := httputils.GetDuration(r, "timeout", 0)
-	if err != nil {
-		dms = 0
-	}
-	d := time.Duration(dms) * time.Millisecond
-	if d <= 0 || d > *maxQueryDuration {
-		d = *maxQueryDuration
-	}
-	return d
-}
-
-var (
-	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-)

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +0,0 @@
-{
-  "files": {
-    "main.css": "./static/css/main.bc07cc78.css",
-    "main.js": "./static/js/main.034044a7.js",
-    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.10add6e7bdf0f1d98cf7.md",
-    "index.html": "./index.html"
-  },
-  "entrypoints": [
-    "static/css/main.bc07cc78.css",
-    "static/js/main.034044a7.js"
-  ]
-}

app/vlselect/vmui/index.html

@@ -1 +0,0 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.034044a7.js"></script><link href="./static/css/main.bc07cc78.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/manifest.json

@@ -1,20 +0,0 @@
-{
-  "short_name": "Victoria Metrics UI",
-  "name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
-  "icons": [
-    {
-      "src": "favicon-32x32.png",
-      "sizes": "32x32",
-      "type": "image/png"
-    },
-    {
-      "src": "apple-touch-icon.png",
-      "type": "image/png",
-      "sizes": "192x192"
-    }
-  ],
-  "start_url": ".",
-  "display": "standalone",
-  "theme_color": "#000000",
-  "background_color": "#ffffff"
-}

app/vlselect/vmui/robots.txt

@@ -1,3 +0,0 @@
-# https://www.robotstxt.org/robotstxt.html
-User-agent: *
-Disallow:

app/vlselect/vmui/static/js/main.034044a7.js.LICENSE.txt

@@ -1,38 +0,0 @@
-/*!
-	Copyright (c) 2018 Jed Watson.
-	Licensed under the MIT License (MIT), see
-	http://jedwatson.github.io/classnames
-*/
-
-/**
- * @remix-run/router v1.15.1
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router DOM v6.22.1
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router v6.22.1
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */

app/vlstorage/main.go

@@ -1,215 +0,0 @@
-package vlstorage
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
-		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
-	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#retention")
-	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory with the VictoriaLogs data; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/#storage")
-	inmemoryDataFlushInterval = flag.Duration("inmemoryDataFlushInterval", 5*time.Second, "The interval for guaranteed saving of in-memory data to disk. "+
-		"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
-		"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
-		"Smaller intervals increase disk IO load. Minimum supported value is 1s")
-	logNewStreams = flag.Bool("logNewStreams", false, "Whether to log creation of new streams; this can be useful for debugging of high cardinality issues with log streams; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows")
-	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
-		"see https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/ ; see also -logNewStreams")
-	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
-		"the storage stops accepting new data")
-)
-
-// Init initializes vlstorage.
-//
-// Stop must be called when vlstorage is no longer needed
-func Init() {
-	if strg != nil {
-		logger.Panicf("BUG: Init() has been already called")
-	}
-
-	if retentionPeriod.Duration() < 24*time.Hour {
-		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
-	}
-	cfg := &logstorage.StorageConfig{
-		Retention:             retentionPeriod.Duration(),
-		FlushInterval:         *inmemoryDataFlushInterval,
-		FutureRetention:       futureRetention.Duration(),
-		LogNewStreams:         *logNewStreams,
-		LogIngestedRows:       *logIngestedRows,
-		MinFreeDiskSpaceBytes: minFreeDiskSpaceBytes.N,
-	}
-	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
-	startTime := time.Now()
-	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
-
-	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
-	logger.Infof("successfully opened storage in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
-		time.Since(startTime).Seconds(), ss.FileParts, ss.FileBlocks, ss.FileRowsCount, ss.CompressedFileSize)
-	storageMetrics = initStorageMetrics(strg)
-
-	metrics.RegisterSet(storageMetrics)
-}
-
-// Stop stops vlstorage.
-func Stop() {
-	metrics.UnregisterSet(storageMetrics)
-	storageMetrics = nil
-
-	strg.MustClose()
-	strg = nil
-}
-
-var strg *logstorage.Storage
-var storageMetrics *metrics.Set
-
-// CanWriteData returns non-nil error if it cannot write data to vlstorage.
-func CanWriteData() error {
-	if strg.IsReadOnly() {
-		return &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
-				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
-			StatusCode: http.StatusTooManyRequests,
-		}
-	}
-	return nil
-}
-
-// MustAddRows adds lr to vlstorage
-//
-// It is advised to call CanWriteData() before calling MustAddRows()
-func MustAddRows(lr *logstorage.LogRows) {
-	strg.MustAddRows(lr)
-}
-
-// RunQuery runs the given q and calls processBlock for the returned data blocks
-func RunQuery(tenantIDs []logstorage.TenantID, q *logstorage.Query, stopCh <-chan struct{}, processBlock func(columns []logstorage.BlockColumn)) {
-	strg.RunQuery(tenantIDs, q, stopCh, processBlock)
-}
-
-func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
-	ssCache := &logstorage.StorageStats{}
-	var ssCacheLock sync.Mutex
-	var lastUpdateTime time.Time
-
-	m := func() *logstorage.StorageStats {
-		ssCacheLock.Lock()
-		defer ssCacheLock.Unlock()
-		if time.Since(lastUpdateTime) < time.Second {
-			return ssCache
-		}
-		var ss logstorage.StorageStats
-		strg.UpdateStats(&ss)
-		ssCache = &ss
-		lastUpdateTime = time.Now()
-		return ssCache
-	}
-
-	ms := metrics.NewSet()
-
-	ms.NewGauge(fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), func() float64 {
-		return float64(fs.MustGetFreeSpace(*storageDataPath))
-	})
-	ms.NewGauge(fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), func() float64 {
-		if m().IsReadOnly {
-			return 1
-		}
-		return 0
-	})
-
-	ms.NewGauge(`vl_active_merges{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryActiveMerges)
-	})
-	ms.NewGauge(`vl_merges_total{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryMergesTotal)
-	})
-	ms.NewGauge(`vl_active_merges{type="file"}`, func() float64 {
-		return float64(m().FileActiveMerges)
-	})
-	ms.NewGauge(`vl_merges_total{type="file"}`, func() float64 {
-		return float64(m().FileMergesTotal)
-	})
-
-	ms.NewGauge(`vl_storage_rows{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryRowsCount)
-	})
-	ms.NewGauge(`vl_storage_rows{type="file"}`, func() float64 {
-		return float64(m().FileRowsCount)
-	})
-	ms.NewGauge(`vl_storage_parts{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryParts)
-	})
-	ms.NewGauge(`vl_storage_parts{type="file"}`, func() float64 {
-		return float64(m().FileParts)
-	})
-	ms.NewGauge(`vl_storage_blocks{type="inmemory"}`, func() float64 {
-		return float64(m().InmemoryBlocks)
-	})
-	ms.NewGauge(`vl_storage_blocks{type="file"}`, func() float64 {
-		return float64(m().FileBlocks)
-	})
-
-	ms.NewGauge(`vl_partitions`, func() float64 {
-		return float64(m().PartitionsCount)
-	})
-	ms.NewGauge(`vl_streams_created_total`, func() float64 {
-		return float64(m().StreamsCreatedTotal)
-	})
-
-	ms.NewGauge(`vl_indexdb_rows`, func() float64 {
-		return float64(m().IndexdbItemsCount)
-	})
-	ms.NewGauge(`vl_indexdb_parts`, func() float64 {
-		return float64(m().IndexdbPartsCount)
-	})
-	ms.NewGauge(`vl_indexdb_blocks`, func() float64 {
-		return float64(m().IndexdbBlocksCount)
-	})
-
-	ms.NewGauge(`vl_data_size_bytes{type="indexdb"}`, func() float64 {
-		return float64(m().IndexdbSizeBytes)
-	})
-	ms.NewGauge(`vl_data_size_bytes{type="storage"}`, func() float64 {
-		dm := m()
-		return float64(dm.CompressedInmemorySize + dm.CompressedFileSize)
-	})
-
-	ms.NewGauge(`vl_compressed_data_size_bytes{type="inmemory"}`, func() float64 {
-		return float64(m().CompressedInmemorySize)
-	})
-	ms.NewGauge(`vl_compressed_data_size_bytes{type="file"}`, func() float64 {
-		return float64(m().CompressedFileSize)
-	})
-	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="inmemory"}`, func() float64 {
-		return float64(m().UncompressedInmemorySize)
-	})
-	ms.NewGauge(`vl_uncompressed_data_size_bytes{type="file"}`, func() float64 {
-		return float64(m().UncompressedFileSize)
-	})
-
-	ms.NewGauge(`vl_rows_dropped_total{reason="too_big_timestamp"}`, func() float64 {
-		return float64(m().RowsDroppedTooBigTimestamp)
-	})
-	ms.NewGauge(`vl_rows_dropped_total{reason="too_small_timestamp"}`, func() float64 {
-		return float64(m().RowsDroppedTooSmallTimestamp)
-	})
-
-	return ms
-}

app/vmagent/Makefile

@@ -85,9 +85,6 @@ vmagent-linux-arm64:
 vmagent-linux-ppc64le:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
 
-vmagent-linux-s390x:
-	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
 vmagent-linux-386:
 	APP_NAME=vmagent CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmagent/csvimport/request_handler.go

@@ -9,8 +9,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -26,8 +26,10 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []parser.Row) error {
-		return insertRows(at, rows, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req, func(rows []parser.Row) error {
+			return insertRows(at, rows, extraLabels)
+		})
 	})
 }
 
@@ -65,9 +67,7 @@ func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.L
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(len(rows))

app/vmagent/datadog/request_handler.go

@@ -0,0 +1,92 @@
+package datadog
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadog"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadog"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadog"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadog"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
+//
+// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := parserCommon.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	return writeconcurrencylimiter.Do(func() error {
+		ce := req.Header.Get("Content-Encoding")
+		return parser.ParseStream(req.Body, ce, func(series []parser.Series) error {
+			return insertRows(at, series, extraLabels)
+		})
+	})
+}
+
+func insertRows(at *auth.Token, series []parser.Series, extraLabels []prompbmarshal.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := 0
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range series {
+		ss := &series[i]
+		rowsTotal += len(ss.Points)
+		labelsLen := len(labels)
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "__name__",
+			Value: ss.Metric,
+		})
+		labels = append(labels, prompbmarshal.Label{
+			Name:  "host",
+			Value: ss.Host,
+		})
+		for _, tag := range ss.Tags {
+			name, value := parser.SplitTag(tag)
+			if name == "host" {
+				name = "exported_host"
+			}
+			labels = append(labels, prompbmarshal.Label{
+				Name:  name,
+				Value: value,
+			})
+		}
+		labels = append(labels, extraLabels...)
+		samplesLen := len(samples)
+		for _, pt := range ss.Points {
+			samples = append(samples, prompbmarshal.Sample{
+				Timestamp: pt.Timestamp(),
+				Value:     pt.Value(),
+			})
+		}
+		tssDst = append(tssDst, prompbmarshal.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	remotewrite.Push(at, &ctx.WriteRequest)
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmagent/datadogsketches/request_handler.go

@@ -1,95 +0,0 @@
-package datadogsketches
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogsketches"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogsketches"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogsketches"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(sketches []*datadogsketches.Sketch) error {
-		return insertRows(at, sketches, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for _, sketch := range sketches {
-		ms := sketch.ToSummary()
-		for _, m := range ms {
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: m.Name,
-			})
-			for _, label := range m.Labels {
-				labels = append(labels, prompbmarshal.Label{
-					Name:  label.Name,
-					Value: label.Value,
-				})
-			}
-			for _, tag := range sketch.Tags {
-				name, value := datadogutils.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
-				labels = append(labels, prompbmarshal.Label{
-					Name:  name,
-					Value: value,
-				})
-			}
-			labels = append(labels, extraLabels...)
-			samplesLen := len(samples)
-			for _, p := range m.Points {
-				samples = append(samples, prompbmarshal.Sample{
-					Timestamp: p.Timestamp,
-					Value:     p.Value,
-				})
-			}
-			rowsTotal += len(m.Points)
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[samplesLen:],
-			})
-		}
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/datadogv1/request_handler.go

@@ -1,99 +0,0 @@
-package datadogv1
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv1"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv1"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv1"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, func(series []datadogv1.Series) error {
-		return insertRows(at, series, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range series {
-		ss := &series[i]
-		rowsTotal += len(ss.Points)
-		labelsLen := len(labels)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: ss.Metric,
-		})
-		if ss.Host != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "host",
-				Value: ss.Host,
-			})
-		}
-		if ss.Device != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "device",
-				Value: ss.Device,
-			})
-		}
-		for _, tag := range ss.Tags {
-			name, value := datadogutils.SplitTag(tag)
-			if name == "host" {
-				name = "exported_host"
-			}
-			labels = append(labels, prompbmarshal.Label{
-				Name:  name,
-				Value: value,
-			})
-		}
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		for _, pt := range ss.Points {
-			samples = append(samples, prompbmarshal.Sample{
-				Timestamp: pt.Timestamp(),
-				Value:     pt.Value(),
-			})
-		}
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/datadogv2/request_handler.go

@@ -1,102 +0,0 @@
-package datadogv2
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="datadogv2"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="datadogv2"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="datadogv2"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for DataDog POST /api/v2/series request.
-//
-// See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ct := req.Header.Get("Content-Type")
-	ce := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, ce, ct, func(series []datadogv2.Series) error {
-		return insertRows(at, series, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range series {
-		ss := &series[i]
-		rowsTotal += len(ss.Points)
-		labelsLen := len(labels)
-		labels = append(labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: ss.Metric,
-		})
-		for _, rs := range ss.Resources {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  rs.Type,
-				Value: rs.Name,
-			})
-		}
-		if ss.SourceTypeName != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "source_type_name",
-				Value: ss.SourceTypeName,
-			})
-		}
-		for _, tag := range ss.Tags {
-			name, value := datadogutils.SplitTag(tag)
-			if name == "host" {
-				name = "exported_host"
-			}
-			labels = append(labels, prompbmarshal.Label{
-				Name:  name,
-				Value: value,
-			})
-		}
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		for _, pt := range ss.Points {
-			samples = append(samples, prompbmarshal.Sample{
-				Timestamp: pt.Timestamp * 1000,
-				Value:     pt.Value,
-			})
-		}
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/graphite/request_handler.go

@@ -5,10 +5,9 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/graphite/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -21,12 +20,12 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
-		return insertRows(nil, rows)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
 	})
 }
 
-func insertRows(at *auth.Token, rows []parser.Row) error {
+func insertRows(rows []parser.Row) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -59,9 +58,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil

app/vmagent/influx/request_handler.go

@@ -15,14 +15,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	measurementFieldSeparator = flag.String("influxMeasurementFieldSeparator", "_", "Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol")
-	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metric name if InfluxDB line contains only a single field")
+	skipSingleField           = flag.Bool("influxSkipSingleField", false, "Uses '{measurement}' instead of '{measurement}{separator}{field_name}' for metic name if InfluxDB line contains only a single field")
 	skipMeasurement           = flag.Bool("influxSkipMeasurement", false, "Uses '{field_name}' as a metric name while ignoring '{measurement}' and '-influxMeasurementFieldSeparator'")
 	dbLabel                   = flag.String("influxDBLabel", "db", "Default label for the DB name sent over '?db={db_name}' query parameter")
 )
@@ -36,9 +36,11 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
-		return insertRows(at, db, rows, nil)
+func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+			return insertRows(nil, db, rows, nil)
+		})
 	})
 }
 
@@ -50,13 +52,15 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	q := req.URL.Query()
-	precision := q.Get("precision")
-	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
-	db := q.Get("db")
-	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
-		return insertRows(at, db, rows, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+		q := req.URL.Query()
+		precision := q.Get("precision")
+		// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
+		db := q.Get("db")
+		return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+			return insertRows(at, db, rows, extraLabels)
+		})
 	})
 }
 
@@ -130,9 +134,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
 	ctx.ctx.Labels = labels
 	ctx.ctx.Samples = samples
 	ctx.commonLabels = commonLabels
-	if !remotewrite.TryPush(at, &ctx.ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.ctx.WriteRequest)
 	rowsInserted.Add(rowsTotal)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(rowsTotal)

app/vmagent/main.go

@@ -8,17 +8,14 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv2"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/native"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/prometheusimport"
@@ -40,40 +37,25 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -tls and -httpListenAddr.useProxyProtocol")
-	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
-		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
-		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
-		"See also -influxListenAddr.useProxyProtocol")
-	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. "+
-		"See also -graphiteListenAddr.useProxyProtocol")
-	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpenTSDB metrics. "+
+		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
-		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
-	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpenTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
-		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
-	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
-		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
-	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
-		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
+		"Usually :4242 must be set. Doesn't work if empty")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
 
@@ -98,71 +80,66 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
+	pushmetrics.Init()
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
 			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("-promscrape.config is ok; exiting with 0 status code")
+		logger.Infof("-promscrape.config is ok; exitting with 0 status code")
 		return
 	}
 	if *dryRun {
-		if err := promscrape.CheckConfig(); err != nil {
-			logger.Fatalf("error when checking -promscrape.config: %s", err)
-		}
 		if err := remotewrite.CheckRelabelConfigs(); err != nil {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
-		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
+		if err := promscrape.CheckConfig(); err != nil {
+			logger.Fatalf("error when checking -promscrape.config: %s", err)
 		}
-		logger.Infof("all the configs are ok; exiting with 0 status code")
+		logger.Infof("all the configs are ok; exitting with 0 status code")
 		return
 	}
 
-	listenAddrs := *httpListenAddrs
-	if len(listenAddrs) == 0 {
-		listenAddrs = []string{":8429"}
-	}
-	logger.Infof("starting vmagent at %q...", listenAddrs)
+	logger.Infof("starting vmagent at %q...", *httpListenAddr)
 	startTime := time.Now()
-	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
+	writeconcurrencylimiter.Init()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, false)
+		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+			return influx.InsertHandlerForReader(r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
-		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, httpInsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, httpInsertHandler)
 	}
 
-	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
+	promscrape.Init(remotewrite.Push)
 
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	if len(*httpListenAddr) > 0 {
+		go httpserver.Serve(*httpListenAddr, requestHandler)
+	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
-	pushmetrics.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	remotewrite.StopIngestionRateLimiter()
-	pushmetrics.Stop()
 
 	startTime = time.Now()
-	logger.Infof("gracefully shutting down webservice at %q", listenAddrs)
-	if err := httpserver.Stop(listenAddrs); err != nil {
-		logger.Fatalf("cannot stop the webservice: %s", err)
+	if len(*httpListenAddr) > 0 {
+		logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr)
+		if err := httpserver.Stop(*httpListenAddr); err != nil {
+			logger.Fatalf("cannot stop the webservice: %s", err)
+		}
+		logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 	}
-	logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
 
 	promscrape.Stop()
 
@@ -215,12 +192,12 @@ func getAuthTokenFromPath(path string) (*auth.Token, error) {
 	if p.Suffix != "opentsdb/api/put" {
 		return nil, fmt.Errorf("unsupported path requested: %q; expecting 'opentsdb/api/put'", p.Suffix)
 	}
-	return auth.NewTokenPossibleMultitenant(p.AuthToken)
+	return auth.NewToken(p.AuthToken)
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if r.URL.Path == "/" {
-		if r.Method != http.MethodGet {
+		if r.Method != "GET" {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
@@ -230,7 +207,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
 			{"service-discovery", "labels before and after relabeling for discovered targets"},
-			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
 			{"metrics", "available service metrics"},
@@ -241,33 +217,13 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
-	if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus") || strings.HasPrefix(path, "/api/v1/import/prometheus") {
-		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(nil, r); err != nil {
-			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		statusCode := http.StatusNoContent
-		if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus/metrics/job/") ||
-			strings.HasPrefix(path, "/api/v1/import/prometheus/metrics/job/") {
-			// Return 200 status code for pushgateway requests.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
-			statusCode = http.StatusOK
-		}
-		w.WriteHeader(statusCode)
-		return true
-	}
-	if strings.HasPrefix(path, "/datadog/") {
+	if strings.HasPrefix(path, "datadog/") {
 		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
 		path = strings.TrimSuffix(path, "/")
 	}
 	switch path {
-	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
-		if common.HandleVMProtoServerHandshake(w, r) {
-			return true
-		}
+	case "/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(nil, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -276,7 +232,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import", "/api/v1/import":
+	case "/api/v1/import":
 		vmimportRequests.Inc()
 		if err := vmimport.InsertHandler(nil, r); err != nil {
 			vmimportErrors.Inc()
@@ -285,7 +241,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import/csv", "/api/v1/import/csv":
+	case "/api/v1/import/csv":
 		csvimportRequests.Inc()
 		if err := csvimport.InsertHandler(nil, r); err != nil {
 			csvimportErrors.Inc()
@@ -294,7 +250,16 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/prometheus/api/v1/import/native", "/api/v1/import/native":
+	case "/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(nil, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
+	case "/api/v1/import/native":
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(nil, r); err != nil {
 			nativeimportErrors.Inc()
@@ -303,7 +268,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/influx/write", "/influx/api/v2/write", "/write", "/api/v2/write":
+	case "/write", "/api/v2/write":
 		influxWriteRequests.Inc()
 		if err := influx.InsertHandlerForHTTP(nil, r); err != nil {
 			influxWriteErrors.Inc()
@@ -312,57 +277,14 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
-	case "/influx/query", "/query":
+	case "/query":
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(nil, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "/newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "/newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(nil, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	case "/datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "/datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogv2WriteErrors.Inc()
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(nil, r); err != nil {
+			datadogWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -371,15 +293,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "/datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(nil, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "/datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -403,29 +316,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprintf(w, `{}`)
 		return true
-	case "/prometheus/targets", "/targets":
+	case "/targets":
 		promscrapeTargetsRequests.Inc()
 		promscrape.WriteHumanReadableTargetsStatus(w, r)
 		return true
-	case "/prometheus/service-discovery", "/service-discovery":
+	case "/service-discovery":
 		promscrapeServiceDiscoveryRequests.Inc()
 		promscrape.WriteServiceDiscovery(w, r)
 		return true
-	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
-		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
-		return true
-	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
-		promscrapeTargetRelabelDebugRequests.Inc()
-		promscrape.WriteTargetRelabelDebug(w, r)
-		return true
-	case "/prometheus/api/v1/targets", "/api/v1/targets":
-		promscrapeAPIV1TargetsRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		state := r.FormValue("state")
-		promscrape.WriteAPIV1Targets(w, state)
-		return true
-	case "/prometheus/target_response", "/target_response":
+	case "/target_response":
 		promscrapeTargetResponseRequests.Inc()
 		if err := promscrape.WriteTargetResponse(w, r); err != nil {
 			promscrapeTargetResponseErrors.Inc()
@@ -433,17 +332,27 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		return true
-	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+	case "/config":
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeConfigRequests.Inc()
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		promscrape.WriteConfigData(w)
 		return true
-	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
+	case "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+			err := &httpserver.ErrorWithStatusCode{
+				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
@@ -452,16 +361,19 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promscrape.WriteConfigData(&bb)
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
-	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
-			return true
-		}
+	case "/api/v1/targets":
+		promscrapeAPIV1TargetsRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		state := r.FormValue("state")
+		promscrape.WriteAPIV1Targets(w, state)
+		return true
+	case "/-/reload":
 		promscrapeConfigReloadRequests.Inc()
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
 	case "/ready":
-		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
+		if rdy := atomic.LoadInt32(&promscrape.PendingScrapeConfigs); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
 			http.Error(w, errMsg, http.StatusTooEarly)
 		} else {
@@ -497,23 +409,13 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
 	}
-	if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus") {
-		prometheusimportRequests.Inc()
-		if err := prometheusimport.InsertHandler(at, r); err != nil {
-			prometheusimportErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(http.StatusNoContent)
-		return true
-	}
 	if strings.HasPrefix(p.Suffix, "datadog/") {
 		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
 		p.Suffix = strings.TrimSuffix(p.Suffix, "/")
 	}
 	switch p.Suffix {
-	case "prometheus/", "prometheus", "prometheus/api/v1/write", "prometheus/api/v1/push":
+	case "prometheus/", "prometheus", "prometheus/api/v1/write":
 		prometheusWriteRequests.Inc()
 		if err := promremotewrite.InsertHandler(at, r); err != nil {
 			prometheusWriteErrors.Inc()
@@ -540,6 +442,15 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "prometheus/api/v1/import/prometheus":
+		prometheusimportRequests.Inc()
+		if err := prometheusimport.InsertHandler(at, r); err != nil {
+			prometheusimportErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	case "prometheus/api/v1/import/native":
 		nativeimportRequests.Inc()
 		if err := native.InsertHandler(at, r); err != nil {
@@ -562,52 +473,10 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
-		opentelemetryPushRequests.Inc()
-		if err := opentelemetry.InsertHandler(at, r); err != nil {
-			opentelemetryPushErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		firehose.WriteSuccessResponse(w, r)
-		return true
-	case "newrelic":
-		newrelicCheckRequest.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "newrelic/inventory/deltas":
-		newrelicInventoryRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"payload":{"version": 1, "state": {}, "reset": "false"}}`)
-		return true
-	case "newrelic/infra/v2/metrics/events/bulk":
-		newrelicWriteRequests.Inc()
-		if err := newrelic.InsertHandlerForHTTP(at, r); err != nil {
-			newrelicWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	case "datadog/api/v1/series":
-		datadogv1WriteRequests.Inc()
-		if err := datadogv1.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv1WriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
-	case "datadog/api/v2/series":
-		datadogv2WriteRequests.Inc()
-		if err := datadogv2.InsertHandlerForHTTP(at, r); err != nil {
-			datadogv2WriteErrors.Inc()
+		datadogWriteRequests.Inc()
+		if err := datadog.InsertHandlerForHTTP(at, r); err != nil {
+			datadogWriteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
@@ -615,15 +484,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		w.WriteHeader(202)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
-	case "datadog/api/beta/sketches":
-		datadogsketchesWriteRequests.Inc()
-		if err := datadogsketches.InsertHandlerForHTTP(at, r); err != nil {
-			datadogsketchesWriteErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		w.WriteHeader(202)
-		return true
 	case "datadog/api/v1/validate":
 		datadogValidateRequests.Inc()
 		// See https://docs.datadoghq.com/api/latest/authentication/#validate-api-key
@@ -674,36 +534,17 @@ var (
 
 	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
-	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
-
-	datadogv2WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-	datadogv2WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v2/series", protocol="datadog"}`)
-
-	datadogsketchesWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
+	datadogWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 
 	datadogValidateRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/check_run", protocol="datadog"}`)
 	datadogIntakeRequests   = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/intake", protocol="datadog"}`)
 	datadogMetadataRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/metadata", protocol="datadog"}`)
 
-	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
-
-	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
-
-	newrelicInventoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/inventory/deltas", protocol="newrelic"}`)
-	newrelicCheckRequest      = metrics.NewCounter(`vm_http_requests_total{path="/newrelic", protocol="newrelic"}`)
-
 	promscrapeTargetsRequests          = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`)
 	promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`)
-
-	promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/metric-relabel-debug"}`)
-	promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target-relabel-debug"}`)
-
-	promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
+	promscrapeAPIV1TargetsRequests     = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`)
 
 	promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`)
 	promscrapeTargetResponseErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`)

app/vmagent/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
+RUN apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmagent/native/request_handler.go

@@ -10,8 +10,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -30,12 +31,14 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
-		return insertRows(at, block, extraLabels)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(req.Body, isGzip, func(block *parser.Block) error {
+			return insertRows(at, block, extraLabels)
+		})
 	})
 }
 
-func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, block *parser.Block, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 
@@ -84,8 +87,6 @@ func insertRows(at *auth.Token, block *stream.Block, extraLabels []prompbmarshal
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(at, &ctx.WriteRequest)
 	return nil
 }

app/vmagent/newrelic/request_handler.go

@@ -1,88 +0,0 @@
-package newrelic
-
-import (
-	"net/http"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="newrelic"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="newrelic"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="newrelic"}`)
-)
-
-// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
-func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	ce := req.Header.Get("Content-Encoding")
-	isGzip := ce == "gzip"
-	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
-		return insertRows(at, rows, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	samplesCount := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range rows {
-		r := &rows[i]
-		tags := r.Tags
-		srcSamples := r.Samples
-		for j := range srcSamples {
-			s := &srcSamples[j]
-			labelsLen := len(labels)
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: bytesutil.ToUnsafeString(s.Name),
-			})
-			for k := range tags {
-				t := &tags[k]
-				labels = append(labels, prompbmarshal.Label{
-					Name:  bytesutil.ToUnsafeString(t.Key),
-					Value: bytesutil.ToUnsafeString(t.Value),
-				})
-			}
-			samples = append(samples, prompbmarshal.Sample{
-				Value:     s.Value,
-				Timestamp: r.Timestamp,
-			})
-			tssDst = append(tssDst, prompbmarshal.TimeSeries{
-				Labels:  labels[labelsLen:],
-				Samples: samples[len(samples)-1:],
-			})
-			labels = append(labels, extraLabels...)
-		}
-		samplesCount += len(srcSamples)
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(len(rows))
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(samplesCount)
-	}
-	rowsPerInsert.Update(float64(samplesCount))
-	return nil
-}

app/vmagent/opentelemetry/request_handler.go

@@ -1,77 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="opentelemetry"}`)
-	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="opentelemetry"}`)
-	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
-)
-
-// InsertHandler processes opentelemetry metrics.
-func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := parserCommon.GetExtraLabels(req)
-	if err != nil {
-		return err
-	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	var processBody func([]byte) ([]byte, error)
-	if req.Header.Get("Content-Type") == "application/json" {
-		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
-			processBody = firehose.ProcessRequestBody
-		} else {
-			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-		}
-	}
-	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
-		return insertRows(at, tss, extraLabels)
-	})
-}
-
-func insertRows(at *auth.Token, tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) error {
-	ctx := common.GetPushCtx()
-	defer common.PutPushCtx(ctx)
-
-	rowsTotal := 0
-	tssDst := ctx.WriteRequest.Timeseries[:0]
-	labels := ctx.Labels[:0]
-	samples := ctx.Samples[:0]
-	for i := range tss {
-		ts := &tss[i]
-		rowsTotal += len(ts.Samples)
-		labelsLen := len(labels)
-		labels = append(labels, ts.Labels...)
-		labels = append(labels, extraLabels...)
-		samplesLen := len(samples)
-		samples = append(samples, ts.Samples...)
-		tssDst = append(tssDst, prompbmarshal.TimeSeries{
-			Labels:  labels[labelsLen:],
-			Samples: samples[samplesLen:],
-		})
-	}
-	ctx.WriteRequest.Timeseries = tssDst
-	ctx.Labels = labels
-	ctx.Samples = samples
-	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
-	rowsInserted.Add(rowsTotal)
-	if at != nil {
-		rowsTenantInserted.Get(at).Add(rowsTotal)
-	}
-	rowsPerInsert.Update(float64(rowsTotal))
-	return nil
-}

app/vmagent/opentsdb/request_handler.go

@@ -7,7 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdb/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -20,7 +20,9 @@ var (
 //
 // See http://opentsdb.net/docs/build/html/api_telnet/put.html
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, insertRows)
+	return writeconcurrencylimiter.Do(func() error {
+		return parser.ParseStream(r, insertRows)
+	})
 }
 
 func insertRows(rows []parser.Row) error {
@@ -56,9 +58,7 @@ func insertRows(rows []parser.Row) error {
 	ctx.WriteRequest.Timeseries = tssDst
 	ctx.Labels = labels
 	ctx.Samples = samples
-	if !remotewrite.TryPush(nil, &ctx.WriteRequest) {
-		return remotewrite.ErrQueueFullHTTPRetry
-	}
+	remotewrite.Push(nil, &ctx.WriteRequest)
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
 	return nil