docs/release-guide: add a note about versioning in helm charts and ansible

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -5,10 +5,10 @@ body:
   - type: markdown
     attributes:
       value: |
-        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade) 
+        Before filling a bug report it would be great to [upgrade](https://docs.victoriametrics.com/#how-to-upgrade) 
         to [the latest available release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
         and verify whether the bug is reproducible there.
-        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) first.
+        It's also recommended to read the [troubleshooting docs](https://docs.victoriametrics.com/troubleshooting/) first.
   - type: textarea
     id: describe-the-bug
     attributes:
@@ -64,8 +64,8 @@ body:
           * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
         
         See how to setup monitoring here:
-          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
-          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
+          * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)
+          * [monitoring for VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/#monitoring)
     validations:
       required: false
   - type: textarea

.github/ISSUE_TEMPLATE/question.yml

@@ -24,9 +24,9 @@ body:
       label: Troubleshooting docs
       description: I am familiar with the following troubleshooting docs
       options:
-        - label: General - https://docs.victoriametrics.com/victoriametrics/troubleshooting/
+        - label: General - https://docs.victoriametrics.com/troubleshooting/
           required: false
-        - label: vmagent - https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting
+        - label: vmagent - https://docs.victoriametrics.com/vmagent/#troubleshooting
           required: false
-        - label: vmalert - https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting
+        - label: vmalert - https://docs.victoriametrics.com/vmalert/#troubleshooting
           required: false

.github/pull_request_template.md

@@ -6,4 +6,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 
 The following checks are **mandatory**:
 
-- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
+- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).

.github/workflows/docs.yaml

@@ -1,57 +0,0 @@
-name: publish-docs
-on:
-  push:
-    branches:
-      - 'master'
-    paths:
-      - 'docs/**'
-      - '.github/workflows/docs.yaml'
-  workflow_dispatch: {}
-permissions:
-  contents: read  # This is required for actions/checkout and to commit back image update
-  deployments: write
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-        with:
-          path: __vm
-
-      - name: Checkout private code
-        uses: actions/checkout@v4
-        with:
-          repository: VictoriaMetrics/vmdocs
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: __vm-docs
-
-      - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v6
-        id: import-gpg
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-          git_config_global: true
-
-      - name: Copy docs
-        id: update
-        run: |
-          find docs -type d -maxdepth 1 -mindepth 1 -exec \
-            sh -c 'rsync -zarvh --delete {}/ ../__vm-docs/content/$(basename {})/' \;
-          echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
-        working-directory: __vm
-
-      - name: Push to vmdocs
-        run: |
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          if [[ -n $(git status --porcelain) ]]; then
-            git add .
-            git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.update.outputs.SHORT_SHA }}"
-            git push
-          fi
-        working-directory: __vm-docs

.github/workflows/main.yml

@@ -85,12 +85,12 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
 
       - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
+        run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
         uses: codecov/codecov-action@v5
         with:
-          files: ./coverage.txt
+          file: ./coverage.txt
 
   integration-test:
     name: integration-test

.github/workflows/sync-docs.yml

@@ -0,0 +1,51 @@
+name: publish-docs
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'docs/**'
+  workflow_dispatch: {}
+permissions:
+  contents: read  # This is required for actions/checkout and to commit back image update
+  deployments: write
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+        with:
+          path: main
+      - name: Checkout private code
+        uses: actions/checkout@v4
+        with:
+          repository: VictoriaMetrics/vmdocs
+          token: ${{ secrets.VM_BOT_GH_TOKEN }}
+          path: docs
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@v6
+        with:
+          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+          workdir: docs
+      - name: Set short git commit SHA
+        id: vars
+        run: |
+          calculatedSha=$(git rev-parse --short ${{ github.sha }})
+          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
+        working-directory: main
+      - name: update code and commit
+        run: |
+          rm -rf content
+          cp -r ../main/docs content
+          make clean-after-copy
+          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
+          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
+          git add .
+          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
+          git push
+        working-directory: docs

.gitignore

@@ -27,4 +27,3 @@ _site
 coverage.txt
 cspell.json
 *~
-deployment/docker/provisioning/plugins/

CONTRIBUTING.md

@@ -1 +1 @@
-The document has been moved [here](https://docs.victoriametrics.com/victoriametrics/contributing/).
+The document has been moved [here](https://docs.victoriametrics.com/contributing/).

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2024 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -5,6 +5,7 @@ MAKE_PARALLEL := $(MAKE) -j $(MAKE_CONCURRENCY)
 DATEINFO_TAG ?= $(shell date -u +'%Y%m%d-%H%M%S')
 BUILDINFO_TAG ?= $(shell echo $$(git describe --long --all | tr '/' '-')$$( \
 	      git diff-index --quiet HEAD -- || echo '-dirty-'$$(git diff-index -u HEAD | openssl sha1 | cut -d' ' -f2 | cut -c 1-8)))
+LATEST_TAG ?= cluster-latest
 
 PKG_TAG ?= $(shell git tag -l --points-at HEAD)
 ifeq ($(PKG_TAG),)
@@ -17,7 +18,7 @@ TAR_OWNERSHIP ?= --owner=1000 --group=1000
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include codespell/Makefile
+include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
@@ -107,31 +108,12 @@ package: \
 	package-vmselect \
 	package-vmstorage
 
-publish-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-metrics $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmagent $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmalert-tool $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmauth $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmbackup $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmrestore $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG) APP_NAME=vmctl $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vminsert $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmselect $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-cluster APP_NAME=vmstorage $(MAKE) publish-via-docker-latest && \
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmgateway $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG)-enterprise APP_NAME=vmbackupmanager $(MAKE) publish-via-docker-latest
-
-publish-victoria-logs-latest:
-	PKG_TAG=$(TAG) APP_NAME=victoria-logs $(MAKE) publish-via-docker-latest
-	PKG_TAG=$(TAG) APP_NAME=vlogscli $(MAKE) publish-via-docker-latest
-
 publish-release:
 	rm -rf bin/*
-	git checkout $(TAG) && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-cluster && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise && $(MAKE) release && $(MAKE) publish && \
-		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && $(MAKE) publish
+	git checkout $(TAG) && $(MAKE) release && LATEST_TAG=stable $(MAKE) publish && \
+		git checkout $(TAG)-cluster && $(MAKE) release && LATEST_TAG=cluster-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise && $(MAKE) release && LATEST_TAG=enterprise-stable $(MAKE) publish && \
+		git checkout $(TAG)-enterprise-cluster && $(MAKE) release && LATEST_TAG=enterprise-cluster-stable $(MAKE) publish
 
 release:
 	$(MAKE_PARALLEL) release-vmcluster
@@ -213,7 +195,7 @@ fmt:
 	gofmt -l -w -s ./apptest
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
+	go vet ./lib/...
 	go vet ./app/...
 	go vet ./apptest/...
 
@@ -222,35 +204,35 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test: all
 	go test ./apptest/... -skip="^TestSingle.*"
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
+	go test -bench=. ./lib/...
 	go test -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
 	go get -u ./lib/...
 	go get -u ./app/...
-	go mod tidy -compat=1.24
+	go mod tidy -compat=1.23
 	go mod vendor
 
 app-local:
@@ -273,10 +255,10 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

README.md

@@ -1,19 +1,17 @@
 # VictoriaMetrics
 
-![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)
-![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)
-![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)
-![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)
-![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)
-![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)
-![Slack](https://img.shields.io/badge/Join-4A154B?logo=slack&link=https%3A%2F%2Fslack.victoriametrics.com)
-![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)
-![Reddit](https://img.shields.io/reddit/subreddit-subscribers/VictoriaMetrics?style=flat&label=Join&labelColor=red&logoColor=white&logo=reddit&link=https%3A%2F%2Fwww.reddit.com%2Fr%2FVictoriaMetrics)
+[![Latest Release](https://img.shields.io/github/release/VictoriaMetrics/VictoriaMetrics.svg?style=flat-square)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+[![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics.svg?maxAge=604800)](https://hub.docker.com/r/victoriametrics/victoria-metrics)
+[![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)
+[![GitHub license](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
+[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
+[![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/workflows/main/badge.svg)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions)
+[![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg)](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 
 <picture>
-  <source srcset="docs/victoriametrics/logo_white.webp" media="(prefers-color-scheme: dark)">
-  <source srcset="docs/victoriametrics/logo.webp" media="(prefers-color-scheme: light)">
-  <img src="docs/victoriametrics/logo.webp" width="300" alt="VictoriaMetrics logo">
+  <source srcset="docs/logo_white.webp" media="(prefers-color-scheme: dark)">
+  <source srcset="docs/logo.webp" media="(prefers-color-scheme: light)">
+  <img src="docs/logo.webp" width="300" alt="VictoriaMetrics logo">
 </picture>
 
 VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
@@ -21,10 +19,10 @@ VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and
 Here are some resources and information about VictoriaMetrics:
 
 - Documentation: [docs.victoriametrics.com](https://docs.victoriametrics.com)
-- Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
-- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), docker images [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
-- Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/)
-- Changelog: [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics)
+- Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/casestudies/).
+- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
+- Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/enterprise/)
+- Changelog: [CHANGELOG](https://docs.victoriametrics.com/changelog/), and [How to upgrade](https://docs.victoriametrics.com/#how-to-upgrade-victoriametrics)
 - Community: [Slack](https://slack.victoriametrics.com/), [X (Twitter)](https://x.com/VictoriaMetrics), [LinkedIn](https://www.linkedin.com/company/victoriametrics/), [YouTube](https://www.youtube.com/@VictoriaMetrics)
 
 Yes, we open-source both the single-node VictoriaMetrics and the cluster version.
@@ -35,22 +33,22 @@ VictoriaMetrics is optimized for timeseries data, even when old time series are
 
 * **Long-term storage for Prometheus** or as a drop-in replacement for Prometheus and Graphite in Grafana.
 * **Powerful stream aggregation**: Can be used as a StatsD alternative.
-* **Ideal for big data**: Works well with large amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+* **Ideal for big data**: Works well with large amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data and various [Enterprise workloads](https://docs.victoriametrics.com/enterprise/).
 * **Query language**: Supports both PromQL and the more performant MetricsQL.
 * **Easy to setup**: No dependencies, single [small binary](https://medium.com/@valyala/stripping-dependency-bloat-in-victoriametrics-docker-image-983fb5912b0d), configuration through command-line flags, but the default is also fine-tuned; backup and restore with [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
 * **Global query view**: Multiple Prometheus instances or any other data sources may ingest data into VictoriaMetrics and queried via a single query.
 * **Various Protocols**: Support metric scraping, ingestion and backfilling in various protocol.
-    * [Prometheus exporters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter), [Prometheus remote write API](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/), [Prometheus exposition format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format).
-    * [InfluxDB line protocol](https://docs.victoriametrics.com/victoriametrics/integrations/influxdb/) over HTTP, TCP and UDP.
-    * [Graphite plaintext protocol](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon).
-    * [OpenTSDB put message](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-telnet).
-    * [HTTP OpenTSDB /api/put requests](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http).
-    * [JSON line format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-json-line-format).
-    * [Arbitrary CSV data](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data).
-    * [Native binary format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-native-format).
-    * [DataDog agent or DogStatsD](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/).
-    * [NewRelic infrastructure agent](https://docs.victoriametrics.com/victoriametrics/integrations/newrelic/#sending-data-from-agent).
-    * [OpenTelemetry metrics format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#sending-data-via-opentelemetry).
+    * [Prometheus exporters](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter), [Prometheus remote write API](https://docs.victoriametrics.com/#prometheus-setup), [Prometheus exposition format](https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format).
+    * [InfluxDB line protocol](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf) over HTTP, TCP and UDP.
+    * [Graphite plaintext protocol](https://docs.victoriametrics.com/#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon).
+    * [OpenTSDB put message](https://docs.victoriametrics.com/#sending-data-via-telnet-put-protocol).
+    * [HTTP OpenTSDB /api/put requests](https://docs.victoriametrics.com/#sending-opentsdb-data-via-http-apiput-requests).
+    * [JSON line format](https://docs.victoriametrics.com/#how-to-import-data-in-json-line-format).
+    * [Arbitrary CSV data](https://docs.victoriametrics.com/#how-to-import-csv-data).
+    * [Native binary format](https://docs.victoriametrics.com/#how-to-import-data-in-native-format).
+    * [DataDog agent or DogStatsD](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent).
+    * [NewRelic infrastructure agent](https://docs.victoriametrics.com/#how-to-send-data-from-newrelic-agent).
+    * [OpenTelemetry metrics format](https://docs.victoriametrics.com/#sending-data-via-opentelemetry).
 * **NFS-based storages**: Supports storing data on NFS-based storages such as Amazon EFS, Google Filestore.
 * And many other features such as metrics relabeling, cardinality limiter, etc.
 
@@ -62,9 +60,9 @@ In addition, the Enterprise version includes extra features:
 - **Backup automation**: Automates regular backup procedures.
 - **Multiple retentions**: Reducing storage costs by specifying different retentions for different datasets.
 - **Downsampling**: Reducing storage costs and increasing performance for queries over historical data.
-- **Stable releases** with long-term support lines ([LTS](https://docs.victoriametrics.com/victoriametrics/lts-releases/)).
+- **Stable releases** with long-term support lines ([LTS](https://docs.victoriametrics.com/lts-releases/)).
 - **Comprehensive support**: First-class consulting, feature requests and technical support provided by the core VictoriaMetrics dev team.
-- Many other features, which you can read about on [the Enterprise page](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+- Many other features, which you can read about on [the Enterprise page](https://docs.victoriametrics.com/enterprise/).
 
 [Contact us](mailto:info@victoriametrics.com) if you need enterprise support for VictoriaMetrics. Or you can request a free trial license [here](https://victoriametrics.com/products/enterprise/trial/), downloaded Enterprise binaries are available at [Github Releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
 
@@ -77,7 +75,7 @@ Some good benchmarks VictoriaMetrics achieved:
 * **Minimal memory footprint**: handling millions of unique timeseries with [10x less RAM](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) than InfluxDB, up to [7x less RAM](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f) than Prometheus, Thanos or Cortex.
 * **Highly scalable and performance** for [data ingestion](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b) and [querying](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4), [20x outperforms](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) InfluxDB and TimescaleDB.
 * **High data compression**: [70x more data points](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) may be stored into limited storage than TimescaleDB, [7x less storage](https://valyala.medium.com/prometheus-vs-victoriametrics-benchmark-on-node-exporter-metrics-4ca29c75590f) space is required than Prometheus, Thanos or Cortex.
-* **Reducing storage costs**: [10x more effective](https://docs.victoriametrics.com/victoriametrics/casestudies/#grammarly) than Graphite according to the Grammarly case study.
+* **Reducing storage costs**: [10x more effective](https://docs.victoriametrics.com/casestudies/#grammarly) than Graphite according to the Grammarly case study.
 * **A single-node VictoriaMetrics** can replace medium-sized clusters built with competing solutions such as Thanos, M3DB, Cortex, InfluxDB or TimescaleDB. See [VictoriaMetrics vs Thanos](https://medium.com/@valyala/comparing-thanos-to-victoriametrics-cluster-b193bea1683), [Measuring vertical scalability](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae), [Remote write storage wars - PromCon 2019](https://promcon.io/2019-munich/talks/remote-write-storage-wars/).
 * **Optimized for storage**: [Works well with high-latency IO](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b) and low IOPS (HDD and network storage in AWS, Google Cloud, Microsoft Azure, etc.).
 
@@ -93,7 +91,7 @@ Feel free asking any questions regarding VictoriaMetrics:
 * [Telegram-ru](https://t.me/VictoriaMetrics_ru1)
 * [Mastodon](https://mastodon.social/@victoriametrics/)
 
-If you like VictoriaMetrics and want to contribute, then please [read these docs](https://docs.victoriametrics.com/victoriametrics/contributing/).
+If you like VictoriaMetrics and want to contribute, then please [read these docs](https://docs.victoriametrics.com/contributing/).
 
 ## VictoriaMetrics Logo
 

SECURITY.md

@@ -6,9 +6,9 @@ The following versions of VictoriaMetrics receive regular security fixes:
 
 | Version | Supported          |
 |---------|--------------------|
-| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
-| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
+| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.

app/victoria-logs/main.go

@@ -46,9 +46,7 @@ func main() {
 	vlselect.Init()
 	vlinsert.Init()
 
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaLogs in %.3f seconds; see https://docs.victoriametrics.com/victorialogs/", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()

app/vlinsert/datadog/datadog.go

@@ -3,6 +3,7 @@ package datadog
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"time"
@@ -10,22 +11,14 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#stream-fields")
-	datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Comma-separated list of fields to ignore for logs ingested via DataDog protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#dropping-fields")
-
-	maxRequestSize = flagutil.NewBytes("datadog.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single DataDog request")
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 )
 
 var parserPool fastjson.ParserPool
@@ -47,6 +40,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 	w.Header().Add("Content-Type", "application/json")
 	startTime := time.Now()
 	v2LogsRequestsTotal.Inc()
+	reader := r.Body
 
 	var ts int64
 	if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
@@ -61,17 +55,28 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 		ts = startTime.UnixNano()
 	}
 
-	cp, err := insertutil.GetCommonParams(r)
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot read gzipped logs request: %s", err)
+			return true
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
 	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
 		return true
 	}
 
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = *datadogStreamFields
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *datadogIgnoreFields
+	cp, err := insertutils.GetCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return true
 	}
 
 	if err := vlstorage.CanWriteData(); err != nil {
@@ -79,15 +84,11 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("datadog", false)
-		err := readLogsRequest(ts, data, lmp)
-		lmp.MustClose()
-		return err
-	})
+	lmp := cp.NewLogMessageProcessor("datadog")
+	err = readLogsRequest(ts, data, lmp)
+	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot read DataDog protocol data: %s", err)
+		logger.Warnf("cannot decode log message in /api/v2/logs request: %s, stream fields: %s", err, cp.StreamFields)
 		return true
 	}
 
@@ -95,7 +96,6 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 	// There is no need in updating v2LogsRequestDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	v2LogsRequestDuration.UpdateDuration(startTime)
-	w.WriteHeader(http.StatusAccepted)
 	fmt.Fprintf(w, `{}`)
 	return true
 }
@@ -105,73 +105,9 @@ var (
 	v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
 )
 
-// datadog message field has two formats:
-//   - regular log message with string text
-//   - nested json format for serverless plugins
-//     which has the following format:
-//     {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
-//
-// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
-func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage.Field, error) {
-	switch v.Type() {
-	case fastjson.TypeString:
-		val := v.GetStringBytes()
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: bytesutil.ToUnsafeString(val),
-		})
-	case fastjson.TypeObject:
-		var firstErr error
-		v.GetObject().Visit(func(k []byte, v *fastjson.Value) {
-			if firstErr != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "_msg",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "status":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "status",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "lamdba":
-				obj, err := v.Object()
-				if err != nil {
-					firstErr = err
-					firstErr = fmt.Errorf("unexpected lambda value type for %q:%q; want object", k, v)
-					return
-				}
-				obj.Visit(func(k []byte, v *fastjson.Value) {
-					if firstErr != nil {
-						return
-					}
-					val, err := v.StringBytes()
-					if err != nil {
-						firstErr = fmt.Errorf("unexpected lambda label value type for %q:%q; want string", k, v)
-						return
-					}
-					fields = append(fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(val),
-					})
-				})
-
-			}
-		})
-	default:
-		return fields, fmt.Errorf("unsupported message type %q", v.Type().String())
-	}
-	return fields, nil
-}
-
 // readLogsRequest parses data according to DataDog logs format
 // https://docs.datadoghq.com/api/latest/logs/#send-logs
-func readLogsRequest(ts int64, data []byte, lmp insertutil.LogMessageProcessor) error {
+func readLogsRequest(ts int64, data []byte, lmp insertutils.LogMessageProcessor) error {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
 	v, err := p.ParseBytes(data)
@@ -193,27 +129,19 @@ func readLogsRequest(ts int64, data []byte, lmp insertutil.LogMessageProcessor)
 			if err != nil {
 				return
 			}
-			switch bytesutil.ToUnsafeString(k) {
+			val, e := v.StringBytes()
+			if e != nil {
+				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
+				return
+			}
+			switch string(k) {
 			case "message":
-				fields, err = appendMsgFields(fields, v)
-				if err != nil {
-					return
-				}
-			case "timestamp":
-				val, e := v.Int64()
-				if e != nil {
-					err = fmt.Errorf("failed to parse timestamp for %q:%q", k, v)
-				}
-				if val > 0 {
-					ts = val * 1e6
-				}
+				fields = append(fields, logstorage.Field{
+					Name:  "_msg",
+					Value: bytesutil.ToUnsafeString(val),
+				})
 			case "ddtags":
 				// https://docs.datadoghq.com/getting_started/tagging/
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
 				var pair []byte
 				idx := 0
 				for idx >= 0 {
@@ -240,20 +168,12 @@ func readLogsRequest(ts int64, data []byte, lmp insertutil.LogMessageProcessor)
 					}
 				}
 			default:
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
 				fields = append(fields, logstorage.Field{
 					Name:  bytesutil.ToUnsafeString(k),
 					Value: bytesutil.ToUnsafeString(val),
 				})
 			}
 		})
-		if err != nil {
-			return err
-		}
 		lmp.AddRow(ts, fields, nil)
 		fields = fields[:0]
 	}

app/vlinsert/datadog/datadog_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestReadLogsRequestFailure(t *testing.T) {
@@ -13,7 +13,7 @@ func TestReadLogsRequestFailure(t *testing.T) {
 
 		ts := time.Now().UnixNano()
 
-		lmp := &insertutil.TestLogMessageProcessor{}
+		lmp := &insertutils.TestLogMessageProcessor{}
 		if err := readLogsRequest(ts, []byte(data), lmp); err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -37,7 +37,7 @@ func TestReadLogsRequestSuccess(t *testing.T) {
 		for i := 0; i < rowsExpected; i++ {
 			timestampsExpected = append(timestampsExpected, ts)
 		}
-		lmp := &insertutil.TestLogMessageProcessor{}
+		lmp := &insertutils.TestLogMessageProcessor{}
 		if err := readLogsRequest(ts, []byte(data), lmp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -54,12 +54,6 @@ func TestReadLogsRequestSuccess(t *testing.T) {
 			"hostname":"127.0.0.1",
 			"message":"bar",
 			"service":"test"
-		}, {
-			"ddsource":"nginx",
-                        "ddtags":"tag1:value1,tag2:value2",
-                        "hostname":"127.0.0.1",
-                        "message":{"message": "nested"},
-                        "service":"test"
 		}, {
 			"ddsource":"nginx",
 			"ddtags":"tag1:value1,tag2:value2",
@@ -92,9 +86,8 @@ func TestReadLogsRequestSuccess(t *testing.T) {
                         "service":"test"
 		}
 	]`
-	rowsExpected := 7
+	rowsExpected := 6
 	resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"nested","service":"test"}
 {"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
 {"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
 {"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -10,14 +10,14 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 )
 
@@ -60,7 +60,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	switch path {
-	case "/", "":
+	case "/":
 		switch r.Method {
 		case http.MethodGet:
 			// Return fake response for Elasticsearch ping request.
@@ -90,7 +90,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		startTime := time.Now()
 		bulkRequestsTotal.Inc()
 
-		cp, err := insertutil.GetCommonParams(r)
+		cp, err := insertutils.GetCommonParams(r)
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -99,10 +99,10 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk", true)
-		encoding := r.Header.Get("Content-Encoding")
+		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk")
+		isGzip := r.Header.Get("Content-Encoding") == "gzip"
 		streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-		n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeFields, cp.MsgFields, lmp)
+		n, err := readBulkRequest(streamName, r.Body, isGzip, cp.TimeField, cp.MsgFields, lmp)
 		lmp.MustClose()
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
@@ -131,23 +131,26 @@ var (
 	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
+func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return 0, fmt.Errorf("cannot decode Elasticsearch protocol data: %w", err)
+	if isGzip {
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
+		}
+		defer common.PutGzipReader(zr)
+		r = zr
 	}
-	defer protoparserutil.PutUncompressedReader(reader)
 
-	wcr := writeconcurrencylimiter.GetReader(reader)
+	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
-	lr := insertutil.NewLineReader(streamName, wcr)
+	lr := insertutils.NewLineReader(streamName, wcr)
 
 	n := 0
 	for {
-		ok, err := readBulkLine(lr, timeFields, msgFields, lmp)
+		ok, err := readBulkLine(lr, timeField, msgFields, lmp)
 		wcr.DecConcurrency()
 		if err != nil || !ok {
 			return n, err
@@ -156,7 +159,7 @@ func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields
 	}
 }
 
-func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
+func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 
 	// Read the command, must be "create" or "index"
@@ -190,7 +193,7 @@ func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
 
-	ts, err := extractTimestampFromFields(timeFields, p.Fields)
+	ts, err := extractTimestampFromFields(timeField, p.Fields)
 	if err != nil {
 		return false, fmt.Errorf("cannot parse timestamp: %w", err)
 	}
@@ -204,20 +207,18 @@ func readBulkLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp
 	return true, nil
 }
 
-func extractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			timestamp, err := parseElasticsearchTimestamp(f.Value)
-			if err != nil {
-				return 0, err
-			}
-			f.Value = ""
-			return timestamp, nil
+func extractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
 		}
+		timestamp, err := parseElasticsearchTimestamp(f.Value)
+		if err != nil {
+			return 0, err
+		}
+		f.Value = ""
+		return timestamp, nil
 	}
 	return 0, nil
 }
@@ -230,7 +231,7 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
 	}
 	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
 		// Try parsing timestamp in seconds or milliseconds
-		return insertutil.ParseUnixTimestamp(s)
+		return insertutils.ParseUnixTimestamp(s)
 	}
 	if len(s) == len("YYYY-MM-DD") {
 		t, err := time.Parse("2006-01-02", s)

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -2,24 +2,20 @@ package elasticsearch
 
 import (
 	"bytes"
+	"compress/gzip"
 	"fmt"
-	"github.com/golang/snappy"
-	"github.com/klauspost/compress/gzip"
-	"github.com/klauspost/compress/zlib"
-	"github.com/klauspost/compress/zstd"
-	"io"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestReadBulkRequest_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", []string{"_time"}, []string{"_msg"}, tlp)
+		rows, err := readBulkRequest("test", r, false, "_time", []string{"_msg"}, tlp)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -37,16 +33,15 @@ foobar`)
 }
 
 func TestReadBulkRequest_Success(t *testing.T) {
-	f := func(data, encoding, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
+	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		timeFields := []string{"non_existing_foo", timeField, "non_existing_bar"}
 		msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, "", timeFields, msgFields, tlp)
+		rows, err := readBulkRequest("test", r, false, timeField, msgFields, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -58,12 +53,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
 		}
 
 		// Read the request with compression
-		tlp = &insertutil.TestLogMessageProcessor{}
-		if encoding != "" {
-			data = compressData(data, encoding)
-		}
-		r = bytes.NewBufferString(data)
-		rows, err = readBulkRequest("test", r, encoding, timeFields, msgFields, tlp)
+		tlp = &insertutils.TestLogMessageProcessor{}
+		compressedData := compressData(data)
+		r = bytes.NewBufferString(compressedData)
+		rows, err = readBulkRequest("test", r, true, timeField, msgFields, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -76,9 +69,9 @@ func TestReadBulkRequest_Success(t *testing.T) {
 	}
 
 	// Verify an empty data
-	f("", "gzip", "_time", "_msg", nil, "")
-	f("\n", "gzip", "_time", "_msg", nil, "")
-	f("\n\n", "gzip", "_time", "_msg", nil, "")
+	f("", "_time", "_msg", nil, "")
+	f("\n", "_time", "_msg", nil, "")
+	f("\n\n", "_time", "_msg", nil, "")
 
 	// Verify non-empty data
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
@@ -89,35 +82,20 @@ func TestReadBulkRequest_Success(t *testing.T) {
 {"message":"xyz","@timestamp":"1686026893735","x":"y"}
 {"create":{"_index":"filebeat-8.8.0"}}
 {"message":"qwe rty","@timestamp":"1686026893"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty float","@timestamp":"1686026123.62"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000, 1686026123620000000}
+	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
 	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"_msg":"baz"}
 {"_msg":"xyz","x":"y"}
-{"_msg":"qwe rty"}
-{"_msg":"qwe rty float"}`
-	f(data, "zstd", timeField, msgField, timestampsExpected, resultExpected)
+{"_msg":"qwe rty"}`
+	f(data, timeField, msgField, timestampsExpected, resultExpected)
 }
 
-func compressData(s string, encoding string) string {
+func compressData(s string) string {
 	var bb bytes.Buffer
-	var zw io.WriteCloser
-	switch encoding {
-	case "gzip":
-		zw = gzip.NewWriter(&bb)
-	case "zstd":
-		zw, _ = zstd.NewWriter(&bb)
-	case "snappy":
-		return string(snappy.Encode(nil, []byte(s)))
-	case "deflate":
-		zw = zlib.NewWriter(&bb)
-	default:
-		panic(fmt.Errorf("%q encoding is not supported", encoding))
-	}
+	zw := gzip.NewWriter(&bb)
 	if _, err := zw.Write([]byte(s)); err != nil {
 		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
 	}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -5,29 +5,20 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("encoding:none", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "")
+	b.Run("gzip:off", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, false)
 	})
-	b.Run("encoding:gzip", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "gzip")
-	})
-	b.Run("encoding:zstd", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "zstd")
-	})
-	b.Run("encoding:deflate", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "deflate")
-	})
-	b.Run("encoding:snappy", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, "snappy")
+	b.Run("gzip:on", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, true)
 	})
 }
 
-func benchmarkReadBulkRequest(b *testing.B, encoding string) {
+func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
 {"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"create":{"_index":"filebeat-8.8.0"}}
@@ -35,14 +26,14 @@ func benchmarkReadBulkRequest(b *testing.B, encoding string) {
 {"create":{"_index":"filebeat-8.8.0"}}
 {"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
-	if encoding != "" {
-		data = compressData(data, encoding)
+	if isGzip {
+		data = compressData(data)
 	}
 	dataBytes := bytesutil.ToUnsafeBytes(data)
 
-	timeFields := []string{"@timestamp"}
+	timeField := "@timestamp"
 	msgFields := []string{"message"}
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 
 	b.ReportAllocs()
 	b.SetBytes(int64(len(data)))
@@ -50,7 +41,7 @@ func benchmarkReadBulkRequest(b *testing.B, encoding string) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest("test", r, encoding, timeFields, msgFields, blp)
+			_, err := readBulkRequest("test", r, isGzip, timeField, msgFields, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutil/common_params.go

@@ -1,308 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-)
-
-var (
-	defaultMsgValue = flag.String("defaultMsgValue", "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field",
-		"Default value for _msg field if the ingested log entry doesn't contain it; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field")
-)
-
-// CommonParams contains common HTTP parameters used by log ingestion APIs.
-//
-// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
-type CommonParams struct {
-	TenantID         logstorage.TenantID
-	TimeFields       []string
-	MsgFields        []string
-	StreamFields     []string
-	IgnoreFields     []string
-	DecolorizeFields []string
-	ExtraFields      []logstorage.Field
-
-	Debug           bool
-	DebugRequestURI string
-	DebugRemoteAddr string
-}
-
-// GetCommonParams returns CommonParams from r.
-func GetCommonParams(r *http.Request) (*CommonParams, error) {
-	// Extract tenantID
-	tenantID, err := logstorage.GetTenantIDFromRequest(r)
-	if err != nil {
-		return nil, err
-	}
-
-	timeFields := []string{"_time"}
-	if tfs := httputil.GetArray(r, "_time_field", "VL-Time-Field"); len(tfs) > 0 {
-		timeFields = tfs
-	}
-
-	msgFields := httputil.GetArray(r, "_msg_field", "VL-Msg-Field")
-	streamFields := httputil.GetArray(r, "_stream_fields", "VL-Stream-Fields")
-	ignoreFields := httputil.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
-	decolorizeFields := httputil.GetArray(r, "decolorize_fields", "VL-Decolorize-Fields")
-
-	extraFields, err := getExtraFields(r)
-	if err != nil {
-		return nil, err
-	}
-
-	debug := false
-	if dv := httputil.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
-		debug, err = strconv.ParseBool(dv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
-		}
-	}
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-	if debug {
-		debugRequestURI = httpserver.GetRequestURI(r)
-		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
-	}
-
-	cp := &CommonParams{
-		TenantID:         tenantID,
-		TimeFields:       timeFields,
-		MsgFields:        msgFields,
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-		Debug:            debug,
-		DebugRequestURI:  debugRequestURI,
-		DebugRemoteAddr:  debugRemoteAddr,
-	}
-
-	return cp, nil
-}
-
-func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
-	efs := httputil.GetArray(r, "extra_fields", "VL-Extra-Fields")
-	if len(efs) == 0 {
-		return nil, nil
-	}
-
-	extraFields := make([]logstorage.Field, len(efs))
-	for i, ef := range efs {
-		n := strings.Index(ef, "=")
-		if n <= 0 || n == len(ef)-1 {
-			return nil, fmt.Errorf(`invalid extra_field format: %q; must be in the form "field=value"`, ef)
-		}
-		extraFields[i] = logstorage.Field{
-			Name:  ef[:n],
-			Value: ef[n+1:],
-		}
-	}
-	return extraFields, nil
-}
-
-// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) *CommonParams {
-	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
-	if streamFields == nil {
-		streamFields = []string{
-			"hostname",
-			"app_name",
-			"proc_id",
-		}
-	}
-	cp := &CommonParams{
-		TenantID: tenantID,
-		TimeFields: []string{
-			"timestamp",
-		},
-		MsgFields: []string{
-			"message",
-		},
-		StreamFields:     streamFields,
-		IgnoreFields:     ignoreFields,
-		DecolorizeFields: decolorizeFields,
-		ExtraFields:      extraFields,
-	}
-
-	return cp
-}
-
-// LogMessageProcessor is an interface for log message processors.
-type LogMessageProcessor interface {
-	// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
-	//
-	// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
-	//
-	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can reuse them.
-	AddRow(timestamp int64, fields, streamFields []logstorage.Field)
-
-	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
-	MustClose()
-}
-
-type logMessageProcessor struct {
-	mu            sync.Mutex
-	wg            sync.WaitGroup
-	stopCh        chan struct{}
-	lastFlushTime time.Time
-
-	cp *CommonParams
-	lr *logstorage.LogRows
-
-	rowsIngestedTotal  *metrics.Counter
-	bytesIngestedTotal *metrics.Counter
-}
-
-func (lmp *logMessageProcessor) initPeriodicFlush() {
-	lmp.lastFlushTime = time.Now()
-
-	lmp.wg.Add(1)
-	go func() {
-		defer lmp.wg.Done()
-
-		d := timeutil.AddJitterToDuration(time.Second)
-		ticker := time.NewTicker(d)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-lmp.stopCh:
-				return
-			case <-ticker.C:
-				lmp.mu.Lock()
-				if time.Since(lmp.lastFlushTime) >= d {
-					lmp.flushLocked()
-				}
-				lmp.mu.Unlock()
-			}
-		}
-	}()
-}
-
-// AddRow adds new log message to lmp with the given timestamp and fields.
-//
-// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
-func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// InsertRowProcessor is used by native data ingestion protocol parser.
-type InsertRowProcessor interface {
-	// AddInsertRow must add r to the underlying storage.
-	AddInsertRow(r *logstorage.InsertRow)
-}
-
-// AddInsertRow adds r to lmp.
-func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(r.Fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(r.Fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAddInsertRow(r)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
-// flushLocked must be called under locked lmp.mu.
-func (lmp *logMessageProcessor) flushLocked() {
-	lmp.lastFlushTime = time.Now()
-	vlstorage.MustAddRows(lmp.lr)
-	lmp.lr.ResetKeepSettings()
-}
-
-// MustClose flushes the remaining data to the underlying storage and closes lmp.
-func (lmp *logMessageProcessor) MustClose() {
-	close(lmp.stopCh)
-	lmp.wg.Wait()
-
-	lmp.flushLocked()
-	logstorage.PutLogRows(lmp.lr)
-	lmp.lr = nil
-}
-
-// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
-//
-// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
-func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode bool) LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.DecolorizeFields, cp.ExtraFields, *defaultMsgValue)
-	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
-	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
-	lmp := &logMessageProcessor{
-		cp: cp,
-		lr: lr,
-
-		rowsIngestedTotal:  rowsIngestedTotal,
-		bytesIngestedTotal: bytesIngestedTotal,
-
-		stopCh: make(chan struct{}),
-	}
-
-	if isStreamMode {
-		lmp.initPeriodicFlush()
-	}
-
-	return lmp
-}
-
-var (
-	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
-)

app/vlinsert/insertutil/flags.go

@@ -1,17 +0,0 @@
-package insertutil
-
-import (
-	"flag"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-)
-
-var (
-	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
-
-	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
-		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
-)

app/vlinsert/insertutil/timestamp.go

@@ -1,103 +0,0 @@
-package insertutil
-
-import (
-	"fmt"
-	"math"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// ExtractTimestampFromFields extracts timestamp in nanoseconds from the first field the name from timeFields at fields.
-//
-// The value for the corresponding timeFields is set to empty string after returning from the function,
-// so it could be ignored during data ingestion.
-//
-// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampFromFields(timeFields []string, fields []logstorage.Field) (int64, error) {
-	for _, timeField := range timeFields {
-		for i := range fields {
-			f := &fields[i]
-			if f.Name != timeField {
-				continue
-			}
-			nsecs, err := parseTimestamp(f.Value)
-			if err != nil {
-				return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", f.Name, err)
-			}
-			f.Value = ""
-			if nsecs == 0 {
-				nsecs = time.Now().UnixNano()
-			}
-			return nsecs, nil
-		}
-	}
-	return time.Now().UnixNano(), nil
-}
-
-func parseTimestamp(s string) (int64, error) {
-	if s == "" || s == "0" {
-		return time.Now().UnixNano(), nil
-	}
-	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
-		return ParseUnixTimestamp(s)
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
-	}
-	return nsecs, nil
-}
-
-// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
-func ParseUnixTimestamp(s string) (int64, error) {
-	if strings.IndexByte(s, '.') >= 0 {
-		// Parse timestamp as floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-		}
-		if f < (1<<31) && f >= (-1<<31) {
-			// The timestamp is in seconds.
-			return int64(f * 1e9), nil
-		}
-		if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
-			// The timestamp is in milliseconds.
-			return int64(f * 1e6), nil
-		}
-		if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
-			// The timestamp is in microseconds.
-			return int64(f * 1e3), nil
-		}
-		// The timestamp is in nanoseconds
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		return int64(f), nil
-	}
-
-	// Parse timestamp as integer
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-	}
-	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds.
-		return n * 1e9, nil
-	}
-	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
-		// The timestamp is in milliseconds.
-		return n * 1e6, nil
-	}
-	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
-		// The timestamp is in microseconds.
-		return n * 1e3, nil
-	}
-	// The timestamp is in nanoseconds
-	return n, nil
-}

app/vlinsert/insertutil/timestamp_test.go

@@ -1,158 +0,0 @@
-package insertutil
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-func TestParseUnixTimestamp_Success(t *testing.T) {
-	f := func(s string, timestampExpected int64) {
-		t.Helper()
-
-		timestamp, err := ParseUnixTimestamp(s)
-		if err != nil {
-			t.Fatalf("unexpected error in ParseUnixTimestamp(%q): %s", s, err)
-		}
-		if timestamp != timestampExpected {
-			t.Fatalf("unexpected timestamp returned from ParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
-		}
-	}
-
-	f("0", 0)
-
-	// nanoseconds
-	f("-1234567890123456789", -1234567890123456789)
-	f("1234567890123456789", 1234567890123456789)
-
-	// microseconds
-	f("-1234567890123456", -1234567890123456000)
-	f("1234567890123456", 1234567890123456000)
-	f("1234567890123456.789", 1234567890123456768)
-
-	// milliseconds
-	f("-1234567890123", -1234567890123000000)
-	f("1234567890123", 1234567890123000000)
-	f("1234567890123.456", 1234567890123456000)
-
-	// seconds
-	f("-1234567890", -1234567890000000000)
-	f("1234567890", 1234567890000000000)
-	f("-1234567890.123456", -1234567890123456000)
-}
-
-func TestParseUnixTimestamp_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := ParseUnixTimestamp(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error in ParseUnixTimestamp(%q)", s)
-		}
-	}
-
-	// non-numeric timestamp
-	f("")
-	f("foobar")
-	f("foo.bar")
-
-	// too big timestamp
-	f("12345678901234567890")
-	f("-12345678901234567890")
-	f("12345678901234567890.235424")
-	f("-12345678901234567890.235424")
-}
-
-func TestExtractTimestampFromFields_Success(t *testing.T) {
-	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
-		t.Helper()
-
-		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if nsecs != nsecsExpected {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
-		}
-
-		for _, f := range fields {
-			if f.Name == timeField {
-				if f.Value != "" {
-					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
-				}
-			}
-		}
-	}
-
-	// UTC time
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20Z"},
-	}, 1718753840000000000)
-
-	// Time with timezone
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
-	}, 1718725040000000000)
-
-	// SQL datetime format
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
-	}, 1718773640123000000)
-
-	// Time with nanosecond precision
-	f("time", []logstorage.Field{
-		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
-		{Name: "foo", Value: "bar"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in nanoseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456789"},
-	}, 1718773640123456789)
-
-	// Unix timestamp in microseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123456"},
-	}, 1718773640123456000)
-
-	// Unix timestamp in milliseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123"},
-	}, 1718773640123000000)
-
-	// Unix timestamp in seconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640"},
-	}, 1718773640000000000)
-}
-
-func TestExtractTimestampFromFields_Error(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		fields := []logstorage.Field{
-			{Name: "time", Value: s},
-		}
-		nsecs, err := ExtractTimestampFromFields([]string{"time"}, fields)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if nsecs != 0 {
-			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
-		}
-	}
-
-	// invalid time
-	f("foobar")
-
-	// incomplete time
-	f("2024-06-18")
-	f("2024-06-18T23:37")
-}

app/vlinsert/insertutils/common_params.go

@@ -0,0 +1,262 @@
+package insertutils
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+)
+
+var (
+	defaultMsgValue = flag.String("defaultMsgValue", "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field",
+		"Default value for _msg field if the ingested log entry doesn't contain it; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field")
+)
+
+// CommonParams contains common HTTP parameters used by log ingestion APIs.
+//
+// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
+type CommonParams struct {
+	TenantID     logstorage.TenantID
+	TimeField    string
+	MsgFields    []string
+	StreamFields []string
+	IgnoreFields []string
+	ExtraFields  []logstorage.Field
+
+	Debug           bool
+	DebugRequestURI string
+	DebugRemoteAddr string
+}
+
+// GetCommonParams returns CommonParams from r.
+func GetCommonParams(r *http.Request) (*CommonParams, error) {
+	// Extract tenantID
+	tenantID, err := logstorage.GetTenantIDFromRequest(r)
+	if err != nil {
+		return nil, err
+	}
+
+	timeField := "_time"
+	if tf := httputils.GetRequestValue(r, "_time_field", "VL-Time-Field"); tf != "" {
+		timeField = tf
+	}
+
+	msgFields := httputils.GetArray(r, "_msg_field", "VL-Msg-Field")
+	streamFields := httputils.GetArray(r, "_stream_fields", "VL-Stream-Fields")
+	ignoreFields := httputils.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
+
+	extraFields, err := getExtraFields(r)
+	if err != nil {
+		return nil, err
+	}
+
+	debug := false
+	if dv := httputils.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
+		debug, err = strconv.ParseBool(dv)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
+		}
+	}
+	debugRequestURI := ""
+	debugRemoteAddr := ""
+	if debug {
+		debugRequestURI = httpserver.GetRequestURI(r)
+		debugRemoteAddr = httpserver.GetQuotedRemoteAddr(r)
+	}
+
+	cp := &CommonParams{
+		TenantID:        tenantID,
+		TimeField:       timeField,
+		MsgFields:       msgFields,
+		StreamFields:    streamFields,
+		IgnoreFields:    ignoreFields,
+		ExtraFields:     extraFields,
+		Debug:           debug,
+		DebugRequestURI: debugRequestURI,
+		DebugRemoteAddr: debugRemoteAddr,
+	}
+
+	return cp, nil
+}
+
+func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
+	efs := httputils.GetArray(r, "extra_fields", "VL-Extra-Fields")
+	if len(efs) == 0 {
+		return nil, nil
+	}
+
+	extraFields := make([]logstorage.Field, len(efs))
+	for i, ef := range efs {
+		n := strings.Index(ef, "=")
+		if n <= 0 || n == len(ef)-1 {
+			return nil, fmt.Errorf(`invalid extra_field format: %q; must be in the form "field=value"`, ef)
+		}
+		extraFields[i] = logstorage.Field{
+			Name:  ef[:n],
+			Value: ef[n+1:],
+		}
+	}
+	return extraFields, nil
+}
+
+// GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields []string, extraFields []logstorage.Field) *CommonParams {
+	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
+	if streamFields == nil {
+		streamFields = []string{
+			"hostname",
+			"app_name",
+			"proc_id",
+		}
+	}
+	cp := &CommonParams{
+		TenantID:  tenantID,
+		TimeField: "timestamp",
+		MsgFields: []string{
+			"message",
+		},
+		StreamFields: streamFields,
+		IgnoreFields: ignoreFields,
+		ExtraFields:  extraFields,
+	}
+
+	return cp
+}
+
+// LogMessageProcessor is an interface for log message processors.
+type LogMessageProcessor interface {
+	// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
+	//
+	// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
+	//
+	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
+	AddRow(timestamp int64, fields, streamFields []logstorage.Field)
+
+	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
+	MustClose()
+}
+
+type logMessageProcessor struct {
+	mu            sync.Mutex
+	wg            sync.WaitGroup
+	stopCh        chan struct{}
+	lastFlushTime time.Time
+
+	cp *CommonParams
+	lr *logstorage.LogRows
+
+	rowsIngestedTotal  *metrics.Counter
+	bytesIngestedTotal *metrics.Counter
+}
+
+func (lmp *logMessageProcessor) initPeriodicFlush() {
+	lmp.lastFlushTime = time.Now()
+
+	lmp.wg.Add(1)
+	go func() {
+		defer lmp.wg.Done()
+
+		d := timeutil.AddJitterToDuration(time.Second)
+		ticker := time.NewTicker(d)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-lmp.stopCh:
+				return
+			case <-ticker.C:
+				lmp.mu.Lock()
+				if time.Since(lmp.lastFlushTime) >= d {
+					lmp.flushLocked()
+				}
+				lmp.mu.Unlock()
+			}
+		}
+	}()
+}
+
+// AddRow adds new log message to lmp with the given timestamp and fields.
+//
+// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
+func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
+	lmp.mu.Lock()
+	defer lmp.mu.Unlock()
+
+	lmp.rowsIngestedTotal.Inc()
+	n := logstorage.EstimatedJSONRowLen(fields)
+	lmp.bytesIngestedTotal.Add(n)
+
+	if len(fields) > *MaxFieldsPerLine {
+		rf := logstorage.RowFormatter(fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		rowsDroppedTotalTooManyFields.Inc()
+		return
+	}
+
+	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
+	if lmp.cp.Debug {
+		s := lmp.lr.GetRowString(0)
+		lmp.lr.ResetKeepSettings()
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		rowsDroppedTotalDebug.Inc()
+		return
+	}
+	if lmp.lr.NeedFlush() {
+		lmp.flushLocked()
+	}
+}
+
+// flushLocked must be called under locked lmp.mu.
+func (lmp *logMessageProcessor) flushLocked() {
+	lmp.lastFlushTime = time.Now()
+	vlstorage.MustAddRows(lmp.lr)
+	lmp.lr.ResetKeepSettings()
+}
+
+// MustClose flushes the remaining data to the underlying storage and closes lmp.
+func (lmp *logMessageProcessor) MustClose() {
+	close(lmp.stopCh)
+	lmp.wg.Wait()
+
+	lmp.flushLocked()
+	logstorage.PutLogRows(lmp.lr)
+	lmp.lr = nil
+}
+
+// NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
+//
+// MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
+func (cp *CommonParams) NewLogMessageProcessor(protocolName string) LogMessageProcessor {
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.ExtraFields, *defaultMsgValue)
+	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
+	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
+	lmp := &logMessageProcessor{
+		cp: cp,
+		lr: lr,
+
+		rowsIngestedTotal:  rowsIngestedTotal,
+		bytesIngestedTotal: bytesIngestedTotal,
+
+		stopCh: make(chan struct{}),
+	}
+	lmp.initPeriodicFlush()
+
+	return lmp
+}
+
+var (
+	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
+)

app/vlinsert/insertutils/flags.go

@@ -0,0 +1,15 @@
+package insertutils
+
+import (
+	"flag"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+)
+
+var (
+	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+
+	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
+)

app/vlinsert/insertutils/line_reader.go

@@ -1,4 +1,4 @@
-package insertutil
+package insertutils
 
 import (
 	"bytes"
@@ -15,8 +15,6 @@ import (
 // LineReader reads newline-delimited lines from the underlying reader
 type LineReader struct {
 	// Line contains the next line read after the call to NextLine
-	//
-	// The Line contents is valid until the next call to NextLine.
 	Line []byte
 
 	// name is the LineReader name
@@ -28,9 +26,6 @@ type LineReader struct {
 	// buf is a buffer for reading the next line
 	buf []byte
 
-	// bufOffset is the offset at buf to read the next line from
-	bufOffset int
-
 	// err is the last error when reading data from r
 	err error
 
@@ -56,27 +51,26 @@ func NewLineReader(name string, r io.Reader) *LineReader {
 // Check for Err in this case.
 func (lr *LineReader) NextLine() bool {
 	for {
-		if lr.bufOffset >= len(lr.buf) {
+		if len(lr.buf) == 0 {
 			if lr.err != nil || lr.eofReached {
 				return false
 			}
 			if !lr.readMoreData() {
 				return false
 			}
-			if lr.bufOffset >= len(lr.buf) && lr.eofReached {
+			if len(lr.buf) == 0 && lr.eofReached {
 				return false
 			}
 		}
 
-		buf := lr.buf[lr.bufOffset:]
-		if n := bytes.IndexByte(buf, '\n'); n >= 0 {
-			lr.Line = buf[:n]
-			lr.bufOffset += n + 1
+		if n := bytes.IndexByte(lr.buf, '\n'); n >= 0 {
+			lr.Line = append(lr.Line[:0], lr.buf[:n]...)
+			lr.buf = append(lr.buf[:0], lr.buf[n+1:]...)
 			return true
 		}
 		if lr.eofReached {
-			lr.Line = buf
-			lr.bufOffset += len(buf)
+			lr.Line = append(lr.Line[:0], lr.buf...)
+			lr.buf = lr.buf[:0]
 			return true
 		}
 		if !lr.readMoreData() {
@@ -94,11 +88,6 @@ func (lr *LineReader) Err() error {
 }
 
 func (lr *LineReader) readMoreData() bool {
-	if lr.bufOffset > 0 {
-		lr.buf = append(lr.buf[:0], lr.buf[lr.bufOffset:]...)
-		lr.bufOffset = 0
-	}
-
 	bufLen := len(lr.buf)
 	if bufLen >= MaxLineSizeBytes.IntN() {
 		logger.Warnf("%s: the line length exceeds -insert.maxLineSizeBytes=%d; skipping it; line contents=%q", lr.name, MaxLineSizeBytes.IntN(), lr.buf)

app/vlinsert/insertutils/line_reader_test.go

@@ -1,4 +1,4 @@
-package insertutil
+package insertutils
 
 import (
 	"bytes"

app/vlinsert/insertutils/testutils.go

@@ -1,4 +1,4 @@
-package insertutil
+package insertutils
 
 import (
 	"fmt"

app/vlinsert/insertutils/timestamp.go

@@ -0,0 +1,69 @@
+package insertutils
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+//
+// The value for the timeField is set to empty string after returning from the function,
+// so it could be ignored during data ingestion.
+//
+// The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
+func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+	for i := range fields {
+		f := &fields[i]
+		if f.Name != timeField {
+			continue
+		}
+		nsecs, err := parseTimestamp(f.Value)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", timeField, err)
+		}
+		f.Value = ""
+		if nsecs == 0 {
+			nsecs = time.Now().UnixNano()
+		}
+		return nsecs, nil
+	}
+	return time.Now().UnixNano(), nil
+}
+
+func parseTimestamp(s string) (int64, error) {
+	if s == "" || s == "0" {
+		return time.Now().UnixNano(), nil
+	}
+	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
+		return ParseUnixTimestamp(s)
+	}
+	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
+	if !ok {
+		return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
+	}
+	return nsecs, nil
+}
+
+// ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.
+func ParseUnixTimestamp(s string) (int64, error) {
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
+	}
+	if n < (1<<31) && n >= (-1<<31) {
+		// The timestamp is in seconds. Convert it to milliseconds
+		n *= 1e3
+	}
+	if n > int64(math.MaxInt64)/1e6 {
+		return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+	}
+	if n < int64(math.MinInt64)/1e6 {
+		return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+	}
+	n *= 1e6
+	return n, nil
+}

app/vlinsert/insertutils/timestamp_test.go

@@ -0,0 +1,88 @@
+package insertutils
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
+	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
+		t.Helper()
+
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if nsecs != nsecsExpected {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, nsecsExpected)
+		}
+
+		for _, f := range fields {
+			if f.Name == timeField {
+				if f.Value != "" {
+					t.Fatalf("unexpected value for field %s; got %q; want %q", timeField, f.Value, "")
+				}
+			}
+		}
+	}
+
+	// UTC time
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20Z"},
+	}, 1718753840000000000)
+
+	// Time with timezone
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
+	}, 1718725040000000000)
+
+	// SQL datetime format
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
+	}, 1718773640123000000)
+
+	// Time with nanosecond precision
+	f("time", []logstorage.Field{
+		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
+		{Name: "foo", Value: "bar"},
+	}, 1718773640123456789)
+
+	// Unix timestamp in milliseconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640123"},
+	}, 1718773640123000000)
+
+	// Unix timestamp in seconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640"},
+	}, 1718773640000000000)
+}
+
+func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+
+		fields := []logstorage.Field{
+			{Name: "time", Value: s},
+		}
+		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if nsecs != 0 {
+			t.Fatalf("unexpected nsecs; got %d; want %d", nsecs, 0)
+		}
+	}
+
+	f("foobar")
+
+	// incomplete time
+	f("2024-06-18")
+	f("2024-06-18T23:37")
+}

app/vlinsert/internalinsert/internalinsert.go

@@ -1,96 +0,0 @@
-package internalinsert
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	disableInsert  = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
-	maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
-)
-
-// RequestHandler processes /internal/insert requests.
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	if *disableInsert {
-		httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable command-line flag")
-		return
-	}
-
-	startTime := time.Now()
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-	version := r.FormValue("version")
-	if version != netinsert.ProtocolVersion {
-		httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("internalinsert", false)
-		irp := lmp.(insertutil.InsertRowProcessor)
-		err := parseData(irp, data)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
-		return
-	}
-
-	requestDuration.UpdateDuration(startTime)
-}
-
-func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
-	r := logstorage.GetInsertRow()
-	src := data
-	i := 0
-	for len(src) > 0 {
-		tail, err := r.UnmarshalInplace(src)
-		if err != nil {
-			return fmt.Errorf("cannot parse row #%d: %s", i, err)
-		}
-		src = tail
-		i++
-
-		irp.AddInsertRow(r)
-	}
-	logstorage.PutInsertRow(r)
-
-	return nil
-}
-
-var (
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
-
-	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
-)

app/vlinsert/journald/journald.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"slices"
@@ -12,37 +13,39 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
-// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
-const journaldEntryMaxNameLen = 64
-
-var allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
-
-var (
-	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
-	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Comma-separated list of fields to ignore for logs ingested over journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#dropping-fields")
-	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Field to use as a log timestamp for logs ingested via journald protocol. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
-	journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
-	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
-
-	maxRequestSize = flagutil.NewBytes("journald.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single journald request")
+const (
+	journaldEntryMaxNameLen = 64
 )
 
-func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
+var (
+	bodyBufferPool                bytesutil.ByteBufferPool
+	allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]+`)
+)
+
+var (
+	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Journal fields to be used as stream fields. "+
+		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
+	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Journal fields to ignore. "+
+		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
+	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Journal field to be used as time field. "+
+		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
+	journaldTenantID             = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint.")
+	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
+)
+
+func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
+	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		return nil, err
 	}
@@ -53,8 +56,8 @@ func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
 		}
 		cp.TenantID = tenantID
 	}
-	if len(cp.TimeFields) == 0 {
-		cp.TimeFields = []string{*journaldTimeField}
+	if cp.TimeField != "" {
+		cp.TimeField = *journaldTimeField
 	}
 	if len(cp.StreamFields) == 0 {
 		cp.StreamFields = *journaldStreamFields
@@ -86,36 +89,45 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsJournaldTotal.Inc()
 
-	cp, err := getCommonParams(r)
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-
 	if err := vlstorage.CanWriteData(); err != nil {
-		errorsTotal.Inc()
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
 
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("journald", false)
-		err := parseJournaldRequest(data, lmp, cp)
-		lmp.MustClose()
-		return err
-	})
+	reader := r.Body
+	var err error
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
 	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
+	writeconcurrencylimiter.PutReader(wcr)
+	bb := bodyBufferPool.Get()
+	defer bodyBufferPool.Put(bb)
+	if r.Header.Get("Content-Encoding") == "zstd" {
+		bb.B, err = zstd.Decompress(bb.B[:0], data)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot decompress zstd-encoded request with length %d: %s", len(data), err)
+			return
+		}
+		data = bb.B
+	}
+	cp, err := getCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
 	}
 
-	// systemd starting release v258 will support compression, which starts working after negotiation: it expects supported compression
-	// algorithms list in Accept-Encoding response header in a format "<algorithm_1>[:<priority_1>][;<algorithm_2>:<priority_2>]"
-	// See https://github.com/systemd/systemd/pull/34822
-	w.Header().Set("Accept-Encoding", "zstd")
+	lmp := cp.NewLogMessageProcessor("journald")
+	err = parseJournaldRequest(data, lmp, cp)
+	lmp.MustClose()
+	if err != nil {
+		errorsTotal.Inc()
+		httpserver.Errorf(w, r, "cannot parse Journald protobuf request: %s", err)
+		return
+	}
 
 	// update requestJournaldDuration only for successfully parsed requests
 	// There is no need in updating requestJournaldDuration for request errors,
@@ -131,7 +143,7 @@ var (
 )
 
 // See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
+func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *insertutils.CommonParams) error {
 	var fields []logstorage.Field
 	var ts int64
 	var size uint64
@@ -181,7 +193,7 @@ func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *i
 			if err != nil {
 				return fmt.Errorf("failed to extract binary field %q value size: %w", name, err)
 			}
-			// skip binary data size
+			// skip binary data sise
 			data = data[idx:]
 			if size == 0 {
 				return fmt.Errorf("unexpected zero binary data size decoded %d", size)
@@ -201,13 +213,14 @@ func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *i
 			}
 			data = data[1:]
 		}
+		// https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
 		if len(name) > journaldEntryMaxNameLen {
 			return fmt.Errorf("journald entry name should not exceed %d symbols, got: %q", journaldEntryMaxNameLen, name)
 		}
 		if !allowedJournaldEntryNameChars.MatchString(name) {
 			return fmt.Errorf("journald entry name should consist of `A-Z0-9_` characters and must start from non-digit symbol")
 		}
-		if slices.Contains(cp.TimeFields, name) {
+		if name == cp.TimeField {
 			n, err := strconv.ParseInt(value, 10, 64)
 			if err != nil {
 				return fmt.Errorf("failed to parse Journald timestamp, %w", err)

app/vlinsert/journald/journald_test.go

@@ -3,16 +3,16 @@ package journald
 import (
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestPushJournaldOk(t *testing.T) {
 	f := func(src string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
-		tlp := &insertutil.TestLogMessageProcessor{}
-		cp := &insertutil.CommonParams{
-			TimeFields: []string{"__REALTIME_TIMESTAMP"},
-			MsgFields:  []string{"MESSAGE"},
+		tlp := &insertutils.TestLogMessageProcessor{}
+		cp := &insertutils.CommonParams{
+			TimeField: "__REALTIME_TIMESTAMP",
+			MsgFields: []string{"MESSAGE"},
 		}
 		if err := parseJournaldRequest([]byte(src), tlp, cp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -35,19 +35,19 @@ func TestPushJournaldOk(t *testing.T) {
 	)
 
 	// Parse binary data
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\nE=JobStateChanged\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
+	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
 		[]int64{1729698775704404000},
-		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
+		"{\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
 	)
 }
 
 func TestPushJournald_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
-		tlp := &insertutil.TestLogMessageProcessor{}
-		cp := &insertutil.CommonParams{
-			TimeFields: []string{"__REALTIME_TIMESTAMP"},
-			MsgFields:  []string{"MESSAGE"},
+		tlp := &insertutils.TestLogMessageProcessor{}
+		cp := &insertutils.CommonParams{
+			TimeField: "__REALTIME_TIMESTAMP",
+			MsgFields: []string{"MESSAGE"},
 		}
 		if err := parseJournaldRequest([]byte(data), tlp, cp); err == nil {
 			t.Fatalf("expected non nil error")

app/vlinsert/jsonline/jsonline.go

@@ -6,12 +6,12 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -28,7 +28,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 
 	requestsTotal.Inc()
 
-	cp, err := insertutil.GetCommonParams(r)
+	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
@@ -38,59 +38,54 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	encoding := r.Header.Get("Content-Encoding")
-	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
-	if err != nil {
-		logger.Errorf("cannot decode jsonline request: %s", err)
-		return
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			logger.Errorf("cannot read gzipped jsonline request: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
 	}
-	defer protoparserutil.PutUncompressedReader(reader)
 
-	lmp := cp.NewLogMessageProcessor("jsonline", true)
+	lmp := cp.NewLogMessageProcessor("jsonline")
 	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, cp.TimeFields, cp.MsgFields, lmp)
+	err = processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
 	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot process jsonline request; error: %s", err)
-		return
-	}
 
-	requestDuration.UpdateDuration(startTime)
+	if err != nil {
+		logger.Errorf("jsonline: %s", err)
+	} else {
+		// update requestDuration only for successfully parsed requests.
+		// There is no need in updating requestDuration for request errors,
+		// since their timings are usually much smaller than the timing for successful request parsing.
+		requestDuration.UpdateDuration(startTime)
+	}
 }
 
-func processStreamInternal(streamName string, r io.Reader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) error {
+func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) error {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
-	lr := insertutil.NewLineReader(streamName, wcr)
+	lr := insertutils.NewLineReader(streamName, wcr)
 
 	n := 0
-	errors := 0
-	var lastError error
 	for {
-		ok, err := readLine(lr, timeFields, msgFields, lmp)
+		ok, err := readLine(lr, timeField, msgFields, lmp)
 		wcr.DecConcurrency()
 		if err != nil {
-			lastError = err
-			errors++
-			logger.Warnf("jsonline: cannot read line #%d in /jsonline request: %s", n, err)
+			errorsTotal.Inc()
+			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
 		}
 		if !ok {
-			break
+			return nil
 		}
 		n++
 	}
-	errorsTotal.Add(errors)
-
-	if errors > 0 && n == errors {
-		// Return an error if no logs were processed and there were errors
-		return lastError
-	}
-
-	return nil
 }
 
-func readLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (bool, error) {
+func readLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 	for len(line) == 0 {
 		if !lr.NextLine() {
@@ -101,17 +96,16 @@ func readLine(lr *insertutil.LineReader, timeFields, msgFields []string, lmp ins
 	}
 
 	p := logstorage.GetJSONParser()
-	defer logstorage.PutJSONParser(p)
-
 	if err := p.ParseLogMessage(line); err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
+		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
-	ts, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
+	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
 	if err != nil {
-		return true, fmt.Errorf("%s; line contents: %q", err, line)
+		return false, fmt.Errorf("cannot get timestamp: %w", err)
 	}
 	logstorage.RenameField(p.Fields, msgFields, "_msg")
 	lmp.AddRow(ts, p.Fields, nil)
+	logstorage.PutJSONParser(p)
 
 	return true, nil
 }

app/vlinsert/jsonline/jsonline_test.go

@@ -2,21 +2,19 @@ package jsonline
 
 import (
 	"bytes"
-	"strings"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestProcessStreamInternalSuccess(t *testing.T) {
+func TestProcessStreamInternal_Success(t *testing.T) {
 	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		timeFields := []string{timeField}
 		msgFields := []string{msgField}
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, timeFields, msgFields, tlp); err != nil {
+		if err := processStreamInternal("test", r, timeField, msgFields, tlp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 
@@ -47,51 +45,22 @@ func TestProcessStreamInternalSuccess(t *testing.T) {
 	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
 {"message":"baz"}`
 	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// invalid lines among valid lines
-	data = `
-dsfodmasd
-
-{"time":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-invalid line
-{"time":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-asbsdf
-
-`
-	timeField = "time"
-	msgField = "message"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
 }
 
-func TestProcessStreamInternalFailure(t *testing.T) {
+func TestProcessStreamInternal_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
-		r := strings.NewReader(data)
-		if err := processStreamInternal("test", r, []string{"time"}, nil, tlp); err == nil {
-			t.Fatalf("expected error, got nil")
-		}
-
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
+		tlp := &insertutils.TestLogMessageProcessor{}
+		r := bytes.NewBufferString(data)
+		if err := processStreamInternal("test", r, "time", nil, tlp); err == nil {
+			t.Fatalf("expecting non-nil error")
 		}
 	}
 
 	// invalid json
 	f("foobar")
 
-	f(`foo
-bar`)
-
-	f(`
-foo
-
-`)
-
 	// invalid timestamp field
 	f(`{"time":"foobar"}`)
 }

app/vlinsert/loki/loki.go

@@ -1,18 +1,12 @@
 package loki
 
 import (
-	"flag"
-	"fmt"
 	"net/http"
-	"strconv"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whether to disable automatic parsing of JSON-encoded log fields inside Loki log message into distinct log fields")
-
 // RequestHandler processes Loki insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
@@ -41,17 +35,8 @@ func handleInsert(r *http.Request, w http.ResponseWriter) {
 	}
 }
 
-type commonParams struct {
-	cp *insertutil.CommonParams
-
-	// Whether to parse JSON inside plaintext log message.
-	//
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8486
-	parseMessage bool
-}
-
-func getCommonParams(r *http.Request) (*commonParams, error) {
-	cp, err := insertutil.GetCommonParams(r)
+func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
+	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		return nil, err
 	}
@@ -70,17 +55,5 @@ func getCommonParams(r *http.Request) (*commonParams, error) {
 
 	}
 
-	parseMessage := !*disableMessageParsing
-	if rv := httputil.GetRequestValue(r, "disable_message_parsing", "VL-Loki-Disable-Message-Parsing"); rv != "" {
-		bv, err := strconv.ParseBool(rv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse dusable_message_parsing=%q: %s", rv, err)
-		}
-		parseMessage = !bv
-	}
-
-	return &commonParams{
-		cp:           cp,
-		parseMessage: parseMessage,
-	}, nil
+	return cp, nil
 }

app/vlinsert/loki/loki_json.go

@@ -2,28 +2,47 @@ package loki
 
 import (
 	"fmt"
+	"io"
+	"math"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 )
 
-var maxRequestSize = flagutil.NewBytes("loki.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single Loki request")
-
 var parserPool fastjson.ParserPool
 
 func handleJSON(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsJSONTotal.Inc()
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
@@ -34,17 +53,12 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_json", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseJSONRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
+	lmp := cp.NewLogMessageProcessor("loki_json")
+	useDefaultStreamFields := len(cp.StreamFields) == 0
+	err = parseJSONRequest(data, lmp, useDefaultStreamFields)
+	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki json data: %s", err)
+		httpserver.Errorf(w, r, "cannot parse Loki json request: %s; data=%s", err, data)
 		return
 	}
 
@@ -52,9 +66,6 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 	// There is no need in updating requestJSONDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	requestJSONDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
 }
 
 var (
@@ -62,10 +73,9 @@ var (
 	requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
-func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
+func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
-
 	v, err := p.ParseBytes(data)
 	if err != nil {
 		return fmt.Errorf("cannot parse JSON request body: %w", err)
@@ -80,20 +90,11 @@ func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields
 		return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
 	}
 
-	fields := getFields()
-	defer putFields(fields)
-
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
-	}
-
 	currentTimestamp := time.Now().UnixNano()
-
+	var commonFields []logstorage.Field
 	for _, stream := range streams {
 		// populate common labels from `stream` dict
-		fields.fields = fields.fields[:0]
+		commonFields = commonFields[:0]
 		labelsV := stream.Get("stream")
 		var labels *fastjson.Object
 		if labelsV != nil {
@@ -109,7 +110,7 @@ func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields
 				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
 				return
 			}
-			fields.fields = append(fields.fields, logstorage.Field{
+			commonFields = append(commonFields, logstorage.Field{
 				Name:  bytesutil.ToUnsafeString(k),
 				Value: bytesutil.ToUnsafeString(vStr),
 			})
@@ -128,10 +129,8 @@ func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields
 			return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
 		}
 
-		commonFieldsLen := len(fields.fields)
+		fields := commonFields
 		for _, line := range lines {
-			fields.fields = fields.fields[:commonFieldsLen]
-
 			lineA, err := line.Array()
 			if err != nil {
 				return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
@@ -153,6 +152,17 @@ func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields
 				ts = currentTimestamp
 			}
 
+			// parse log message
+			msg, err := lineA[1].StringBytes()
+			if err != nil {
+				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
+			}
+
+			fields = append(fields[:len(commonFields)], logstorage.Field{
+				Name:  "_msg",
+				Value: bytesutil.ToUnsafeString(msg),
+			})
+
 			// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
 			if len(lineA) > 2 {
 				structuredMetadata, err := lineA[2].Object()
@@ -167,7 +177,7 @@ func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields
 						return
 					}
 
-					fields.fields = append(fields.fields, logstorage.Field{
+					fields = append(fields, logstorage.Field{
 						Name:  bytesutil.ToUnsafeString(k),
 						Value: bytesutil.ToUnsafeString(vStr),
 					})
@@ -176,51 +186,39 @@ func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields
 					return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
 				}
 			}
-
-			// parse log message
-			msg, err := lineA[1].StringBytes()
-			if err != nil {
-				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
-			}
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, bytesutil.ToUnsafeString(msg))
-
 			var streamFields []logstorage.Field
 			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
+				streamFields = commonFields
 			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
+			lmp.AddRow(ts, fields, streamFields)
 		}
 	}
 
 	return nil
 }
 
-func addMsgField(dst []logstorage.Field, msgParser *logstorage.JSONParser, msg string) ([]logstorage.Field, bool) {
-	if msgParser == nil || len(msg) < 2 || msg[0] != '{' || msg[len(msg)-1] != '}' {
-		return append(dst, logstorage.Field{
-			Name:  "_msg",
-			Value: msg,
-		}), false
-	}
-	if msgParser != nil && len(msg) >= 2 && msg[0] == '{' && msg[len(msg)-1] == '}' {
-		if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msg)); err == nil {
-			return append(dst, msgParser.Fields...), true
-		}
-	}
-	return append(dst, logstorage.Field{
-		Name:  "_msg",
-		Value: msg,
-	}), false
-}
-
 func parseLokiTimestamp(s string) (int64, error) {
 	if s == "" {
 		// Special case - an empty timestamp must be substituted with the current time by the caller.
 		return 0, nil
 	}
-	return insertutil.ParseUnixTimestamp(s)
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		// Fall back to parsing floating-point value
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return 0, err
+		}
+		if f > math.MaxInt64 {
+			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
+		}
+		if f < math.MinInt64 {
+			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
+		}
+		n = int64(f)
+	}
+	if n < 0 {
+		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
+	}
+	return n, nil
 }

app/vlinsert/loki/loki_json_test.go

@@ -3,15 +3,15 @@ package loki
 import (
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestParseJSONRequest_Failure(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err == nil {
+		tlp := &insertutils.TestLogMessageProcessor{}
+		if err := parseJSONRequest([]byte(s), tlp, false); err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
 		if err := tlp.Verify(nil, ""); err != nil {
@@ -63,9 +63,9 @@ func TestParseJSONRequest_Success(t *testing.T) {
 	f := func(s string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
+		if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
@@ -89,9 +89,9 @@ func TestParseJSONRequest_Success(t *testing.T) {
 	"label2": "value2"
 },"values":[
 	["1577836800000000001", "foo bar"],
-	["1686026123.62", "abc"],
+	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1686026123620000000, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
 {"label1":"value1","label2":"value2","_msg":"abc"}
 {"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
@@ -122,48 +122,6 @@ func TestParseJSONRequest_Success(t *testing.T) {
 {"x":"y","_msg":"yx"}`)
 
 	// values with metadata
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"metadata_1":"md_value","_msg":"foo bar"}`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar","metadata_1":"md_value"}`)
 	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 }
-
-func TestParseJSONRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &insertutil.TestLogMessageProcessor{}
-
-		if err := parseJSONRequest([]byte(s), tlp, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"111\",\"parent_id\":\"abc\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"111","parent_id":"abc"}`)
-}

app/vlinsert/loki/loki_json_timing_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func BenchmarkParseJSONRequest(b *testing.B) {
@@ -22,13 +22,13 @@ func BenchmarkParseJSONRequest(b *testing.B) {
 }
 
 func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseJSONRequest(data, blp, nil, false, true); err != nil {
+			if err := parseJSONRequest(data, blp, false); err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}

app/vlinsert/loki/loki_protobuf.go

@@ -2,27 +2,38 @@ package loki
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
 )
 
 var (
+	bytesBufPool bytesutil.ByteBufferPool
 	pushReqsPool sync.Pool
 )
 
 func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsProtobufTotal.Inc()
+	wcr := writeconcurrencylimiter.GetReader(r.Body)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
@@ -33,22 +44,12 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	if encoding == "" {
-		// Loki protocol uses snappy compression by default.
-		// See https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-		encoding = "snappy"
-	}
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.cp.NewLogMessageProcessor("loki_protobuf", false)
-		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
-		err := parseProtobufRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
-		lmp.MustClose()
-		return err
-	})
+	lmp := cp.NewLogMessageProcessor("loki_protobuf")
+	useDefaultStreamFields := len(cp.StreamFields) == 0
+	err = parseProtobufRequest(data, lmp, useDefaultStreamFields)
+	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot read Loki protobuf data: %s", err)
+		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
 		return
 	}
 
@@ -56,9 +57,6 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	// There is no need in updating requestProtobufDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	requestProtobufDuration.UpdateDuration(startTime)
-
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
-	w.WriteHeader(http.StatusNoContent)
 }
 
 var (
@@ -66,11 +64,20 @@ var (
 	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
-func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
+func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
+	bb := bytesBufPool.Get()
+	defer bytesBufPool.Put(bb)
+
+	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
+	if err != nil {
+		return fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
+	}
+	bb.B = buf
+
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err := req.UnmarshalProtobuf(data)
+	err = req.UnmarshalProtobuf(bb.B)
 	if err != nil {
 		return fmt.Errorf("cannot parse request body: %w", err)
 	}
@@ -78,15 +85,8 @@ func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFi
 	fields := getFields()
 	defer putFields(fields)
 
-	var msgParser *logstorage.JSONParser
-	if parseMessage {
-		msgParser = logstorage.GetJSONParser()
-		defer logstorage.PutJSONParser(msgParser)
-	}
-
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
-
 	for i := range streams {
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
@@ -109,8 +109,10 @@ func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFi
 				})
 			}
 
-			allowMsgRenaming := false
-			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, e.Line)
+			fields.fields = append(fields.fields, logstorage.Field{
+				Name:  "_msg",
+				Value: e.Line,
+			})
 
 			ts := e.Timestamp.UnixNano()
 			if ts == 0 {
@@ -121,9 +123,6 @@ func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFi
 			if useDefaultStreamFields {
 				streamFields = fields.fields[:commonFieldsLen]
 			}
-			if allowMsgRenaming {
-				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
-			}
 			lmp.AddRow(ts, fields.fields, streamFields)
 		}
 	}

app/vlinsert/loki/loki_protobuf_test.go

@@ -6,8 +6,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/golang/snappy"
 )
 
 type testLogMessageProcessor struct {
@@ -52,7 +53,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 		t.Helper()
 
 		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
+		if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if len(tlp.pr.Streams) != len(timestampsExpected) {
@@ -60,9 +61,10 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 		}
 
 		data := tlp.pr.MarshalProtobuf(nil)
+		encodedData := snappy.Encode(nil, data)
 
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, nil, false, false); err != nil {
+		tlp2 := &insertutils.TestLogMessageProcessor{}
+		if err := parseProtobufRequest(encodedData, tlp2, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
@@ -88,7 +90,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
 {"label1":"value1","label2":"value2","_msg":"abc"}
 {"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
@@ -119,57 +121,6 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 {"x":"y","_msg":"yx"}`)
 }
 
-func TestParseProtobufRequest_ParseMessage(t *testing.T) {
-	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-
-		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
-		}
-
-		data := tlp.pr.MarshalProtobuf(nil)
-
-		tlp2 := &insertutil.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(data, tlp2, msgFields, false, true); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	f(`{
-	"streams": [
-		{
-			"stream": {
-				"foo": "bar",
-				"a": "b"
-			},
-			"values": [
-				["1577836800000000001", "{\"user_id\":\"123\"}"],
-				["1577836900005000002", "abc", {"trace_id":"pqw"}],
-				["1577836900005000003", "{def}"]
-			]
-		},
-		{
-			"stream": {
-				"x": "y"
-			},
-			"values": [
-				["1877836900005000004", "{\"trace_id\":\"432\",\"parent_id\":\"qwerty\"}"]
-			]
-		}
-	]
-}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
-{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
-{"foo":"bar","a":"b","_msg":"{def}"}
-{"x":"y","_msg":"432","parent_id":"qwerty"}`)
-}
-
 func TestParsePromLabels_Success(t *testing.T) {
 	f := func(s string) {
 		t.Helper()

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -6,7 +6,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
@@ -23,13 +25,13 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
 }
 
 func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseProtobufRequest(body, blp, nil, false, true); err != nil {
+			if err := parseProtobufRequest(body, blp, false); err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}
@@ -76,5 +78,8 @@ func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
 		Streams: streams,
 	}
 
-	return pr.MarshalProtobuf(nil)
+	body := pr.MarshalProtobuf(nil)
+	encodedBody := snappy.Encode(nil, body)
+
+	return encodedBody
 }

app/vlinsert/main.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
@@ -29,11 +28,6 @@ func Stop() {
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 
-	if path == "/internal/insert" {
-		internalinsert.RequestHandler(w, r)
-		return true
-	}
-
 	if !strings.HasPrefix(path, "/insert/") {
 		// Skip requests, which do not start with /insert/, since these aren't our requests.
 		return false
@@ -52,9 +46,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	switch {
-	case strings.HasPrefix(path, "/elasticsearch"):
-		// some clients may omit trailing slash
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
+	case strings.HasPrefix(path, "/elasticsearch/"):
 		path = strings.TrimPrefix(path, "/elasticsearch")
 		return elasticsearch.RequestHandler(path, w, r)
 	case strings.HasPrefix(path, "/loki/"):

app/vlinsert/opentelemetry/opentelemetry.go

@@ -2,21 +2,21 @@ package opentelemetry
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
-var maxRequestSize = flagutil.NewBytes("opentelemetry.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single OpenTelemetry request")
-
 // RequestHandler processes Opentelemetry insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
@@ -37,8 +37,26 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsProtobufTotal.Inc()
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
 
-	cp, err := insertutil.GetCommonParams(r)
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
+
+	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
@@ -48,16 +66,12 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		return
 	}
 
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf", false)
-		useDefaultStreamFields := len(cp.StreamFields) == 0
-		err := pushProtobufRequest(data, lmp, cp.MsgFields, useDefaultStreamFields)
-		lmp.MustClose()
-		return err
-	})
+	lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf")
+	useDefaultStreamFields := len(cp.StreamFields) == 0
+	err = pushProtobufRequest(data, lmp, useDefaultStreamFields)
+	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot read OpenTelemetry protocol data: %s", err)
+		httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
 		return
 	}
 
@@ -74,7 +88,7 @@ var (
 	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
 )
 
-func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {
+func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
 	var req pb.ExportLogsServiceRequest
 	if err := req.UnmarshalProtobuf(data); err != nil {
 		errorsTotal.Inc()
@@ -83,41 +97,33 @@ func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFie
 
 	var commonFields []logstorage.Field
 	for _, rl := range req.ResourceLogs {
-		commonFields = commonFields[:0]
-		commonFields = appendKeyValues(commonFields, rl.Resource.Attributes, "")
+		attributes := rl.Resource.Attributes
+		commonFields = slicesutil.SetLength(commonFields, len(attributes))
+		for i, attr := range attributes {
+			commonFields[i].Name = attr.Key
+			commonFields[i].Value = attr.Value.FormatString()
+		}
 		commonFieldsLen := len(commonFields)
 		for _, sc := range rl.ScopeLogs {
-			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, msgFields, useDefaultStreamFields)
+			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, useDefaultStreamFields)
 		}
 	}
 
 	return nil
 }
 
-func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) []logstorage.Field {
+func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) []logstorage.Field {
 	fields := commonFields
 	for _, lr := range sc.LogRecords {
 		fields = fields[:len(commonFields)]
-		if lr.Body.KeyValueList != nil {
-			fields = appendKeyValues(fields, lr.Body.KeyValueList.Values, "")
-			logstorage.RenameField(fields[len(commonFields):], msgFields, "_msg")
-		} else {
+		fields = append(fields, logstorage.Field{
+			Name:  "_msg",
+			Value: lr.Body.FormatString(),
+		})
+		for _, attr := range lr.Attributes {
 			fields = append(fields, logstorage.Field{
-				Name:  "_msg",
-				Value: lr.Body.FormatString(true),
-			})
-		}
-		fields = appendKeyValues(fields, lr.Attributes, "")
-		if len(lr.TraceID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "trace_id",
-				Value: lr.TraceID,
-			})
-		}
-		if len(lr.SpanID) > 0 {
-			fields = append(fields, logstorage.Field{
-				Name:  "span_id",
-				Value: lr.SpanID,
+				Name:  attr.Key,
+				Value: attr.Value.FormatString(),
 			})
 		}
 		fields = append(fields, logstorage.Field{
@@ -133,22 +139,3 @@ func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field,
 	}
 	return fields
 }
-
-func appendKeyValues(fields []logstorage.Field, kvs []*pb.KeyValue, parentField string) []logstorage.Field {
-	for _, attr := range kvs {
-		fieldName := attr.Key
-		if parentField != "" {
-			fieldName = parentField + "." + fieldName
-		}
-
-		if attr.Value.KeyValueList != nil {
-			fields = appendKeyValues(fields, attr.Value.KeyValueList.Values, fieldName)
-		} else {
-			fields = append(fields, logstorage.Field{
-				Name:  fieldName,
-				Value: attr.Value.FormatString(true),
-			})
-		}
-	}
-	return fields
-}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -3,7 +3,7 @@ package opentelemetry
 import (
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 )
 
@@ -15,8 +15,8 @@ func TestPushProtoOk(t *testing.T) {
 		}
 
 		pData := lr.MarshalProtobuf(nil)
-		tlp := &insertutil.TestLogMessageProcessor{}
-		if err := pushProtobufRequest(pData, tlp, nil, false); err != nil {
+		tlp := &insertutils.TestLogMessageProcessor{}
+		if err := pushProtobufRequest(pData, tlp, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 
@@ -24,7 +24,6 @@ func TestPushProtoOk(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
-
 	// single line without resource attributes
 	f([]pb.ResourceLogs{
 		{
@@ -40,27 +39,6 @@ func TestPushProtoOk(t *testing.T) {
 		[]int64{1234},
 		`{"_msg":"log-line-message","severity":"Trace"}`,
 	)
-
-	// severities mapping
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 13, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 24, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1234, 1234},
-		`{"_msg":"log-line-message","severity":"Trace"}
-{"_msg":"log-line-message","severity":"Warn"}
-{"_msg":"log-line-message","severity":"Fatal4"}`,
-	)
-
 	// multi-line with resource attributes
 	f([]pb.ResourceLogs{
 		{
@@ -80,7 +58,7 @@ func TestPushProtoOk(t *testing.T) {
 				{
 					LogRecords: []pb.LogRecord{
 						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 25, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
 						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
 					},
 				},
@@ -88,9 +66,9 @@ func TestPushProtoOk(t *testing.T) {
 		},
 	},
 		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
+		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
+{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
 	)
 
 	// multi-scope with resource attributes and multi-line
@@ -128,79 +106,20 @@ func TestPushProtoOk(t *testing.T) {
 				{
 					LogRecords: []pb.LogRecord{
 						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
-						{TraceID: "1234", SpanID: "45", ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
-						{TraceID: "4bf92f3577b34da6a3ce929d0e0e4736", SpanID: "00f067aa0ba902b7", ObservedTimeUnixNano: 3333, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-2")}},
+						{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
 					},
 				},
 			},
 		},
 	},
-		[]int64{1234, 1235, 2345, 2346, 2347, 2348, 3333},
-		`{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints.role":"dev","node_taints.cluster_load_percent":"0.55","_msg":"log-line-message-msg-2","severity":"Debug"}
+		[]int64{1234, 1235, 2345, 2346, 2347, 2348},
+		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
 {"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-1","trace_id":"1234","span_id":"45","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-2","trace_id":"4bf92f3577b34da6a3ce929d0e0e4736","span_id":"00f067aa0ba902b7","severity":"Unspecified"}`,
+{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
 	)
-
-	// nested fields
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{
-							TimeUnixNano: 1234,
-							Body:         pb.AnyValue{StringValue: ptrTo("nested fields")},
-							Attributes: []*pb.KeyValue{
-								{Key: "error", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-									{
-										Key:   "type",
-										Value: &pb.AnyValue{StringValue: ptrTo("document_parsing_exception")},
-									},
-									{
-										Key:   "reason",
-										Value: &pb.AnyValue{StringValue: ptrTo("failed to parse field [_msg] of type [text]")},
-									},
-									{
-										Key: "caused_by",
-										Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-											{
-												Key:   "type",
-												Value: &pb.AnyValue{StringValue: ptrTo("x_content_parse_exception")},
-											},
-											{
-												Key:   "reason",
-												Value: &pb.AnyValue{StringValue: ptrTo("unexpected end-of-input in VALUE_STRING")},
-											},
-											{
-												Key: "caused_by",
-												Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{Values: []*pb.KeyValue{
-													{
-														Key:   "type",
-														Value: &pb.AnyValue{StringValue: ptrTo("json_e_o_f_exception")},
-													},
-													{
-														Key:   "reason",
-														Value: &pb.AnyValue{StringValue: ptrTo("eof")},
-													},
-												}}},
-											},
-										}}},
-									},
-								}}}},
-							},
-						},
-					},
-				},
-			},
-		},
-	}, []int64{1234},
-		`{"_msg":"nested fields","error.type":"document_parsing_exception","error.reason":"failed to parse field [_msg] of type [text]",`+
-			`"error.caused_by.type":"x_content_parse_exception","error.caused_by.reason":"unexpected end-of-input in VALUE_STRING",`+
-			`"error.caused_by.caused_by.type":"json_e_o_f_exception","error.caused_by.caused_by.reason":"eof","severity":"Unspecified"}`)
 }
 
 func ptrTo[T any](s T) *T {

app/vlinsert/opentelemetry/opentelemetry_timing_test.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 )
 
@@ -21,13 +21,13 @@ func BenchmarkParseProtobufRequest(b *testing.B) {
 }
 
 func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutil.BenchmarkLogMessageProcessor{}
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
 	b.ReportAllocs()
 	b.SetBytes(int64(streams * rows))
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			if err := pushProtobufRequest(body, blp, nil, false); err != nil {
+			if err := pushProtobufRequest(body, blp, false); err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}

app/vlinsert/syslog/syslog.go

@@ -16,7 +16,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/klauspost/compress/gzip"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
@@ -25,7 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
@@ -45,11 +47,6 @@ var (
 	ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
 		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
 
-	decolorizeFieldsTCP = flagutil.NewArrayString("syslog.decolorizeFields.tcp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-	decolorizeFieldsUDP = flagutil.NewArrayString("syslog.decolorizeFields.udp", "Fields to remove ANSI color codes across logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#decolorizing-fields`)
-
 	extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
 		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
 	extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
@@ -193,12 +190,6 @@ func runUDPListener(addr string, argIdx int) {
 		logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
 	}
 
-	decolorizeFieldsStr := decolorizeFieldsUDP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.udp=%q for -syslog.listenAddr.udp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
 	extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
 	extraFields, err := parseExtraFields(extraFieldsStr)
 	if err != nil {
@@ -207,7 +198,7 @@ func runUDPListener(addr string, argIdx int) {
 
 	doneCh := make(chan struct{})
 	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
+		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
 		close(doneCh)
 	}()
 
@@ -260,12 +251,6 @@ func runTCPListener(addr string, argIdx int) {
 		logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
 	}
 
-	decolorizeFieldsStr := decolorizeFieldsTCP.GetOptionalArg(argIdx)
-	decolorizeFields, err := parseFieldsList(decolorizeFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.decolorizeFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", decolorizeFieldsStr, addr, err)
-	}
-
 	extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
 	extraFields, err := parseExtraFields(extraFieldsStr)
 	if err != nil {
@@ -274,7 +259,7 @@ func runTCPListener(addr string, argIdx int) {
 
 	doneCh := make(chan struct{})
 	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, decolorizeFields, extraFields)
+		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
 		close(doneCh)
 	}()
 
@@ -289,14 +274,14 @@ func runTCPListener(addr string, argIdx int) {
 
 func checkCompressMethod(compressMethod, addr, protocol string) {
 	switch compressMethod {
-	case "", "none", "zstd", "gzip", "deflate":
+	case "", "none", "gzip", "deflate":
 		return
 	default:
-		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
+		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
 	}
 }
 
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
 	gomaxprocs := cgroup.AvailableCPUs()
 	var wg sync.WaitGroup
 	localAddr := ln.LocalAddr()
@@ -304,7 +289,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
+			cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
 			var bb bytesutil.ByteBuffer
 			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
 			for {
@@ -329,7 +314,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 				}
 				bb.B = bb.B[:n]
 				udpRequestsTotal.Inc()
-				if err := processStream("udp", bb.NewReader(), encoding, useLocalTimestamp, cp); err != nil {
+				if err := processStream("udp", bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
 					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
 				}
 			}
@@ -338,7 +323,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string,
 	wg.Wait()
 }
 
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields, decolorizeFields []string, extraFields []logstorage.Field) {
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
 	var cm ingestserver.ConnsMap
 	cm.Init("syslog")
 
@@ -368,8 +353,8 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us
 
 		wg.Add(1)
 		go func() {
-			cp := insertutil.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, decolorizeFields, extraFields)
-			if err := processStream("tcp", c, encoding, useLocalTimestamp, cp); err != nil {
+			cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
+			if err := processStream("tcp", c, compressMethod, useLocalTimestamp, cp); err != nil {
 				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
 			}
 
@@ -384,29 +369,52 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us
 }
 
 // processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutil.CommonParams) error {
+func processStream(protocol string, r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
 	if err := vlstorage.CanWriteData(); err != nil {
 		return err
 	}
 
-	lmp := cp.NewLogMessageProcessor("syslog_"+protocol, true)
-	err := processStreamInternal(r, encoding, useLocalTimestamp, lmp)
+	lmp := cp.NewLogMessageProcessor("syslog_" + protocol)
+	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
 	lmp.MustClose()
 
 	return err
 }
 
-func processStreamInternal(r io.Reader, encoding string, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
-	reader, err := protoparserutil.GetUncompressedReader(r, encoding)
-	if err != nil {
-		return fmt.Errorf("cannot decode syslog data: %w", err)
+func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	switch compressMethod {
+	case "", "none":
+	case "gzip":
+		zr, err := common.GetGzipReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read gzipped data: %w", err)
+		}
+		r = zr
+	case "deflate":
+		zr, err := common.GetZlibReader(r)
+		if err != nil {
+			return fmt.Errorf("cannot read deflated data: %w", err)
+		}
+		r = zr
+	default:
+		logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
 	}
-	defer protoparserutil.PutUncompressedReader(reader)
 
-	return processUncompressedStream(reader, useLocalTimestamp, lmp)
+	err := processUncompressedStream(r, useLocalTimestamp, lmp)
+
+	switch compressMethod {
+	case "gzip":
+		zr := r.(*gzip.Reader)
+		common.PutGzipReader(zr)
+	case "deflate":
+		zr := r.(io.ReadCloser)
+		common.PutZlibReader(zr)
+	}
+
+	return err
 }
 
-func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
+func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
@@ -491,7 +499,7 @@ again:
 			slr.err = fmt.Errorf("cannot parse message length from %q: %w", msgLenStr, err)
 			return false
 		}
-		if maxMsgLen := insertutil.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
+		if maxMsgLen := insertutils.MaxLineSizeBytes.IntN(); msgLen > uint64(maxMsgLen) {
 			slr.err = fmt.Errorf("cannot read message longer than %d bytes; msgLen=%d", maxMsgLen, msgLen)
 			return false
 		}
@@ -543,7 +551,7 @@ func putSyslogLineReader(slr *syslogLineReader) {
 
 var syslogLineReaderPool sync.Pool
 
-func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutil.LogMessageProcessor) error {
+func processLine(line []byte, currentYear int, timezone *time.Location, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
 	p := logstorage.GetSyslogParser(currentYear, timezone)
 	lineStr := bytesutil.ToUnsafeString(line)
 	p.Parse(lineStr)
@@ -552,7 +560,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	if useLocalTimestamp {
 		ts = time.Now().UnixNano()
 	} else {
-		nsecs, err := insertutil.ExtractTimestampFromFields(timeFields, p.Fields)
+		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
 		if err != nil {
 			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
 		}
@@ -565,7 +573,6 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	return nil
 }
 
-var timeFields = []string{"timestamp"}
 var msgFields = []string{"message"}
 
 var (

app/vlinsert/syslog/syslog_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
 func TestSyslogLineReader_Success(t *testing.T) {
@@ -84,7 +84,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 		globalTimezone = time.UTC
 		globalCurrentYear.Store(int64(currentYear))
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
 		if err := processStreamInternal(r, "", false, tlp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -113,7 +113,7 @@ func TestProcessStreamInternal_Failure(t *testing.T) {
 		MustInit()
 		defer MustStop()
 
-		tlp := &insertutil.TestLogMessageProcessor{}
+		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
 		if err := processStreamInternal(r, "", false, tlp); err == nil {
 			t.Fatalf("expecting non-nil error")

app/vlogscli/json_prettifier.go

@@ -154,7 +154,7 @@ func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
 		}
 		value, ok := t.(string)
 		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object value: %v; want string", t)
+			return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
 		}
 
 		fields = append(fields, logstorage.Field{
@@ -176,7 +176,7 @@ func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
 		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
 		return err
 	}
-	if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
+	if len(fields) == 2 && fields[0].Name == "_time" || fields[1].Name == "_time" {
 		// Write _time\tfieldValue as is
 		if fields[0].Name == "_time" {
 			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)

app/vlogscli/less_wrapper.go

@@ -17,7 +17,7 @@ func isTerminal() bool {
 	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
 }
 
-func readWithLess(r io.Reader, disableColors, wrapLongLines bool) error {
+func readWithLess(r io.Reader, wrapLongLines bool) error {
 	if !isTerminal() {
 		// Just write everything to stdout if no terminal is available.
 		_, err := io.Copy(os.Stdout, r)
@@ -49,9 +49,6 @@ func readWithLess(r io.Reader, disableColors, wrapLongLines bool) error {
 		return fmt.Errorf("cannot find 'less' command: %w", err)
 	}
 	opts := []string{"less", "-F", "-X"}
-	if !disableColors {
-		opts = append(opts, "-R")
-	}
 	if !wrapLongLines {
 		opts = append(opts, "-S")
 	}

app/vlogscli/main.go

@@ -91,7 +91,6 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 	}
 
 	outputMode := outputModeJSONMultiline
-	disableColors := true
 	wrapLongLines := false
 	s := ""
 	for {
@@ -101,7 +100,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			case io.EOF:
 				if s != "" {
 					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode, disableColors, wrapLongLines)
+					executeQuery(context.Background(), rl, s, outputMode, wrapLongLines)
 				}
 				return
 			case readline.ErrInterrupt:
@@ -177,24 +176,6 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			s = ""
 			continue
 		}
-		if s == `\disable_colors` {
-			if !disableColors {
-				disableColors = true
-				fmt.Fprintf(rl, `disabled colors in compact output mode; enter \enable_colors for enabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\enable_colors` {
-			if disableColors {
-				disableColors = false
-				fmt.Fprintf(rl, `enabled colors in compact output mode; type \disable_colors for disabling it`+"\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
 		if line != "" && !strings.HasSuffix(line, ";") {
 			// Assume the query is incomplete and allow the user finishing the query on the next line
 			s += "\n"
@@ -204,7 +185,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 
 		// Execute the query
 		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode, disableColors, wrapLongLines)
+		executeQuery(ctx, rl, s, outputMode, wrapLongLines)
 		cancel()
 
 		historyLines = pushToHistory(rl, historyLines, s)
@@ -289,18 +270,16 @@ func printCommandsHelp(w io.Writer) {
 \h - show this help
 \s - singleline json output mode
 \m - multiline json output mode
-\c - compact output mode
+\c - compact output
 \logfmt - logfmt output mode
 \wrap_long_lines - toggles wrapping long lines
-\enable_colors - enable ANSI colors in compact output mode
-\disable_colors - disable ANSI colors in compact output mode
 \tail <query> - live tail <query> results
 
 See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
 `)
 }
 
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, disableColors, wrapLongLines bool) {
+func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, wrapLongLines bool) {
 	if strings.HasPrefix(qStr, `\tail `) {
 		tailQuery(ctx, output, qStr, outputMode)
 		return
@@ -314,7 +293,7 @@ func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode
 		_ = respBody.Close()
 	}()
 
-	if err := readWithLess(respBody, disableColors, wrapLongLines); err != nil {
+	if err := readWithLess(respBody, wrapLongLines); err != nil {
 		fmt.Fprintf(output, "error when reading query response: %s\n", err)
 		return
 	}

app/vlogsgenerator/main.go

@@ -18,20 +18,20 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 var (
 	addr    = flag.String("addr", "stdout", "HTTP address to push the generated logs to; if it is set to stdout, then logs are generated to stdout")
 	workers = flag.Int("workers", 1, "The number of workers to use to push logs to -addr")
 
-	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
-	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats")
+	start         = newTimeFlag("start", "-1d", "Generated logs start from this time; see https://docs.victoriametrics.com/#timestamp-formats")
+	end           = newTimeFlag("end", "0s", "Generated logs end at this time; see https://docs.victoriametrics.com/#timestamp-formats")
 	activeStreams = flag.Int("activeStreams", 100, "The number of active log streams to generate; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields")
 	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
 		"during data generation")
 	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
-	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constant values to generate per each log entry; "+
+	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
@@ -45,8 +45,6 @@ var (
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	i64FieldsPerLog = flag.Int("i64FieldsPerLog", 1, "The number of fields with int64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
@@ -177,10 +175,7 @@ func generateAndPushLogs(cfg *workerConfig, workerID int) {
 	sw := &statWriter{
 		w: pw,
 	}
-
-	// The 1MB write buffer increases data ingestion performance by reducing the number of send() syscalls
-	bw := bufio.NewWriterSize(sw, 1024*1024)
-
+	bw := bufio.NewWriter(sw)
 	doneCh := make(chan struct{})
 	go func() {
 		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)
@@ -206,8 +201,6 @@ func generateAndPushLogs(cfg *workerConfig, workerID int) {
 	if err != nil {
 		logger.Fatalf("cannot perform request to %q: %s", cfg.url, err)
 	}
-	defer resp.Body.Close()
-
 	if resp.StatusCode/100 != 2 {
 		logger.Fatalf("unexpected status code got from %q: %d; want 2xx", cfg.url, err)
 	}
@@ -261,9 +254,6 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 		for j := 0; j < *u64FieldsPerLog; j++ {
 			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
 		}
-		for j := 0; j < *i64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"i64_%d":"%d"`, j, int64(rand.Uint64()))
-		}
 		for j := 0; j < *floatFieldsPerLog; j++ {
 			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
 		}
@@ -311,7 +301,7 @@ type timeFlag struct {
 }
 
 func (tf *timeFlag) Set(s string) error {
-	msec, err := timeutil.ParseTimeMsec(s)
+	msec, err := promutils.ParseTimeMsec(s)
 	if err != nil {
 		return fmt.Errorf("cannot parse time from %q: %w", s, err)
 	}

app/vlselect/internalselect/internalselect.go

@@ -1,324 +0,0 @@
-package internalselect
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-)
-
-var disableSelect = flag.Bool("internalselect.disable", false, "Whether to disable /internal/select/* HTTP endpoints")
-
-// RequestHandler processes requests to /internal/select/*
-func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	if *disableSelect {
-		httpserver.Errorf(w, r, "requests to /internal/select/* are disabled with -internalselect.disable command-line flag")
-		return
-	}
-
-	startTime := time.Now()
-
-	path := r.URL.Path
-	rh := requestHandlers[path]
-	if rh == nil {
-		httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
-		return
-	}
-
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
-	if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		// The return is skipped intentionally in order to track the duration of failed queries.
-	}
-	metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
-}
-
-var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
-	"/internal/select/query":               processQueryRequest,
-	"/internal/select/field_names":         processFieldNamesRequest,
-	"/internal/select/field_values":        processFieldValuesRequest,
-	"/internal/select/stream_field_names":  processStreamFieldNamesRequest,
-	"/internal/select/stream_field_values": processStreamFieldValuesRequest,
-	"/internal/select/streams":             processStreamsRequest,
-	"/internal/select/stream_ids":          processStreamIDsRequest,
-}
-
-func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	var wLock sync.Mutex
-	var dataLenBuf []byte
-
-	sendBuf := func(bb *bytesutil.ByteBuffer) error {
-		if len(bb.B) == 0 {
-			return nil
-		}
-
-		data := bb.B
-		if !cp.DisableCompression {
-			bufLen := len(bb.B)
-			bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
-			data = bb.B[bufLen:]
-		}
-
-		wLock.Lock()
-		dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
-		_, err := w.Write(dataLenBuf)
-		if err == nil {
-			_, err = w.Write(data)
-		}
-		wLock.Unlock()
-
-		// Reset the sent buf
-		bb.Reset()
-
-		return err
-	}
-
-	var bufs atomicutil.Slice[bytesutil.ByteBuffer]
-
-	var errGlobalLock sync.Mutex
-	var errGlobal error
-
-	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
-		if errGlobal != nil {
-			return
-		}
-
-		bb := bufs.Get(workerID)
-
-		bb.B = db.Marshal(bb.B)
-
-		if len(bb.B) < 1024*1024 {
-			// Fast path - the bb is too small to be sent to the client yet.
-			return
-		}
-
-		// Slow path - the bb must be sent to the client.
-		if err := sendBuf(bb); err != nil {
-			errGlobalLock.Lock()
-			if errGlobal != nil {
-				errGlobal = err
-			}
-			errGlobalLock.Unlock()
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
-		return err
-	}
-	if errGlobal != nil {
-		return errGlobal
-	}
-
-	// Send the remaining data
-	for _, bb := range bufs.All() {
-		if err := sendBuf(bb); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streams, cp.DisableCompression)
-}
-
-func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
-}
-
-type commonParams struct {
-	TenantIDs []logstorage.TenantID
-	Query     *logstorage.Query
-
-	DisableCompression bool
-}
-
-func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
-	version := r.FormValue("version")
-	if version != expectedProtocolVersion {
-		return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
-	}
-
-	tenantIDsStr := r.FormValue("tenant_ids")
-	tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
-	}
-
-	timestamp, err := getInt64FromRequest(r, "timestamp")
-	if err != nil {
-		return nil, err
-	}
-
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
-	}
-
-	s := r.FormValue("disable_compression")
-	disableCompression, err := strconv.ParseBool(s)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
-	}
-
-	cp := &commonParams{
-		TenantIDs: tenantIDs,
-		Query:     q,
-
-		DisableCompression: disableCompression,
-	}
-	return cp, nil
-}
-
-func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
-	var b []byte
-	for i := range vhs {
-		b = vhs[i].Marshal(b)
-	}
-
-	if !disableCompression {
-		b = zstd.CompressLevel(nil, b, 1)
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	if _, err := w.Write(b); err != nil {
-		return fmt.Errorf("cannot send response to the client: %w", err)
-	}
-
-	return nil
-}
-
-func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
-	s := r.FormValue(argName)
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
-	}
-	return n, nil
-}

app/vlselect/logsql/buffered_writer.go

@@ -0,0 +1,47 @@
+package logsql
+
+import (
+	"bufio"
+	"io"
+	"sync"
+)
+
+func getBufferedWriter(w io.Writer) *bufferedWriter {
+	v := bufferedWriterPool.Get()
+	if v == nil {
+		return &bufferedWriter{
+			bw: bufio.NewWriter(w),
+		}
+	}
+	bw := v.(*bufferedWriter)
+	bw.bw.Reset(w)
+	return bw
+}
+
+func putBufferedWriter(bw *bufferedWriter) {
+	bw.reset()
+	bufferedWriterPool.Put(bw)
+}
+
+var bufferedWriterPool sync.Pool
+
+type bufferedWriter struct {
+	mu sync.Mutex
+	bw *bufio.Writer
+}
+
+func (bw *bufferedWriter) reset() {
+	// nothing to do
+}
+
+func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
+	bw.mu.Lock()
+	_, _ = bw.bw.Write(p)
+	bw.mu.Unlock()
+}
+
+func (bw *bufferedWriter) FlushIgnoreErrors() {
+	bw.mu.Lock()
+	_ = bw.bw.Flush()
+	bw.mu.Unlock()
+}

app/vlselect/logsql/logsql.go

@@ -3,7 +3,6 @@ package logsql
 import (
 	"context"
 	"fmt"
-	"io"
 	"math"
 	"net/http"
 	"regexp"
@@ -12,20 +11,18 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 // ProcessFacetsRequest handles /select/logsql/facets request.
@@ -38,38 +35,32 @@ func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	limit, err := httputil.GetInt(r, "limit")
+	limit, err := httputils.GetInt(r, "limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	maxValuesPerField, err := httputil.GetInt(r, "max_values_per_field")
+	maxValuesPerField, err := httputils.GetInt(r, "max_values_per_field")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	maxValueLen, err := httputil.GetInt(r, "max_value_len")
+	maxValueLen, err := httputils.GetInt(r, "max_value_len")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	keepConstFields := httputil.GetBool(r, "keep_const_fields")
+	keepConstFields := httputils.GetBool(r, "keep_const_fields")
 
-	// Pipes must be dropped, since it is expected facets are obtained
-	// from the real logs stored in the database.
 	q.DropAllPipes()
-
 	q.AddFacetsPipe(limit, maxValuesPerField, maxValueLen, keepConstFields)
 
 	var mLock sync.Mutex
 	m := make(map[string][]facetEntry)
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		if rowsCount == 0 {
+	writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
 			return
 		}
-
-		columns := db.Columns
 		if len(columns) != 3 {
 			logger.Panicf("BUG: expecting 3 columns; got %d columns", len(columns))
 		}
@@ -125,7 +116,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	if stepStr == "" {
 		stepStr = "1d"
 	}
-	step, err := timeutil.ParseDuration(stepStr)
+	step, err := promutils.ParseDuration(stepStr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse 'step' arg: %s", err)
 		return
@@ -140,7 +131,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	if offsetStr == "" {
 		offsetStr = "0s"
 	}
-	offset, err := timeutil.ParseDuration(offsetStr)
+	offset, err := promutils.ParseDuration(offsetStr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse 'offset' arg: %s", err)
 		return
@@ -150,7 +141,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	fields := r.Form["field"]
 
 	// Obtain limit on the number of top fields entries.
-	fieldsLimit, err := httputil.GetInt(r, "fields_limit")
+	fieldsLimit, err := httputils.GetInt(r, "fields_limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
@@ -159,27 +150,23 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		fieldsLimit = 0
 	}
 
-	// Pipes must be dropped, since it is expected hits are obtained
-	// from the real logs stored in the database.
+	// Prepare the query for hits count.
 	q.DropAllPipes()
-
 	q.AddCountByTimePipe(int64(step), int64(offset), fields)
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		if rowsCount == 0 {
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
 			return
 		}
 
-		columns := db.Columns
 		timestampValues := columns[0].Values
 		hitsValues := columns[len(columns)-1].Values
 		columns = columns[1 : len(columns)-1]
 
 		bb := blockResultPool.Get()
-		for i := 0; i < rowsCount; i++ {
+		for i := range timestamps {
 			timestampStr := strings.Clone(timestampValues[i])
 			hitsStr := strings.Clone(hitsValues[i])
 			hits, err := strconv.ParseUint(hitsStr, 10, 64)
@@ -218,8 +205,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	WriteHitsSeries(w, m)
 }
 
-var blockResultPool bytesutil.ByteBufferPool
-
 func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
 	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
 		return m
@@ -295,10 +280,6 @@ func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Pipes must be dropped, since it is expected field names are obtained
-	// from the real logs stored in the database.
-	q.DropAllPipes()
-
 	// Obtain field names for the given query
 	fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
 	if err != nil {
@@ -329,7 +310,7 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
 	}
 
 	// Parse limit query arg
-	limit, err := httputil.GetInt(r, "limit")
+	limit, err := httputils.GetInt(r, "limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
@@ -338,10 +319,6 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
 		limit = 0
 	}
 
-	// Pipes must be dropped, since it is expected field values are obtained
-	// from the real logs stored in the database.
-	q.DropAllPipes()
-
 	// Obtain unique values for the given field
 	values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
@@ -364,10 +341,6 @@ func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter,
 		return
 	}
 
-	// Pipes must be dropped, since it is expected stream field names are obtained
-	// from the real logs stored in the database.
-	q.DropAllPipes()
-
 	// Obtain stream field names for the given query
 	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
 	if err != nil {
@@ -397,7 +370,7 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
 	}
 
 	// Parse limit query arg
-	limit, err := httputil.GetInt(r, "limit")
+	limit, err := httputils.GetInt(r, "limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
@@ -406,10 +379,6 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
 		limit = 0
 	}
 
-	// Pipes must be dropped, since it is expected stream field values are obtained
-	// from the real logs stored in the database.
-	q.DropAllPipes()
-
 	// Obtain stream field values for the given query and the given fieldName
 	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
@@ -432,7 +401,7 @@ func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http
 	}
 
 	// Parse limit query arg
-	limit, err := httputil.GetInt(r, "limit")
+	limit, err := httputils.GetInt(r, "limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
@@ -441,10 +410,6 @@ func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http
 		limit = 0
 	}
 
-	// Pipes must be dropped, since it is expected stream ids are obtained
-	// from the real logs stored in the database.
-	q.DropAllPipes()
-
 	// Obtain streamIDs for the given query
 	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
@@ -467,7 +432,7 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 	}
 
 	// Parse limit query arg
-	limit, err := httputil.GetInt(r, "limit")
+	limit, err := httputils.GetInt(r, "limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
@@ -476,10 +441,6 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 		limit = 0
 	}
 
-	// Pipes must be dropped, since it is expected stream are obtained
-	// from the real logs stored in the database.
-	q.DropAllPipes()
-
 	// Obtain streams for the given query
 	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
@@ -509,21 +470,21 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 
-	refreshIntervalMsecs, err := httputil.GetDuration(r, "refresh_interval", 1000)
+	refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
 	refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
 
-	startOffsetMsecs, err := httputil.GetDuration(r, "start_offset", 5*1000)
+	startOffsetMsecs, err := httputils.GetDuration(r, "start_offset", 5*1000)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
 	startOffset := startOffsetMsecs * 1e6
 
-	offsetMsecs, err := httputil.GetDuration(r, "offset", 1000)
+	offsetMsecs, err := httputils.GetDuration(r, "offset", 1000)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
@@ -543,11 +504,6 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 	if !ok {
 		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
 	}
-
-	w.Header().Set("Content-Type", "application/x-ndjson")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-	flusher.Flush()
-
 	qOrig := q
 	for {
 		q = qOrig.CloneWithTimeFilter(end, start, end)
@@ -610,8 +566,8 @@ func newTailProcessor(cancel func()) *tailProcessor {
 	}
 }
 
-func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
-	if db.RowsCount() == 0 {
+func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	if len(timestamps) == 0 {
 		return
 	}
 
@@ -623,8 +579,14 @@ func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
 	}
 
 	// Make sure columns contain _time field, since it is needed for proper tail work.
-	timestamps, ok := db.GetTimestamps(nil)
-	if !ok {
+	hasTime := false
+	for _, c := range columns {
+		if c.Name == "_time" {
+			hasTime = true
+			break
+		}
+	}
+	if !hasTime {
 		tp.err = fmt.Errorf("missing _time field")
 		tp.cancel()
 		return
@@ -633,8 +595,8 @@ func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
 	// Copy block rows to tp.perStreamRows
 	for i, timestamp := range timestamps {
 		streamID := ""
-		fields := make([]logstorage.Field, len(db.Columns))
-		for j, c := range db.Columns {
+		fields := make([]logstorage.Field, len(columns))
+		for j, c := range columns {
 			name := strings.Clone(c.Name)
 			value := strings.Clone(c.Values[i])
 
@@ -703,7 +665,7 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	if stepStr == "" {
 		stepStr = "1d"
 	}
-	step, err := timeutil.ParseDuration(stepStr)
+	step, err := promutils.ParseDuration(stepStr)
 	if err != nil {
 		err = fmt.Errorf("cannot parse 'step' arg: %s", err)
 		httpserver.SendPrometheusError(w, r, err)
@@ -726,25 +688,19 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	m := make(map[string]*statsSeries)
 	var mLock sync.Mutex
 
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-
-		columns := db.Columns
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := 0; i < rowsCount; i++ {
-			// Do not move q.GetTimestamp() outside writeBlock, since ts
-			// must be initialized to query timestamp for every processed log row.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8312
-			ts := q.GetTimestamp()
+		for i := range timestamps {
+			timestamp := q.GetTimestamp()
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if c.Name == "_time" {
 					nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
 					if ok {
-						ts = nsec
+						timestamp = nsec
 						continue
 					}
 				}
@@ -765,7 +721,7 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 					dst = logstorage.MarshalFieldsToJSON(dst, labels)
 					key := string(dst)
 					p := statsPoint{
-						Timestamp: ts,
+						Timestamp: timestamp,
 						Value:     strings.Clone(c.Values[i]),
 					}
 
@@ -843,14 +799,12 @@ func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *htt
 	var rowsLock sync.Mutex
 
 	timestamp := q.GetTimestamp()
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		columns := db.Columns
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := 0; i < rowsCount; i++ {
+		for i := range timestamps {
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if slices.Contains(byFields, c.Name) {
@@ -906,27 +860,17 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 	}
 
 	// Parse limit query arg
-	limit, err := httputil.GetInt(r, "limit")
+	limit, err := httputils.GetInt(r, "limit")
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
 
-	sw := &syncWriter{
-		w: w,
-	}
-
-	var bwShards atomicutil.Slice[bufferedWriter]
-	bwShards.Init = func(shard *bufferedWriter) {
-		shard.sw = sw
-	}
+	bw := getBufferedWriter(w)
 	defer func() {
-		shards := bwShards.All()
-		for _, shard := range shards {
-			shard.FlushIgnoreErrors()
-		}
+		bw.FlushIgnoreErrors()
+		putBufferedWriter(bw)
 	}()
-
 	w.Header().Set("Content-Type", "application/stream+json")
 
 	if limit > 0 {
@@ -936,34 +880,32 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 				httpserver.Errorf(w, r, "%s", err)
 				return
 			}
-			bw := bwShards.Get(0)
+			bb := blockResultPool.Get()
+			b := bb.B
 			for i := range rows {
-				bw.buf = logstorage.MarshalFieldsToJSON(bw.buf, rows[i].fields)
-				bw.buf = append(bw.buf, '\n')
-				if len(bw.buf) > 16*1024 {
-					bw.FlushIgnoreErrors()
-				}
+				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
+				b = append(b, '\n')
+				bw.WriteIgnoreErrors(b)
 			}
+			bb.B = b
+			blockResultPool.Put(bb)
 			return
 		}
 
 		q.AddPipeLimit(uint64(limit))
 	}
 
-	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		if rowsCount == 0 {
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
 			return
 		}
-		columns := db.Columns
 
-		bw := bwShards.Get(workerID)
-		for i := 0; i < rowsCount; i++ {
-			WriteJSONRow(bw, columns, i)
-			if len(bw.buf) > 16*1024 {
-				bw.FlushIgnoreErrors()
-			}
+		bb := blockResultPool.Get()
+		for i := range timestamps {
+			WriteJSONRow(bb, columns, i)
 		}
+		bw.WriteIgnoreErrors(bb.B)
+		blockResultPool.Put(bb)
 	}
 
 	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
@@ -972,37 +914,14 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 	}
 }
 
-type syncWriter struct {
-	mu sync.Mutex
-	w  io.Writer
+var blockResultPool bytesutil.ByteBufferPool
+
+type row struct {
+	timestamp int64
+	fields    []logstorage.Field
 }
 
-func (sw *syncWriter) Write(p []byte) (int, error) {
-	sw.mu.Lock()
-	n, err := sw.w.Write(p)
-	sw.mu.Unlock()
-	return n, err
-}
-
-type bufferedWriter struct {
-	buf []byte
-	sw  *syncWriter
-}
-
-func (bw *bufferedWriter) Write(p []byte) (int, error) {
-	bw.buf = append(bw.buf, p...)
-
-	// Do not send bw.buf to bw.sw here, since the data at bw.buf may be incomplete (it must end with '\n')
-
-	return len(p), nil
-}
-
-func (bw *bufferedWriter) FlushIgnoreErrors() {
-	_, _ = bw.sw.Write(bw.buf)
-	bw.buf = bw.buf[:0]
-}
-
-func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
+func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
 
@@ -1071,39 +990,28 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 	}
 }
 
-func getLastNRows(rows []logRow, limit int) []logRow {
-	sortLogRows(rows)
+func getLastNRows(rows []row, limit int) []row {
+	sort.Slice(rows, func(i, j int) bool {
+		return rows[i].timestamp < rows[j].timestamp
+	})
 	if len(rows) > limit {
 		rows = rows[len(rows)-limit:]
 	}
 	return rows
 }
 
-func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
+func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
 	ctxWithCancel, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	var missingTimeColumn atomic.Bool
-	var rows []logRow
+	var rows []row
 	var rowsLock sync.Mutex
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		if missingTimeColumn.Load() {
-			return
-		}
-
-		columns := db.Columns
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
 
-		timestamps, ok := db.GetTimestamps(nil)
-		if !ok {
-			missingTimeColumn.Store(true)
-			cancel()
-			return
-		}
-
 		for i, timestamp := range timestamps {
 			fields := make([]logstorage.Field, len(columns))
 			for j := range columns {
@@ -1113,7 +1021,7 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			}
 
 			rowsLock.Lock()
-			rows = append(rows, logRow{
+			rows = append(rows, row{
 				timestamp: timestamp,
 				fields:    fields,
 			})
@@ -1124,13 +1032,11 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			cancel()
 		}
 	}
-	err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock)
-
-	if missingTimeColumn.Load() {
-		return nil, fmt.Errorf("missing _time column in the result for the query [%s]", q)
+	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
+		return nil, err
 	}
 
-	return rows, err
+	return rows, nil
 }
 
 func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {
@@ -1189,22 +1095,20 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	}
 
 	// Parse optional extra_filters
-	for _, extraFiltersStr := range r.Form["extra_filters"] {
-		extraFilters, err := parseExtraFilters(extraFiltersStr)
-		if err != nil {
-			return nil, nil, err
-		}
-		q.AddExtraFilters(extraFilters)
+	extraFiltersStr := r.FormValue("extra_filters")
+	extraFilters, err := parseExtraFilters(extraFiltersStr)
+	if err != nil {
+		return nil, nil, err
 	}
+	q.AddExtraFilters(extraFilters)
 
 	// Parse optional extra_stream_filters
-	for _, extraStreamFiltersStr := range r.Form["extra_stream_filters"] {
-		extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
-		if err != nil {
-			return nil, nil, err
-		}
-		q.AddExtraFilters(extraStreamFilters)
+	extraStreamFiltersStr := r.FormValue("extra_stream_filters")
+	extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
+	if err != nil {
+		return nil, nil, err
 	}
+	q.AddExtraFilters(extraStreamFilters)
 
 	return q, tenantIDs, nil
 }
@@ -1215,7 +1119,7 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
 		return 0, false, nil
 	}
 	currentTimestamp := time.Now().UnixNano()
-	nsecs, err := timeutil.ParseTimeAt(s, currentTimestamp)
+	nsecs, err := promutils.ParseTimeAt(s, currentTimestamp)
 	if err != nil {
 		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
 	}

app/vlselect/logsql/logsql_test.go

@@ -28,7 +28,7 @@ func TestParseExtraFilters_Success(t *testing.T) {
 	// LogsQL filter
 	f(`foobar`, `foobar`)
 	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
+	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
 }
 
 func TestParseExtraFilters_Failure(t *testing.T) {
@@ -77,7 +77,7 @@ func TestParseExtraStreamFilters_Success(t *testing.T) {
 	// LogsQL filter
 	f(`foobar`, `foobar`)
 	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
+	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
 }
 
 func TestParseExtraStreamFilters_Failure(t *testing.T) {

app/vlselect/main.go

@@ -9,11 +9,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/internalselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -72,8 +71,7 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
 // RequestHandler handles select requests for VictoriaLogs
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
-
-	if !strings.HasPrefix(path, "/select/") && !strings.HasPrefix(path, "/internal/select/") {
+	if !strings.HasPrefix(path, "/select/") {
 		// Skip requests, which do not start with /select/, since these aren't our requests.
 		return false
 	}
@@ -100,75 +98,25 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	ctx := r.Context()
-	if path == "/select/logsql/tail" {
-		logsqlTailRequests.Inc()
-		// Process live tailing request without timeout, since it is OK to run live tailing requests for very long time.
-		// Also do not apply concurrency limit to tail requests, since these limits are intended for non-tail requests.
-		logsql.ProcessLiveTailRequest(ctx, w, r)
-		return true
-	}
-
 	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
 	startTime := time.Now()
+	ctx := r.Context()
 	d := getMaxQueryDuration(r)
 	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
 	defer cancel()
 
-	if !incRequestConcurrency(ctxWithTimeout, w, r) {
-		return true
-	}
-	defer decRequestConcurrency()
-
-	if strings.HasPrefix(path, "/internal/select/") {
-		// Process internal request from vlselect without timeout (e.g. use ctx instead of ctxWithTimeout),
-		// since the timeout must be controlled by the vlselect.
-		internalselect.RequestHandler(ctx, w, r)
-		return true
-	}
-
-	ok := processSelectRequest(ctxWithTimeout, w, r, path)
-	if !ok {
-		return false
-	}
-
-	logRequestErrorIfNeeded(ctxWithTimeout, w, r, startTime)
-	return true
-}
-
-func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http.Request, startTime time.Time) {
-	err := ctx.Err()
-	switch err {
-	case nil:
-		// nothing to do
-	case context.Canceled:
-		// do not log canceled requests, since they are expected and legal.
-	case context.DeadlineExceeded:
-		err = &httpserver.ErrorWithStatusCode{
-			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
-				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", time.Since(startTime).Seconds(), maxQueryDuration),
-			StatusCode: http.StatusServiceUnavailable,
-		}
-		httpserver.Errorf(w, r, "%s", err)
-	default:
-		httpserver.Errorf(w, r, "unexpected error: %s", err)
-	}
-}
-
-func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {
-	startTime := time.Now()
-	stopCh := ctx.Done()
+	stopCh := ctxWithTimeout.Done()
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		return true
+		defer func() { <-concurrencyLimitCh }()
 	default:
 		// Sleep for a while until giving up. This should resolve short bursts in requests.
 		concurrencyLimitReached.Inc()
 		select {
 		case concurrencyLimitCh <- struct{}{}:
-			return true
+			defer func() { <-concurrencyLimitCh }()
 		case <-stopCh:
-			switch ctx.Err() {
+			switch ctxWithTimeout.Err() {
 			case context.Canceled:
 				remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 				requestURI := httpserver.GetRequestURI(r)
@@ -181,78 +129,97 @@ func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.R
 						"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+
 						"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration=%s; to increase -search.maxConcurrentRequests; "+
 						"to pass bigger value to 'timeout' query arg",
-						time.Since(startTime).Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
+						d.Seconds(), *maxConcurrentRequests, maxQueueDuration, maxQueryDuration),
 					StatusCode: http.StatusServiceUnavailable,
 				}
 				httpserver.Errorf(w, r, "%s", err)
 			}
-			return false
+			return true
 		}
 	}
-}
 
-func decRequestConcurrency() {
-	<-concurrencyLimitCh
+	if path == "/select/logsql/tail" {
+		logsqlTailRequests.Inc()
+		// Process live tailing request without timeout (e.g. use ctx instead of ctxWithTimeout),
+		// since it is OK to run live tailing requests for very long time.
+		logsql.ProcessLiveTailRequest(ctx, w, r)
+		return true
+	}
+
+	ok := processSelectRequest(ctxWithTimeout, w, r, path)
+	if !ok {
+		return false
+	}
+
+	err := ctxWithTimeout.Err()
+	switch err {
+	case nil:
+		// nothing to do
+	case context.Canceled:
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Infof("client has canceled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+			time.Since(startTime).Seconds(), remoteAddr, requestURI)
+	case context.DeadlineExceeded:
+		err = &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
+				"to increase -search.maxQueryDuration=%s; to pass bigger value to 'timeout' query arg", d.Seconds(), maxQueryDuration),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+		httpserver.Errorf(w, r, "%s", err)
+	default:
+		httpserver.Errorf(w, r, "unexpected error: %s", err)
+	}
+
+	return true
 }
 
 func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
 	httpserver.EnableCORS(w, r)
-	startTime := time.Now()
 	switch path {
 	case "/select/logsql/facets":
 		logsqlFacetsRequests.Inc()
 		logsql.ProcessFacetsRequest(ctx, w, r)
-		logsqlFacetsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/field_names":
 		logsqlFieldNamesRequests.Inc()
 		logsql.ProcessFieldNamesRequest(ctx, w, r)
-		logsqlFieldNamesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/field_values":
 		logsqlFieldValuesRequests.Inc()
 		logsql.ProcessFieldValuesRequest(ctx, w, r)
-		logsqlFieldValuesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/hits":
 		logsqlHitsRequests.Inc()
 		logsql.ProcessHitsRequest(ctx, w, r)
-		logsqlHitsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/query":
 		logsqlQueryRequests.Inc()
 		logsql.ProcessQueryRequest(ctx, w, r)
-		logsqlQueryDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stats_query":
 		logsqlStatsQueryRequests.Inc()
 		logsql.ProcessStatsQueryRequest(ctx, w, r)
-		logsqlStatsQueryDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stats_query_range":
 		logsqlStatsQueryRangeRequests.Inc()
 		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
-		logsqlStatsQueryRangeDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_field_names":
 		logsqlStreamFieldNamesRequests.Inc()
 		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
-		logsqlStreamFieldNamesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_field_values":
 		logsqlStreamFieldValuesRequests.Inc()
 		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
-		logsqlStreamFieldValuesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_ids":
 		logsqlStreamIDsRequests.Inc()
 		logsql.ProcessStreamIDsRequest(ctx, w, r)
-		logsqlStreamIDsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/streams":
 		logsqlStreamsRequests.Inc()
 		logsql.ProcessStreamsRequest(ctx, w, r)
-		logsqlStreamsDuration.UpdateDuration(startTime)
 		return true
 	default:
 		return false
@@ -261,7 +228,7 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 
 // getMaxQueryDuration returns the maximum duration for query from r.
 func getMaxQueryDuration(r *http.Request) time.Duration {
-	dms, err := httputil.GetDuration(r, "timeout", 0)
+	dms, err := httputils.GetDuration(r, "timeout", 0)
 	if err != nil {
 		dms = 0
 	}
@@ -273,39 +240,16 @@ func getMaxQueryDuration(r *http.Request) time.Duration {
 }
 
 var (
-	logsqlFacetsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
-	logsqlFacetsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/facets"}`)
-
-	logsqlFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
-	logsqlFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_names"}`)
-
-	logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
-	logsqlFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_values"}`)
-
-	logsqlHitsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
-	logsqlHitsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/hits"}`)
-
-	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/query"}`)
-
-	logsqlStatsQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
-	logsqlStatsQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query"}`)
-
-	logsqlStatsQueryRangeRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
-	logsqlStatsQueryRangeDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query_range"}`)
-
-	logsqlStreamFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
-	logsqlStreamFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_names"}`)
-
+	logsqlFacetsRequests            = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
+	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
+	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
+	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
+	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlStatsQueryRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
+	logsqlStatsQueryRangeRequests   = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
+	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
 	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
-	logsqlStreamFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_values"}`)
-
-	logsqlStreamIDsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
-	logsqlStreamIDsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_ids"}`)
-
-	logsqlStreamsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
-	logsqlStreamsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/streams"}`)
-
-	// no need to track duration for tail requests, as they usually take long time
-	logsqlTailRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
+	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
+	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
+	logsqlTailRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
 )

app/vlselect/vmui/asset-manifest.json

@@ -0,0 +1,13 @@
+{
+  "files": {
+    "main.css": "./static/css/main.d05122da.css",
+    "main.js": "./static/js/main.6082e5a5.js",
+    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
+    "index.html": "./index.html"
+  },
+  "entrypoints": [
+    "static/css/main.d05122da.css",
+    "static/js/main.6082e5a5.js"
+  ]
+}

app/vlselect/vmui/assets/vendor-D1GxaB_c.css

@@ -1 +0,0 @@
-.uplot,.uplot *,.uplot *:before,.uplot *:after{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol,"Noto Color Emoji";line-height:1.5;width:min-content}.u-title{text-align:center;font-size:18px;font-weight:700}.u-wrap{position:relative;-webkit-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;position:relative;width:100%;height:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{vertical-align:middle;display:inline-block}.u-legend .u-marker{width:1em;height:1em;margin-right:4px;background-clip:padding-box!important}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:#00000012;position:absolute;pointer-events:none}.u-cursor-x,.u-cursor-y{position:absolute;left:0;top:0;pointer-events:none;will-change:transform}.u-hz .u-cursor-x,.u-vt .u-cursor-y{height:100%;border-right:1px dashed #607D8B}.u-hz .u-cursor-y,.u-vt .u-cursor-x{width:100%;border-bottom:1px dashed #607D8B}.u-cursor-pt{position:absolute;top:0;left:0;border-radius:50%;border:0 solid;pointer-events:none;will-change:transform;background-clip:padding-box!important}.u-axis.u-off,.u-select.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-cursor-pt.u-off{display:none}

app/vlselect/vmui/index.html

@@ -1,57 +1 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="./favicon.svg" />
-  <link rel="apple-touch-icon" href="./favicon.svg" />
-  <link rel="mask-icon" href="./favicon.svg" color="#000000">
-
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta name="description" content="Explore your log data with VictoriaLogs UI"/>
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="./manifest.json"/>
-  <!--
-    Notice the use of in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>UI for VictoriaLogs</title>
-
-  <meta name="twitter:card" content="summary">
-  <meta name="twitter:title" content="UI for VictoriaLogs">
-  <meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/">
-  <meta name="twitter:description" content="Explore your log data with VictoriaLogs UI">
-  <meta name="twitter:image" content="./preview.jpg">
-
-  <meta property="og:type" content="website">
-  <meta property="og:title" content="UI for VictoriaLogs">
-  <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
-  <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-BaRvaPfA.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
-  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-</body>
-</html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.6082e5a5.js"></script><link href="./static/css/main.d05122da.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/js/main.6082e5a5.js.LICENSE.txt

@@ -0,0 +1,38 @@
+/*!
+	Copyright (c) 2018 Jed Watson.
+	Licensed under the MIT License (MIT), see
+	http://jedwatson.github.io/classnames
+*/
+
+/**
+ * @remix-run/router v1.19.2
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */
+
+/**
+ * React Router DOM v6.26.2
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */
+
+/**
+ * React Router v6.26.2
+ *
+ * Copyright (c) Remix Software Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE.md file in the root directory of this source tree.
+ *
+ * @license MIT
+ */

app/vlstorage/main.go

@@ -10,14 +10,11 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
@@ -41,59 +38,15 @@ var (
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
 		"the storage stops accepting new data")
 
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge . It overrides -httpAuth.* . "+
-		"See https://docs.victoriametrics.com/victorialogs/#forced-merge")
-	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush . It overrides -httpAuth.* . "+
-		"See https://docs.victoriametrics.com/victorialogs/#forced-flush")
-
-	storageNodeAddrs = flagutil.NewArrayString("storageNode", "Comma-separated list of TCP addresses for storage nodes to route the ingested logs to and to send select queries to. "+
-		"If the list is empty, then the ingested logs are stored and queried locally from -storageDataPath")
-	insertConcurrency        = flag.Int("insert.concurrency", 2, "The average number of concurrent data ingestion requests, which can be sent to every -storageNode")
-	insertDisableCompression = flag.Bool("insert.disableCompression", false, "Whether to disable compression when sending the ingested data to -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-	selectDisableCompression = flag.Bool("select.disableCompression", false, "Whether to disable compression for select query responses received from -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-
-	storageNodeUsername     = flagutil.NewArrayString("storageNode.username", "Optional basic auth username to use for the corresponding -storageNode")
-	storageNodePassword     = flagutil.NewArrayString("storageNode.password", "Optional basic auth password to use for the corresponding -storageNode")
-	storageNodePasswordFile = flagutil.NewArrayString("storageNode.passwordFile", "Optional path to basic auth password to use for the corresponding -storageNode. "+
-		"The file is re-read every second")
-	storageNodeBearerToken     = flagutil.NewArrayString("storageNode.bearerToken", "Optional bearer auth token to use for the corresponding -storageNode")
-	storageNodeBearerTokenFile = flagutil.NewArrayString("storageNode.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -storageNode. "+
-		"The token is re-read from the file every second")
-
-	storageNodeTLS = flagutil.NewArrayBool("storageNode.tls", "Whether to use TLS (HTTPS) protocol for communicating with the corresponding -storageNode. "+
-		"By default communication is performed via HTTP")
-	storageNodeTLSCAFile = flagutil.NewArrayString("storageNode.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -storageNode. "+
-		"By default, system CA is used")
-	storageNodeTLSCertFile = flagutil.NewArrayString("storageNode.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
-		"to the corresponding -storageNode")
-	storageNodeTLSKeyFile    = flagutil.NewArrayString("storageNode.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -storageNode")
-	storageNodeTLSServerName = flagutil.NewArrayString("storageNode.tlsServerName", "Optional TLS server name to use for connections to the corresponding -storageNode. "+
-		"By default, the server name from -storageNode is used")
-	storageNodeTLSInsecureSkipVerify = flagutil.NewArrayBool("storageNode.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -storageNode")
+	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 )
 
-var localStorage *logstorage.Storage
-var localStorageMetrics *metrics.Set
-
-var netstorageInsert *netinsert.Storage
-var netstorageSelect *netselect.Storage
-
 // Init initializes vlstorage.
 //
 // Stop must be called when vlstorage is no longer needed
 func Init() {
-	if len(*storageNodeAddrs) == 0 {
-		initLocalStorage()
-	} else {
-		initNetworkStorage()
-	}
-}
-
-func initLocalStorage() {
-	if localStorage != nil {
-		logger.Panicf("BUG: initLocalStorage() has been already called")
+	if strg != nil {
+		logger.Panicf("BUG: Init() has been already called")
 	}
 
 	if retentionPeriod.Duration() < 24*time.Hour {
@@ -110,157 +63,60 @@ func initLocalStorage() {
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
-	localStorage = logstorage.MustOpenStorage(*storageDataPath, cfg)
+	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
 
 	var ss logstorage.StorageStats
-	localStorage.UpdateStats(&ss)
+	strg.UpdateStats(&ss)
 	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
 		"smallPartSize: %d bytes; bigPartSize: %d bytes",
 		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
 		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
 
-	// register local storage metrics
-	localStorageMetrics = metrics.NewSet()
-	localStorageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, localStorage)
+	// register storage metrics
+	storageMetrics = metrics.NewSet()
+	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
+		writeStorageMetrics(w, strg)
 	})
-	metrics.RegisterSet(localStorageMetrics)
-}
-
-func initNetworkStorage() {
-	if netstorageInsert != nil || netstorageSelect != nil {
-		logger.Panicf("BUG: initNetworkStorage() has been already called")
-	}
-
-	authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
-	isTLSs := make([]bool, len(*storageNodeAddrs))
-	for i := range authCfgs {
-		authCfgs[i] = newAuthConfigForStorageNode(i)
-		isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
-	}
-
-	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
-	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
-
-	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
-	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
-
-	logger.Infof("initialized all the network services")
-}
-
-func newAuthConfigForStorageNode(argIdx int) *promauth.Config {
-	username := storageNodeUsername.GetOptionalArg(argIdx)
-	password := storageNodePassword.GetOptionalArg(argIdx)
-	passwordFile := storageNodePasswordFile.GetOptionalArg(argIdx)
-	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
-		basicAuthCfg = &promauth.BasicAuthConfig{
-			Username:     username,
-			Password:     promauth.NewSecret(password),
-			PasswordFile: passwordFile,
-		}
-	}
-
-	token := storageNodeBearerToken.GetOptionalArg(argIdx)
-	tokenFile := storageNodeBearerTokenFile.GetOptionalArg(argIdx)
-
-	tlsCfg := &promauth.TLSConfig{
-		CAFile:             storageNodeTLSCAFile.GetOptionalArg(argIdx),
-		CertFile:           storageNodeTLSCertFile.GetOptionalArg(argIdx),
-		KeyFile:            storageNodeTLSKeyFile.GetOptionalArg(argIdx),
-		ServerName:         storageNodeTLSServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: storageNodeTLSInsecureSkipVerify.GetOptionalArg(argIdx),
-	}
-
-	opts := &promauth.Options{
-		BasicAuth:       basicAuthCfg,
-		BearerToken:     token,
-		BearerTokenFile: tokenFile,
-		TLSConfig:       tlsCfg,
-	}
-	ac, err := opts.NewConfig()
-	if err != nil {
-		logger.Panicf("FATAL: cannot populate auth config for storage node #%d: %s", argIdx, err)
-	}
-
-	return ac
+	metrics.RegisterSet(storageMetrics)
 }
 
 // Stop stops vlstorage.
 func Stop() {
-	if localStorage != nil {
-		metrics.UnregisterSet(localStorageMetrics, true)
-		localStorageMetrics = nil
+	metrics.UnregisterSet(storageMetrics, true)
+	storageMetrics = nil
 
-		localStorage.MustClose()
-		localStorage = nil
-	} else {
-		netstorageInsert.MustStop()
-		netstorageInsert = nil
-
-		netstorageSelect.MustStop()
-		netstorageSelect = nil
-	}
+	strg.MustClose()
+	strg = nil
 }
 
 // RequestHandler is a storage request handler.
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
-	switch path {
-	case "/internal/force_merge":
-		return processForceMerge(w, r)
-	case "/internal/force_flush":
-		return processForceFlush(w, r)
+	if path == "/internal/force_merge" {
+		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
+			return true
+		}
+		// Run force merge in background
+		partitionNamePrefix := r.FormValue("partition_prefix")
+		go func() {
+			activeForceMerges.Inc()
+			defer activeForceMerges.Dec()
+			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
+			startTime := time.Now()
+			strg.MustForceMerge(partitionNamePrefix)
+			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
+		}()
+		return true
 	}
 	return false
 }
 
-func processForceMerge(w http.ResponseWriter, r *http.Request) bool {
-	if localStorage == nil {
-		// Force merge isn't supported by non-local storage
-		return false
-	}
+var strg *logstorage.Storage
+var storageMetrics *metrics.Set
 
-	if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-		return true
-	}
-
-	// Run force merge in background
-	partitionNamePrefix := r.FormValue("partition_prefix")
-	go func() {
-		activeForceMerges.Inc()
-		defer activeForceMerges.Dec()
-		logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-		startTime := time.Now()
-		localStorage.MustForceMerge(partitionNamePrefix)
-		logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-	}()
-	return true
-}
-
-func processForceFlush(w http.ResponseWriter, r *http.Request) bool {
-	if localStorage == nil {
-		// Force merge isn't supported by non-local storage
-		return false
-	}
-
-	if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey) {
-		return true
-	}
-
-	logger.Infof("flushing storage to make pending data available for reading")
-	localStorage.DebugFlush()
-	return true
-}
-
-// CanWriteData returns non-nil error if it cannot write data to vlstorage
+// CanWriteData returns non-nil error if it cannot write data to vlstorage.
 func CanWriteData() error {
-	if localStorage == nil {
-		// The data can be always written in non-local mode.
-		return nil
-	}
-
-	if localStorage.IsReadOnly() {
+	if strg.IsReadOnly() {
 		return &httpserver.ErrorWithStatusCode{
 			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
 				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
@@ -274,77 +130,50 @@ func CanWriteData() error {
 //
 // It is advised to call CanWriteData() before calling MustAddRows()
 func MustAddRows(lr *logstorage.LogRows) {
-	if localStorage != nil {
-		// Store lr in the local storage.
-		localStorage.MustAddRows(lr)
-	} else {
-		// Store lr across the remote storage nodes.
-		lr.ForEachRow(netstorageInsert.AddRow)
-	}
+	strg.MustAddRows(lr)
 }
 
 // RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	if localStorage != nil {
-		return localStorage.RunQuery(ctx, tenantIDs, q, writeBlock)
-	}
-	return netstorageSelect.RunQuery(ctx, tenantIDs, q, writeBlock)
+func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
+	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
 }
 
 // GetFieldNames executes q and returns field names seen in results.
 func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetFieldNames(ctx, tenantIDs, q)
+	return strg.GetFieldNames(ctx, tenantIDs, q)
 }
 
 // GetFieldValues executes q and returns unique values for the fieldName seen in results.
 //
 // If limit > 0, then up to limit unique values are returned.
 func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreamFieldNames executes q and returns stream field names seen in results.
 func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetStreamFieldNames(ctx, tenantIDs, q)
+	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
 }
 
 // GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
 //
 // If limit > 0, then up to limit unique stream field values are returned.
 func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreams executes q and returns streams seen in query results.
 //
 // If limit > 0, then up to limit unique streams are returned.
 func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreams(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreams(ctx, tenantIDs, q, limit)
+	return strg.GetStreams(ctx, tenantIDs, q, limit)
 }
 
 // GetStreamIDs executes q and returns streamIDs seen in query results.
 //
 // If limit > 0, then up to limit unique streamIDs are returned.
 func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamIDs(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreamIDs(ctx, tenantIDs, q, limit)
+	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
 }
 
 func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {

app/vlstorage/netinsert/netinsert.go

@@ -1,369 +0,0 @@
-package netinsert
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/valyala/fastrand"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-)
-
-// the maximum size of a single data block sent to storage node.
-const maxInsertBlockSize = 2 * 1024 * 1024
-
-// ProtocolVersion is the version of the data ingestion protocol.
-//
-// It must be changed every time the data encoding at /internal/insert HTTP endpoint is changed.
-const ProtocolVersion = "v1"
-
-// Storage is a network storage for sending data to remote storage nodes in the cluster.
-type Storage struct {
-	sns []*storageNode
-
-	disableCompression bool
-
-	srt *streamRowsTracker
-
-	pendingDataBuffers chan *bytesutil.ByteBuffer
-
-	stopCh chan struct{}
-	wg     sync.WaitGroup
-}
-
-type storageNode struct {
-	// scheme is http or https scheme to communicate with addr
-	scheme string
-
-	// addr is TCP address of storage node to send the ingested data to
-	addr string
-
-	// s is a storage, which holds the given storageNode
-	s *Storage
-
-	// c is an http client used for sending data blocks to addr.
-	c *http.Client
-
-	// ac is auth config used for setting request headers such as Authorization and Host.
-	ac *promauth.Config
-
-	// pendingData contains pending data, which must be sent to the storage node at the addr.
-	pendingDataMu        sync.Mutex
-	pendingData          *bytesutil.ByteBuffer
-	pendingDataLastFlush time.Time
-
-	// the unix timestamp until the storageNode is disabled for data writing.
-	disabledUntil atomic.Uint64
-}
-
-func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
-	tr := httputil.NewTransport(false, "vlinsert_backend")
-	tr.TLSHandshakeTimeout = 20 * time.Second
-	tr.DisableCompression = true
-
-	scheme := "http"
-	if isTLS {
-		scheme = "https"
-	}
-
-	sn := &storageNode{
-		scheme: scheme,
-		addr:   addr,
-		s:      s,
-		c: &http.Client{
-			Transport: ac.NewRoundTripper(tr),
-		},
-		ac: ac,
-
-		pendingData: &bytesutil.ByteBuffer{},
-	}
-
-	s.wg.Add(1)
-	go func() {
-		defer s.wg.Done()
-		sn.backgroundFlusher()
-	}()
-
-	return sn
-}
-
-func (sn *storageNode) backgroundFlusher() {
-	t := time.NewTicker(time.Second)
-	defer t.Stop()
-
-	for {
-		select {
-		case <-sn.s.stopCh:
-			return
-		case <-t.C:
-			sn.flushPendingData()
-		}
-	}
-}
-
-func (sn *storageNode) flushPendingData() {
-	sn.pendingDataMu.Lock()
-	if time.Since(sn.pendingDataLastFlush) < time.Second {
-		// nothing to flush
-		sn.pendingDataMu.Unlock()
-		return
-	}
-
-	pendingData := sn.grabPendingDataForFlushLocked()
-	sn.pendingDataMu.Unlock()
-
-	sn.mustSendInsertRequest(pendingData)
-}
-
-func (sn *storageNode) addRow(r *logstorage.InsertRow) {
-	bb := bbPool.Get()
-	b := bb.B
-
-	b = r.Marshal(b)
-
-	if len(b) > maxInsertBlockSize {
-		logger.Warnf("skipping too long log entry, since its length exceeds %d bytes; the actual log entry length is %d bytes; log entry contents: %s", maxInsertBlockSize, len(b), b)
-		return
-	}
-
-	var pendingData *bytesutil.ByteBuffer
-	sn.pendingDataMu.Lock()
-	if sn.pendingData.Len()+len(b) > maxInsertBlockSize {
-		pendingData = sn.grabPendingDataForFlushLocked()
-	}
-	sn.pendingData.MustWrite(b)
-	sn.pendingDataMu.Unlock()
-
-	bb.B = b
-	bbPool.Put(bb)
-
-	if pendingData != nil {
-		sn.mustSendInsertRequest(pendingData)
-	}
-}
-
-var bbPool bytesutil.ByteBufferPool
-
-func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
-	sn.pendingDataLastFlush = time.Now()
-	pendingData := sn.pendingData
-	sn.pendingData = <-sn.s.pendingDataBuffers
-
-	return pendingData
-}
-
-func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
-	defer func() {
-		pendingData.Reset()
-		sn.s.pendingDataBuffers <- pendingData
-	}()
-
-	err := sn.sendInsertRequest(pendingData)
-	if err == nil {
-		return
-	}
-
-	if !errors.Is(err, errTemporarilyDisabled) {
-		logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
-	}
-	for !sn.s.sendInsertRequestToAnyNode(pendingData) {
-		logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
-
-		t := timerpool.Get(time.Second)
-		select {
-		case <-sn.s.stopCh:
-			timerpool.Put(t)
-			logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
-			return
-		case <-t.C:
-			timerpool.Put(t)
-		}
-	}
-}
-
-func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
-	dataLen := pendingData.Len()
-	if dataLen == 0 {
-		// Nothing to send.
-		return nil
-	}
-
-	if sn.disabledUntil.Load() > fasttime.UnixTimestamp() {
-		return errTemporarilyDisabled
-	}
-
-	ctx, cancel := contextutil.NewStopChanContext(sn.s.stopCh)
-	defer cancel()
-
-	var body io.Reader
-	if !sn.s.disableCompression {
-		bb := zstdBufPool.Get()
-		defer zstdBufPool.Put(bb)
-
-		bb.B = zstd.CompressLevel(bb.B[:0], pendingData.B, 1)
-		body = bb.NewReader()
-	} else {
-		body = pendingData.NewReader()
-	}
-
-	reqURL := sn.getRequestURL("/internal/insert")
-	req, err := http.NewRequestWithContext(ctx, "POST", reqURL, body)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating an http request: %s", err)
-	}
-	req.Header.Set("Content-Type", "application/octet-stream")
-	if !sn.s.disableCompression {
-		req.Header.Set("Content-Encoding", "zstd")
-	}
-	if err := sn.ac.SetHeaders(req, true); err != nil {
-		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
-	}
-
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		// Disable sn for data writing for 10 seconds.
-		sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-		return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 == 2 {
-		return nil
-	}
-
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		respBody = []byte(fmt.Sprintf("%s", err))
-	}
-
-	// Disable sn for data writing for 10 seconds.
-	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-	return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
-}
-
-func (sn *storageNode) getRequestURL(path string) string {
-	return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
-}
-
-var zstdBufPool bytesutil.ByteBufferPool
-
-// NewStorage returns new Storage for the given addrs with the given authCfgs.
-//
-// The concurrency is the average number of concurrent connections per every addr.
-//
-// If disableCompression is set, then the data is sent uncompressed to the remote storage.
-//
-// Call MustStop on the returned storage when it is no longer needed.
-func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
-	pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
-	for i := 0; i < cap(pendingDataBuffers); i++ {
-		pendingDataBuffers <- &bytesutil.ByteBuffer{}
-	}
-
-	s := &Storage{
-		disableCompression: disableCompression,
-		pendingDataBuffers: pendingDataBuffers,
-		stopCh:             make(chan struct{}),
-	}
-
-	sns := make([]*storageNode, len(addrs))
-	for i, addr := range addrs {
-		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
-	}
-	s.sns = sns
-
-	s.srt = newStreamRowsTracker(len(sns))
-
-	return s
-}
-
-// MustStop stops the s.
-func (s *Storage) MustStop() {
-	close(s.stopCh)
-	s.wg.Wait()
-	s.sns = nil
-}
-
-// AddRow adds the given log row into s.
-func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
-	idx := s.srt.getNodeIdx(streamHash)
-	sn := s.sns[idx]
-	sn.addRow(r)
-}
-
-func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
-	startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
-	for i := range s.sns {
-		idx := (startIdx + i) % len(s.sns)
-		sn := s.sns[idx]
-		err := sn.sendInsertRequest(pendingData)
-		if err == nil {
-			return true
-		}
-		if !errors.Is(err, errTemporarilyDisabled) {
-			logger.Warnf("cannot send pending data to the storage node %q: %s; trying to send it to another storage node", sn.addr, err)
-		}
-	}
-	return false
-}
-
-var errTemporarilyDisabled = fmt.Errorf("writing to the node is temporarily disabled")
-
-type streamRowsTracker struct {
-	mu sync.Mutex
-
-	nodesCount    int64
-	rowsPerStream map[uint64]uint64
-}
-
-func newStreamRowsTracker(nodesCount int) *streamRowsTracker {
-	return &streamRowsTracker{
-		nodesCount:    int64(nodesCount),
-		rowsPerStream: make(map[uint64]uint64),
-	}
-}
-
-func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
-	if srt.nodesCount == 1 {
-		// Fast path for a single node.
-		return 0
-	}
-
-	srt.mu.Lock()
-	defer srt.mu.Unlock()
-
-	streamRows := srt.rowsPerStream[streamHash] + 1
-	srt.rowsPerStream[streamHash] = streamRows
-
-	if streamRows <= 1000 {
-		// Write the initial rows for the stream to a single storage node for better locality.
-		// This should work great for log streams containing small number of logs, since will be distributed
-		// evenly among available storage nodes because they have different streamHash.
-		return streamHash % uint64(srt.nodesCount)
-	}
-
-	// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
-	// in order to improve query performance over this stream (the data for the log stream
-	// can be processed in parallel on all the storage nodes).
-	//
-	// The random distribution is preferred over round-robin distribution in order to avoid possible
-	// dependency between the order of the ingested logs and the number of storage nodes,
-	// which may lead to non-uniform distribution of logs among storage nodes.
-	return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
-}

app/vlstorage/netinsert/netinsert_test.go

@@ -1,57 +0,0 @@
-package netinsert
-
-import (
-	"fmt"
-	"math"
-	"math/rand"
-	"testing"
-
-	"github.com/cespare/xxhash/v2"
-)
-
-func TestStreamRowsTracker(t *testing.T) {
-	f := func(rowsCount, streamsCount, nodesCount int) {
-		t.Helper()
-
-		// generate stream hashes
-		streamHashes := make([]uint64, streamsCount)
-		for i := range streamHashes {
-			streamHashes[i] = xxhash.Sum64([]byte(fmt.Sprintf("stream %d.", i)))
-		}
-
-		srt := newStreamRowsTracker(nodesCount)
-
-		rng := rand.New(rand.NewSource(0))
-		rowsPerNode := make([]uint64, nodesCount)
-		for i := 0; i < rowsCount; i++ {
-			streamIdx := rng.Intn(streamsCount)
-			h := streamHashes[streamIdx]
-			nodeIdx := srt.getNodeIdx(h)
-			rowsPerNode[nodeIdx]++
-		}
-
-		// Verify that rows are uniformly distributed among nodes.
-		expectedRowsPerNode := float64(rowsCount) / float64(nodesCount)
-		for nodeIdx, nodeRows := range rowsPerNode {
-			if math.Abs(float64(nodeRows)-expectedRowsPerNode)/expectedRowsPerNode > 0.15 {
-				t.Fatalf("non-uniform distribution of rows among nodes; node %d has %d rows, while it must have %v rows; rowsPerNode=%d",
-					nodeIdx, nodeRows, expectedRowsPerNode, rowsPerNode)
-			}
-		}
-	}
-
-	rowsCount := 10000
-	streamsCount := 9
-	nodesCount := 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 10000
-	streamsCount = 100
-	nodesCount = 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 100000
-	streamsCount = 1000
-	nodesCount = 9
-	f(rowsCount, streamsCount, nodesCount)
-}

app/vlstorage/netselect/netselect.go

@@ -1,472 +0,0 @@
-package netselect
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"net/url"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-)
-
-const (
-	// FieldNamesProtocolVersion is the version of the protocol used for /internal/select/field_names HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	FieldNamesProtocolVersion = "v1"
-
-	// FieldValuesProtocolVersion is the version of the protocol used for /internal/select/field_values HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	FieldValuesProtocolVersion = "v1"
-
-	// StreamFieldNamesProtocolVersion is the version of the protocol used for /internal/select/stream_field_names HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamFieldNamesProtocolVersion = "v1"
-
-	// StreamFieldValuesProtocolVersion is the version of the protocol used for /internal/select/stream_field_values HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamFieldValuesProtocolVersion = "v1"
-
-	// StreamsProtocolVersion is the version of the protocol used for /internal/select/streams HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamsProtocolVersion = "v1"
-
-	// StreamIDsProtocolVersion is the version of the protocol used for /internal/select/stream_ids HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamIDsProtocolVersion = "v1"
-
-	// QueryProtocolVersion is the version of the protocol used for /internal/select/query HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	QueryProtocolVersion = "v1"
-)
-
-// Storage is a network storage for querying remote storage nodes in the cluster.
-type Storage struct {
-	sns []*storageNode
-
-	disableCompression bool
-}
-
-type storageNode struct {
-	// scheme is http or https scheme to communicate with addr
-	scheme string
-
-	// addr is TCP address of the storage node to query
-	addr string
-
-	// s is a storage, which holds the given storageNode
-	s *Storage
-
-	// c is an http client used for querying storage node at addr.
-	c *http.Client
-
-	// ac is auth config used for setting request headers such as Authorization and Host.
-	ac *promauth.Config
-}
-
-func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
-	tr := httputil.NewTransport(false, "vlselect_backend")
-	tr.TLSHandshakeTimeout = 20 * time.Second
-	tr.DisableCompression = true
-
-	scheme := "http"
-	if isTLS {
-		scheme = "https"
-	}
-
-	sn := &storageNode{
-		scheme: scheme,
-		addr:   addr,
-		s:      s,
-		c: &http.Client{
-			Transport: ac.NewRoundTripper(tr),
-		},
-		ac: ac,
-	}
-	return sn
-}
-
-func (sn *storageNode) runQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, processBlock func(db *logstorage.DataBlock)) error {
-	args := sn.getCommonArgs(QueryProtocolVersion, tenantIDs, q)
-
-	reqURL := sn.getRequestURL("/internal/select/query", args)
-	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
-	}
-	if err := sn.ac.SetHeaders(req, true); err != nil {
-		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
-	}
-
-	// send the request to the storage node
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		responseBody, err := io.ReadAll(resp.Body)
-		if err != nil {
-			responseBody = []byte(err.Error())
-		}
-		return fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
-	}
-
-	// read the response
-	var dataLenBuf [8]byte
-	var buf []byte
-	var db logstorage.DataBlock
-	var valuesBuf []string
-	for {
-		if _, err := io.ReadFull(resp.Body, dataLenBuf[:]); err != nil {
-			if errors.Is(err, io.EOF) {
-				// The end of response stream
-				return nil
-			}
-			return fmt.Errorf("cannot read block size from %q: %w", reqURL, err)
-		}
-		blockLen := encoding.UnmarshalUint64(dataLenBuf[:])
-		if blockLen > math.MaxInt {
-			return fmt.Errorf("too big data block: %d bytes; mustn't exceed %v bytes", blockLen, math.MaxInt)
-		}
-
-		buf = slicesutil.SetLength(buf, int(blockLen))
-		if _, err := io.ReadFull(resp.Body, buf); err != nil {
-			return fmt.Errorf("cannot read block with size of %d bytes from %q: %w", blockLen, reqURL, err)
-		}
-
-		src := buf
-		if !sn.s.disableCompression {
-			bufLen := len(buf)
-			var err error
-			buf, err = zstd.Decompress(buf, buf)
-			if err != nil {
-				return fmt.Errorf("cannot decompress data block: %w", err)
-			}
-			src = buf[bufLen:]
-		}
-
-		for len(src) > 0 {
-			tail, vb, err := db.UnmarshalInplace(src, valuesBuf[:0])
-			if err != nil {
-				return fmt.Errorf("cannot unmarshal data block received from %q: %w", reqURL, err)
-			}
-			valuesBuf = vb
-			src = tail
-
-			processBlock(&db)
-
-			clear(valuesBuf)
-		}
-	}
-}
-
-func (sn *storageNode) getFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(FieldNamesProtocolVersion, tenantIDs, q)
-
-	return sn.getValuesWithHits(ctx, "/internal/select/field_names", args)
-}
-
-func (sn *storageNode) getFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(FieldValuesProtocolVersion, tenantIDs, q)
-	args.Set("field", fieldName)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/field_values", args)
-}
-
-func (sn *storageNode) getStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamFieldNamesProtocolVersion, tenantIDs, q)
-
-	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_names", args)
-}
-
-func (sn *storageNode) getStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamFieldValuesProtocolVersion, tenantIDs, q)
-	args.Set("field", fieldName)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_values", args)
-}
-
-func (sn *storageNode) getStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamsProtocolVersion, tenantIDs, q)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/streams", args)
-}
-
-func (sn *storageNode) getStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamIDsProtocolVersion, tenantIDs, q)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/stream_ids", args)
-}
-
-func (sn *storageNode) getCommonArgs(version string, tenantIDs []logstorage.TenantID, q *logstorage.Query) url.Values {
-	args := url.Values{}
-	args.Set("version", version)
-	args.Set("tenant_ids", string(logstorage.MarshalTenantIDs(nil, tenantIDs)))
-	args.Set("query", q.String())
-	args.Set("timestamp", fmt.Sprintf("%d", q.GetTimestamp()))
-	args.Set("disable_compression", fmt.Sprintf("%v", sn.s.disableCompression))
-	return args
-}
-
-func (sn *storageNode) getValuesWithHits(ctx context.Context, path string, args url.Values) ([]logstorage.ValueWithHits, error) {
-	data, err := sn.executeRequestAt(ctx, path, args)
-	if err != nil {
-		return nil, err
-	}
-	return unmarshalValuesWithHits(data)
-}
-
-func (sn *storageNode) executeRequestAt(ctx context.Context, path string, args url.Values) ([]byte, error) {
-	reqURL := sn.getRequestURL(path, args)
-	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
-	}
-	if err := sn.ac.SetHeaders(req, true); err != nil {
-		return nil, fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
-	}
-
-	// send the request to the storage node
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		responseBody, err := io.ReadAll(resp.Body)
-		if err != nil {
-			responseBody = []byte(err.Error())
-		}
-		return nil, fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
-	}
-
-	// read the response
-	var bb bytesutil.ByteBuffer
-	if _, err := bb.ReadFrom(resp.Body); err != nil {
-		return nil, fmt.Errorf("cannot read response from %q: %w", reqURL, err)
-	}
-
-	if sn.s.disableCompression {
-		return bb.B, nil
-	}
-
-	bbLen := len(bb.B)
-	bb.B, err = zstd.Decompress(bb.B, bb.B)
-	if err != nil {
-		return nil, err
-	}
-	return bb.B[bbLen:], nil
-}
-
-func (sn *storageNode) getRequestURL(path string, args url.Values) string {
-	return fmt.Sprintf("%s://%s%s?%s", sn.scheme, sn.addr, path, args.Encode())
-}
-
-// NewStorage returns new Storage for the given addrs and the given authCfgs.
-//
-// If disableCompression is set, then uncompressed responses are received from storage nodes.
-//
-// Call MustStop on the returned storage when it is no longer needed.
-func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, disableCompression bool) *Storage {
-	s := &Storage{
-		disableCompression: disableCompression,
-	}
-
-	sns := make([]*storageNode, len(addrs))
-	for i, addr := range addrs {
-		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
-	}
-	s.sns = sns
-
-	return s
-}
-
-// MustStop stops the s.
-func (s *Storage) MustStop() {
-	s.sns = nil
-}
-
-// RunQuery runs the given q and calls writeBlock for the returned data blocks
-func (s *Storage) RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	nqr, err := logstorage.NewNetQueryRunner(ctx, tenantIDs, q, s.RunQuery, writeBlock)
-	if err != nil {
-		return err
-	}
-
-	search := func(stopCh <-chan struct{}, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-		return s.runQuery(stopCh, tenantIDs, q, writeBlock)
-	}
-
-	concurrency := q.GetConcurrency()
-	return nqr.Run(ctx, concurrency, search)
-}
-
-func (s *Storage) runQuery(stopCh <-chan struct{}, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	ctxWithCancel, cancel := contextutil.NewStopChanContext(stopCh)
-	defer cancel()
-
-	errs := make([]error, len(s.sns))
-
-	var wg sync.WaitGroup
-	for i := range s.sns {
-		wg.Add(1)
-		go func(nodeIdx int) {
-			defer wg.Done()
-			sn := s.sns[nodeIdx]
-			err := sn.runQuery(ctxWithCancel, tenantIDs, q, func(db *logstorage.DataBlock) {
-				writeBlock(uint(nodeIdx), db)
-			})
-			if err != nil {
-				// Cancel the remaining parallel queries
-				cancel()
-			}
-
-			errs[nodeIdx] = err
-		}(i)
-	}
-	wg.Wait()
-
-	return getFirstNonCancelError(errs)
-}
-
-// GetFieldNames executes q and returns field names seen in results.
-func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getFieldNames(ctx, tenantIDs, q)
-	})
-}
-
-// GetFieldValues executes q and returns unique values for the fieldName seen in results.
-//
-// If limit > 0, then up to limit unique values are returned.
-func (s *Storage) GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	})
-}
-
-// GetStreamFieldNames executes q and returns stream field names seen in results.
-func (s *Storage) GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreamFieldNames(ctx, tenantIDs, q)
-	})
-}
-
-// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
-//
-// If limit > 0, then up to limit unique stream field values are returned.
-func (s *Storage) GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	})
-}
-
-// GetStreams executes q and returns streams seen in query results.
-//
-// If limit > 0, then up to limit unique streams are returned.
-func (s *Storage) GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreams(ctx, tenantIDs, q, limit)
-	})
-}
-
-// GetStreamIDs executes q and returns streamIDs seen in query results.
-//
-// If limit > 0, then up to limit unique streamIDs are returned.
-func (s *Storage) GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreamIDs(ctx, tenantIDs, q, limit)
-	})
-}
-
-func (s *Storage) getValuesWithHits(ctx context.Context, limit uint64, resetHitsOnLimitExceeded bool,
-	callback func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error)) ([]logstorage.ValueWithHits, error) {
-
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	results := make([][]logstorage.ValueWithHits, len(s.sns))
-	errs := make([]error, len(s.sns))
-
-	var wg sync.WaitGroup
-	for i := range s.sns {
-		wg.Add(1)
-		go func(nodeIdx int) {
-			defer wg.Done()
-
-			sn := s.sns[nodeIdx]
-			vhs, err := callback(ctxWithCancel, sn)
-			results[nodeIdx] = vhs
-			errs[nodeIdx] = err
-
-			if err != nil {
-				// Cancel the remaining parallel requests
-				cancel()
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	if err := getFirstNonCancelError(errs); err != nil {
-		return nil, err
-	}
-
-	vhs := logstorage.MergeValuesWithHits(results, limit, resetHitsOnLimitExceeded)
-
-	return vhs, nil
-}
-
-func getFirstNonCancelError(errs []error) error {
-	for _, err := range errs {
-		if err != nil && !errors.Is(err, context.Canceled) {
-			return err
-		}
-	}
-	return nil
-}
-
-func unmarshalValuesWithHits(src []byte) ([]logstorage.ValueWithHits, error) {
-	var vhs []logstorage.ValueWithHits
-	for len(src) > 0 {
-		var vh logstorage.ValueWithHits
-		tail, err := vh.UnmarshalInplace(src)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unmarshal ValueWithHits #%d: %w", len(vhs), err)
-		}
-		src = tail
-
-		// Clone vh.Value, since it points to src.
-		vh.Value = strings.Clone(vh.Value)
-
-		vhs = append(vhs, vh)
-	}
-
-	return vhs, nil
-}

app/vmagent/README.md

@@ -1,3 +1,3 @@
-See vmagent docs [here](https://docs.victoriametrics.com/victoriametrics/vmagent/).
+See vmagent docs [here](https://docs.victoriametrics.com/vmagent/).
 
-vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmagent.md).
+vmagent docs can be edited at [docs/vmagent.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmagent.md).

app/vmagent/csvimport/request_handler.go

@@ -7,9 +7,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/csvimport/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -22,16 +22,16 @@ var (
 
 // InsertHandler processes csv data from req.
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []csvimport.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, rows []csvimport.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 

app/vmagent/datadogsketches/request_handler.go

@@ -7,10 +7,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogsketches/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -23,7 +23,7 @@ var (
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/beta/sketches request.
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
@@ -56,7 +56,7 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 				})
 			}
 			for _, tag := range sketch.Tags {
-				name, value := datadogutil.SplitTag(tag)
+				name, value := datadogutils.SplitTag(tag)
 				if name == "host" {
 					name = "exported_host"
 				}

app/vmagent/datadogv1/request_handler.go

@@ -7,10 +7,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv1/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -23,7 +23,7 @@ var (
 
 // InsertHandlerForHTTP processes remote write for DataDog POST /api/v1/series request.
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
@@ -62,7 +62,7 @@ func insertRows(at *auth.Token, series []datadogv1.Series, extraLabels []prompbm
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := datadogutil.SplitTag(tag)
+			name, value := datadogutils.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}

app/vmagent/datadogv2/request_handler.go

@@ -7,10 +7,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutil"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/datadogv2/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -25,7 +25,7 @@ var (
 //
 // See https://docs.datadoghq.com/api/latest/metrics/#submit-metrics
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
@@ -65,7 +65,7 @@ func insertRows(at *auth.Token, series []datadogv2.Series, extraLabels []prompbm
 			})
 		}
 		for _, tag := range ss.Tags {
-			name, value := datadogutil.SplitTag(tag)
+			name, value := datadogutils.SplitTag(tag)
 			if name == "host" {
 				name = "exported_host"
 			}

app/vmagent/graphite/request_handler.go

@@ -21,7 +21,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, "", func(rows []parser.Row) error {
+	return stream.Parse(r, false, func(rows []parser.Row) error {
 		return insertRows(nil, rows)
 	})
 }

app/vmagent/influx/request_handler.go

@@ -12,9 +12,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -35,8 +35,8 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
-	return stream.Parse(r, encoding, true, "", "", func(db string, rows []influx.Row) error {
+func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
+	return stream.Parse(r, true, isGzipped, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, nil)
 	})
 }
@@ -45,22 +45,22 @@ func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error
 //
 // See https://github.com/influxdata/influxdb/blob/4cbdc197b8117fee648d62e2e5be75c6575352f0/tsdb/README.md
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	isStreamMode := req.Header.Get("Stream-Mode") == "1"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	encoding := req.Header.Get("Content-Encoding")
-	isStreamMode := req.Header.Get("Stream-Mode") == "1"
-	return stream.Parse(req.Body, encoding, isStreamMode, precision, db, func(db string, rows []influx.Row) error {
+	return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, db string, rows []influx.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := getPushCtx()
 	defer putPushCtx(ctx)
 

app/vmagent/main.go

@@ -30,11 +30,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
@@ -42,8 +41,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
@@ -98,15 +97,6 @@ var (
 )
 
 func main() {
-	// vmagent is optimized for reduced memory allocations,
-	// so it can run with the reduced GOGC in order to reduce the used memory,
-	// while keeping CPU usage spent in GC at low levels.
-	//
-	// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
-	// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
-	// exceeds 0.05 for extended periods of time.
-	cgroup.SetGOGC(50)
-
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
@@ -145,10 +135,10 @@ func main() {
 	startTime := time.Now()
 	remotewrite.StartIngestionRateLimiter()
 	remotewrite.Init()
-	protoparserutil.StartUnmarshalWorkers()
+	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, "")
+			return influx.InsertHandlerForReader(nil, r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
@@ -165,9 +155,7 @@ func main() {
 
 	promscrape.Init(remotewrite.PushDropSamplesOnFailure)
 
-	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
-		UseProxyProtocol: useProxyProtocol,
-	})
+	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -197,7 +185,7 @@ func main() {
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		opentsdbhttpServer.MustStop()
 	}
-	protoparserutil.StopUnmarshalWorkers()
+	common.StopUnmarshalWorkers()
 	remotewrite.Stop()
 
 	logger.Infof("successfully stopped vmagent in %.3f seconds", time.Since(startTime).Seconds())
@@ -244,7 +232,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
-		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
+		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/vmagent/'>https://docs.victoriametrics.com/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{
 			{"targets", "status for discovered active targets"},
@@ -284,7 +272,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	switch path {
 	case "/prometheus/api/v1/write", "/api/v1/write", "/api/v1/push", "/prometheus/api/v1/push":
-		if protoparserutil.HandleVMProtoServerHandshake(w, r) {
+		if common.HandleVMProtoServerHandshake(w, r) {
 			return true
 		}
 		prometheusWriteRequests.Inc()
@@ -333,11 +321,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/influx/query", "/query":
 		influxQueryRequests.Inc()
-		influxutil.WriteDatabaseNames(w)
+		influxutils.WriteDatabaseNames(w)
 		return true
 	case "/influx/health":
 		influxHealthRequests.Inc()
-		influxutil.WriteHealthCheckResponse(w)
+		influxutils.WriteHealthCheckResponse(w)
 		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
@@ -445,10 +433,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		// https://prometheus.io/docs/prometheus/latest/querying/api/#targets
 		state := r.FormValue("state")
-		scrapePool := r.FormValue("scrapePool")
-		promscrape.WriteAPIV1Targets(w, state, scrapePool)
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/prometheus/target_response", "/target_response":
 		promscrapeTargetResponseRequests.Inc()
@@ -591,11 +577,11 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		return true
 	case "influx/query":
 		influxQueryRequests.Inc()
-		influxutil.WriteDatabaseNames(w)
+		influxutils.WriteDatabaseNames(w)
 		return true
 	case "influx/health":
 		influxHealthRequests.Inc()
-		influxutil.WriteHealthCheckResponse(w)
+		influxutils.WriteHealthCheckResponse(w)
 		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
@@ -754,7 +740,7 @@ func usage() {
 	const s = `
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
 
-See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
+See the docs at https://docs.victoriametrics.com/vmagent/ .
 `
 	flagutil.Usage(s)
 }

app/vmagent/multiarch/Dockerfile

@@ -9,5 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
 ARG TARGETARCH
-ARG BINARY_SUFFIX=non-existing
-COPY vmagent-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmagent-prod
+COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod

app/vmagent/native/request_handler.go

@@ -9,8 +9,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -25,12 +25,12 @@ var (
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(block *stream.Block) error {
+	isGzip := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
 		return insertRows(at, block, extraLabels)
 	})
 }

app/vmagent/newrelic/request_handler.go

@@ -10,9 +10,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 )
 
@@ -24,12 +24,13 @@ var (
 
 // InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
 func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(rows []newrelic.Row) error {
+	ce := req.Header.Get("Content-Encoding")
+	isGzip := ce == "gzip"
+	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmagent/opentelemetry/request_handler.go

@@ -8,9 +8,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -23,11 +23,11 @@ var (
 
 // InsertHandler processes opentelemetry metrics.
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
 		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
@@ -36,7 +36,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }

app/vmagent/opentsdbhttp/request_handler.go

@@ -7,9 +7,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentsdbhttp/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -21,16 +21,16 @@ var (
 // InsertHandler processes HTTP OpenTSDB put requests.
 // See http://opentsdb.net/docs/build/html/api_http/put.html
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	return stream.Parse(req, func(rows []opentsdbhttp.Row) error {
+	return stream.Parse(req, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, rows []opentsdbhttp.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 

app/vmagent/prometheusimport/request_handler.go

@@ -8,9 +8,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -23,23 +23,23 @@ var (
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	defaultTimestamp, err := protoparserutil.GetTimestamp(req)
+	defaultTimestamp, err := parserCommon.GetTimestamp(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, func(rows []prometheus.Row) error {
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)
 	})
 }
 
-func insertRows(at *auth.Token, rows []prometheus.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 

app/vmagent/prometheusimport/request_handler_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 )
 
 var (
@@ -26,7 +26,7 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Fatalf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
@@ -44,14 +44,14 @@ func setUp() {
 		log.Fatalf("unable to set %q with value %q, err: %v", remoteWriteFlag, srv.URL, err)
 	}
 	logger.Init()
-	protoparserutil.StartUnmarshalWorkers()
+	common.StartUnmarshalWorkers()
 	remotewrite.Init()
 	testOutput = &bytes.Buffer{}
 	logger.SetOutputForTests(testOutput)
 }
 
 func tearDown() {
-	protoparserutil.StopUnmarshalWorkers()
+	common.StopUnmarshalWorkers()
 	srv.Close()
 	logger.ResetOutputForTest()
 	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()

app/vmagent/promremotewrite/request_handler.go

@@ -8,8 +8,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -22,7 +22,7 @@ var (
 
 // InsertHandler processes remote write for prometheus.
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}

app/vmagent/remotewrite/client.go

@@ -10,29 +10,26 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 var (
 	forcePromProto = flagutil.NewArrayBool("remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
 	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
-		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
 
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
@@ -90,8 +87,7 @@ type client struct {
 	remoteWriteURL string
 
 	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
-	useVMProto          atomic.Bool
-	canDowngradeVMProto atomic.Bool
+	useVMProto bool
 
 	fq *persistentqueue.FastQueue
 	hc *http.Client
@@ -128,14 +124,14 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	if err != nil {
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
-
-	tr := httputil.NewTransport(false, "vmagent_remotewrite")
-	tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
-	tr.MaxConnsPerHost = 2 * concurrency
-	tr.MaxIdleConnsPerHost = 2 * concurrency
-	tr.IdleConnTimeout = time.Minute
-	tr.WriteBufferSize = 64 * 1024
-
+	tr := &http.Transport{
+		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
+		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
+		MaxConnsPerHost:     2 * concurrency,
+		MaxIdleConnsPerHost: 2 * concurrency,
+		IdleConnTimeout:     time.Minute,
+		WriteBufferSize:     64 * 1024,
+	}
 	pURL := proxyURL.GetOptionalArg(argIdx)
 	if len(pURL) > 0 {
 		if !strings.Contains(pURL, "://") {
@@ -170,11 +166,17 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
 	}
 	if !useVMProto && !usePromProto {
-		// The VM protocol could be downgraded later at runtime if unsupported media type response status is received.
-		useVMProto = true
-		c.canDowngradeVMProto.Store(true)
+		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
+		doRequest := func(url string) (*http.Response, error) {
+			return c.doRequest(url, nil)
+		}
+		useVMProto = common.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
+		if !useVMProto {
+			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
+				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", sanitizedURL)
+		}
 	}
-	c.useVMProto.Store(useVMProto)
+	c.useVMProto = useVMProto
 
 	return c
 }
@@ -382,7 +384,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
-	if encoding.IsZstd(body) {
+	if c.useVMProto {
 		h.Set("Content-Encoding", "zstd")
 		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
 	} else {
@@ -418,7 +420,7 @@ again:
 		if retryDuration > maxRetryDuration {
 			retryDuration = maxRetryDuration
 		}
-		remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
 		t := timerpool.Get(retryDuration)
 		select {
@@ -431,7 +433,6 @@ again:
 		c.retriesCount.Inc()
 		goto again
 	}
-
 	statusCode := resp.StatusCode
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
@@ -440,43 +441,21 @@ again:
 		c.blocksSent.Inc()
 		return true
 	}
-
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
-	if statusCode == 409 {
-		logBlockRejected(block, c.sanitizedURL, resp)
-
-		// Just drop block on 409 status code like Prometheus does.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
-		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
+	if statusCode == 409 || statusCode == 400 {
+		body, err := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		c.packetsDropped.Inc()
-		return true
-		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
-		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
-		// - Real-world implementations of v1 use both 400 and 415 status codes.
-		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
-	} else if statusCode == 415 || statusCode == 400 {
-		if c.canDowngradeVMProto.Swap(false) {
-			logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
-				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-			c.useVMProto.Store(false)
+		if err != nil {
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+				"failed to read response body: %s",
+				len(block), c.sanitizedURL, statusCode, err)
+		} else {
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+				len(block), c.sanitizedURL, statusCode, string(body))
 		}
-
-		if encoding.IsZstd(block) {
-			logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
-				"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-
-			block = mustRepackBlockFromZstdToSnappy(block)
-
-			c.retriesCount.Inc()
-			_ = resp.Body.Close()
-			goto again
-		}
-
-		// Just drop snappy blocks on 400 or 415 status codes like Prometheus does.
+		// Just drop block on 409 and 400 status codes like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
-		logBlockRejected(block, c.sanitizedURL, resp)
 		_ = resp.Body.Close()
 		c.packetsDropped.Inc()
 		return true
@@ -509,7 +488,6 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
-var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 
 // getRetryDuration returns retry duration.
 // retryAfterDuration has the highest priority.
@@ -532,28 +510,6 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 	return retryDuration
 }
 
-func mustRepackBlockFromZstdToSnappy(zstdBlock []byte) []byte {
-	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
-	if err != nil {
-		logger.Panicf("FATAL: cannot re-pack block with size %d bytes from Zstd to Snappy: %s", len(zstdBlock), err)
-	}
-
-	return snappy.Encode(nil, plainBlock)
-}
-
-func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
-			"failed to read response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, err)
-	} else {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, string(body))
-	}
-}
-
 // parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
 // retryAfterString should be in either HTTP-date or a number of seconds.
 // It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.

app/vmagent/remotewrite/client_test.go

@@ -5,9 +5,6 @@ import (
 	"net/http"
 	"testing"
 	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/golang/snappy"
 )
 
 func TestCalculateRetryDuration(t *testing.T) {
@@ -100,19 +97,3 @@ func helper(d time.Duration) time.Duration {
 
 	return d + dv
 }
-
-func TestRepackBlockFromZstdToSnappy(t *testing.T) {
-	expectedPlainBlock := []byte(`foobar`)
-
-	zstdBlock := encoding.CompressZSTDLevel(nil, expectedPlainBlock, 1)
-	snappyBlock := mustRepackBlockFromZstdToSnappy(zstdBlock)
-
-	actualPlainBlock, err := snappy.Decode(nil, snappyBlock)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-
-	if string(actualPlainBlock) != string(expectedPlainBlock) {
-		t.Fatalf("unexpected plain block; got %q; want %q", actualPlainBlock, expectedPlainBlock)
-	}
-}

app/vmagent/remotewrite/pendingseries.go

@@ -16,7 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
@@ -29,7 +28,7 @@ var (
 	maxRowsPerBlock      = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize")
 	vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+
 		"Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+
-		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
+		"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol")
 )
 
 type pendingSeries struct {
@@ -40,7 +39,7 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
 	ps.wr.fq = fq
 	ps.wr.isVMRemoteWrite = isVMRemoteWrite
@@ -100,7 +99,7 @@ type writeRequest struct {
 	fq *persistentqueue.FastQueue
 
 	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	isVMRemoteWrite *atomic.Bool
+	isVMRemoteWrite bool
 
 	// How many significant figures must be left before sending the writeRequest to fq.
 	significantFigures int
@@ -119,7 +118,7 @@ type writeRequest struct {
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are reused.
+	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
 
 	wr.wr.Timeseries = nil
 
@@ -138,7 +137,7 @@ func (wr *writeRequest) reset() {
 // This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
 func (wr *writeRequest) mustFlushOnStop() {
 	wr.wr.Timeseries = wr.tss
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
+	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
 		logger.Panicf("BUG: final flush must always return true")
 	}
 	wr.reset()
@@ -152,7 +151,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
 	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
+	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
 		return false
 	}
 	wr.reset()
@@ -198,43 +197,28 @@ func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
 }
 
 func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
-	labelsSrc := src.Labels
-
-	// Pre-allocate memory for labels.
+	labelsDst := wr.labels
 	labelsLen := len(wr.labels)
-	wr.labels = slicesutil.SetLength(wr.labels, labelsLen+len(labelsSrc))
-	labelsDst := wr.labels[labelsLen:]
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
 
-	// Pre-allocate memory for byte slice needed for storing label names and values.
-	neededBufLen := 0
-	for i := range labelsSrc {
-		label := &labelsSrc[i]
-		neededBufLen += len(label.Name) + len(label.Value)
-	}
-	bufLen := len(wr.buf)
-	wr.buf = slicesutil.SetLength(wr.buf, bufLen+neededBufLen)
-	buf := wr.buf[:bufLen]
-
-	// Copy labels
-	for i := range labelsSrc {
-		dstLabel := &labelsDst[i]
-		srcLabel := &labelsSrc[i]
-
-		bufLen := len(buf)
 		buf = append(buf, srcLabel.Name...)
-		dstLabel.Name = bytesutil.ToUnsafeString(buf[bufLen:])
-
-		bufLen = len(buf)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
 		buf = append(buf, srcLabel.Value...)
-		dstLabel.Value = bytesutil.ToUnsafeString(buf[bufLen:])
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
 	}
-	wr.buf = buf
-	dst.Labels = labelsDst
+	dst.Labels = labelsDst[labelsLen:]
 
-	// Copy samples
-	samplesLen := len(wr.samples)
-	wr.samples = append(wr.samples, src.Samples...)
-	dst.Samples = wr.samples[samplesLen:]
+	samplesDst = append(samplesDst, src.Samples...)
+	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
 }
 
 // marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.

app/vmagent/remotewrite/relabel.go

@@ -7,13 +7,10 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -22,10 +19,10 @@ var (
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
 		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
 		"The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
+		"See https://docs.victoriametrics.com/vmagent/#relabeling")
 	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
 		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
+		"See https://docs.victoriametrics.com/vmagent/#relabeling")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -34,59 +31,12 @@ var (
 
 var labelsGlobal []prompbmarshal.Label
 
-var (
-	relabelConfigReloads      *metrics.Counter
-	relabelConfigReloadErrors *metrics.Counter
-	relabelConfigSuccess      *metrics.Gauge
-	relabelConfigTimestamp    *metrics.Counter
-)
-
-func initRelabelMetrics() {
-	relabelConfigReloads = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
-	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
-	relabelConfigTimestamp = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
-}
-
 // CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
 func CheckRelabelConfigs() error {
 	_, err := loadRelabelConfigs()
 	return err
 }
 
-func initRelabelConfigs() {
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		logger.Fatalf("cannot initialize relabel configs: %s", err)
-	}
-	allRelabelConfigs.Store(rcs)
-	if rcs.isSet() {
-		initRelabelMetrics()
-		relabelConfigSuccess.Set(1)
-		relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
-	}
-}
-
-func reloadRelabelConfigs() {
-	rcs := allRelabelConfigs.Load()
-	if !rcs.isSet() {
-		return
-	}
-	relabelConfigReloads.Inc()
-	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		relabelConfigReloadErrors.Inc()
-		relabelConfigSuccess.Set(0)
-		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
-		return
-	}
-	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
-	logger.Infof("successfully reloaded relabel configs")
-}
-
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
@@ -120,21 +70,6 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
-func (rcs *relabelConfigs) isSet() bool {
-	if rcs == nil {
-		return false
-	}
-	if rcs.global.Len() > 0 {
-		return true
-	}
-	for _, pc := range rcs.perURL {
-		if pc.Len() > 0 {
-			return true
-		}
-	}
-	return false
-}
-
 // initLabelsGlobal must be called after parsing command-line flags.
 func initLabelsGlobal() {
 	labelsGlobal = nil

app/vmagent/remotewrite/relabel_test.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestApplyRelabeling(t *testing.T) {
@@ -63,7 +63,7 @@ func TestAppendExtraLabels(t *testing.T) {
 func parseSeries(data string) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	tss = append(tss, prompbmarshal.TimeSeries{
-		Labels: promutil.MustNewLabelsFromString(data).GetLabels(),
+		Labels: promutils.MustNewLabelsFromString(data).GetLabels(),
 	})
 	return tss
 }

app/vmagent/remotewrite/remotewrite.go

@@ -15,7 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -25,7 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
@@ -39,21 +39,22 @@ var (
 		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
 		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set")
 	enableMultitenantHandlers = flag.Bool("enableMultitenantHandlers", false, "Whether to process incoming data via multitenant insert handlers according to "+
-		"https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers "+
-		"according to https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data ."+
-		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy for details")
+		"https://docs.victoriametrics.com/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers "+
+		"according to https://docs.victoriametrics.com/#how-to-import-time-series-data ."+
+		"See https://docs.victoriametrics.com/vmagent/#multitenancy for details")
 
-	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url. "+
-		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages . "+
+	shardByURL = flag.Bool("remoteWrite.shardByURL", false, "Whether to shard outgoing series across all the remote storage systems enumerated via -remoteWrite.url . "+
+		"By default the data is replicated across all the -remoteWrite.url . See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages . "+
 		"See also -remoteWrite.shardByURLReplicas")
 	shardByURLReplicas = flag.Int("remoteWrite.shardByURLReplicas", 1, "How many copies of data to make among remote storage systems enumerated via -remoteWrite.url "+
-		"when -remoteWrite.shardByURL is set. See https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
+		"when -remoteWrite.shardByURL is set. See https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
 	shardByURLLabels = flagutil.NewArrayString("remoteWrite.shardByURL.labels", "Optional list of labels, which must be used for sharding outgoing samples "+
 		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
 		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.ignoreLabels")
 	shardByURLIgnoreLabels = flagutil.NewArrayString("remoteWrite.shardByURL.ignoreLabels", "Optional list of labels, which must be ignored when sharding outgoing samples "+
 		"among remote storage systems if -remoteWrite.shardByURL command-line flag is set. By default all the labels are used for sharding in order to gain "+
 		"even distribution of series over the specified -remoteWrite.url systems. See also -remoteWrite.shardByURL.labels")
+
 	tmpDataPath = flag.String("remoteWrite.tmpDataPath", "vmagent-remotewrite-data", "Path to directory for storing pending data, which isn't sent to the configured -remoteWrite.url . "+
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
@@ -80,30 +81,28 @@ var (
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
 		`Enabled sorting for labels can slow down ingestion performance a bit`)
 	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
-		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
 	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
-		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/vmagent/#cardinality-limiter")
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
 		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
-		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
+		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
 	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
+		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
 )
 
 var (
 	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
-	rwctxsGlobal              []*remoteWriteCtx
-	rwctxsGlobalIdx           []int
-	rwctxConsistentHashGlobal *consistenthash.ConsistentHash
+	rwctxsGlobal []*remoteWriteCtx
 
 	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
 	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
 		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
 			"or remove -remoteWrite.disableOnDiskQueue from vmagent command-line flags, so it could save pending data to -remoteWrite.tmpDataPath; " +
-			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence"),
+			"see https://docs.victoriametrics.com/vmagent/#disabling-on-disk-persistence"),
 		StatusCode: http.StatusTooManyRequests,
 	}
 
@@ -184,7 +183,7 @@ func Init() {
 
 	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
-			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
+			"see https://docs.victoriametrics.com/vmagent/#sharding-among-remote-storages")
 	}
 	shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
 	shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
@@ -196,11 +195,17 @@ func Init() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
 	sighupCh := procutil.NewSighupChan()
 
-	initRelabelConfigs()
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		logger.Fatalf("cannot load relabel configs: %s", err)
+	}
+	allRelabelConfigs.Store(rcs)
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
 	initStreamAggrConfigGlobal()
 
-	initRemoteWriteCtxs(*remoteWriteURLs)
+	rwctxsGlobal = newRemoteWriteCtxs(*remoteWriteURLs)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -262,7 +267,30 @@ func dropDanglingQueues() {
 	}
 }
 
-func initRemoteWriteCtxs(urls []string) {
+func reloadRelabelConfigs() {
+	relabelConfigReloads.Inc()
+	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		relabelConfigReloadErrors.Inc()
+		relabelConfigSuccess.Set(0)
+		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+		return
+	}
+	allRelabelConfigs.Store(rcs)
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	logger.Infof("successfully reloaded relabel configs")
+}
+
+var (
+	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
+	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
+	relabelConfigSuccess      = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
+	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
+)
+
+func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
@@ -278,7 +306,6 @@ func initRemoteWriteCtxs(urls []string) {
 		maxInmemoryBlocks = 2
 	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
-	rwctxIdx := make([]int, len(urls))
 	for i, remoteWriteURLRaw := range urls {
 		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
 		if err != nil {
@@ -289,19 +316,8 @@ func initRemoteWriteCtxs(urls []string) {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
 		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
-		rwctxIdx[i] = i
 	}
-
-	if *shardByURL {
-		consistentHashNodes := make([]string, 0, len(urls))
-		for i, url := range urls {
-			consistentHashNodes = append(consistentHashNodes, fmt.Sprintf("%d:%s", i+1, url))
-		}
-		rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(consistentHashNodes, 0)
-	}
-
-	rwctxsGlobal = rwctxs
-	rwctxsGlobalIdx = rwctxIdx
+	return rwctxs
 }
 
 var (
@@ -485,10 +501,6 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 	return true
 }
 
-// getEligibleRemoteWriteCtxs checks whether writes to configured remote storage systems are blocked and
-// returns only the unblocked rwctx.
-//
-// calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
 func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
 	if !disableOnDiskQueueAny {
 		return rwctxsGlobal, true
@@ -505,12 +517,6 @@ func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamples
 				return nil, false
 			}
 			rowsCount := getRowsCount(tss)
-			if *shardByURL {
-				// Todo: When shardByURL is enabled, the following metrics won't be 100% accurate. Because vmagent don't know
-				// which rwctx should data be pushed to yet. Let's consider the hashing algorithm fair and will distribute
-				// data to all rwctxs evenly.
-				rowsCount = rowsCount / len(rwctxsGlobal)
-			}
 			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 		}
 	}
@@ -522,7 +528,6 @@ func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
 	if len(rwctxs) == 0 {
 		return
 	}
-
 	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
 		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
 	}
@@ -573,68 +578,7 @@ func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []pr
 	defer putTSSShards(x)
 
 	shards := x.shards
-	shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
-
-	// Push sharded samples to remote storage systems in parallel in order to reduce
-	// the time needed for sending the data to multiple remote storage systems.
-	var wg sync.WaitGroup
-	var anyPushFailed atomic.Bool
-	for i, rwctx := range rwctxs {
-		shard := shards[i]
-		if len(shard) == 0 {
-			continue
-		}
-		wg.Add(1)
-		go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
-			defer wg.Done()
-			if !rwctx.TryPush(tss, forceDropSamplesOnFailure) {
-				anyPushFailed.Store(true)
-			}
-		}(rwctx, shard)
-	}
-	wg.Wait()
-	return !anyPushFailed.Load()
-}
-
-// calculateHealthyRwctxIdx returns the index of healthyRwctxs in rwctxsGlobal.
-// It relies on the order of rwctx in healthyRwctxs, which is appended by getEligibleRemoteWriteCtxs.
-func calculateHealthyRwctxIdx(healthyRwctxs []*remoteWriteCtx) ([]int, []int) {
-	// fast path: all rwctxs are healthy.
-	if len(healthyRwctxs) == len(rwctxsGlobal) {
-		return rwctxsGlobalIdx, nil
-	}
-
-	unhealthyIdx := make([]int, 0, len(rwctxsGlobal))
-	healthyIdx := make([]int, 0, len(rwctxsGlobal))
-
-	var i int
-	for j := range rwctxsGlobal {
-		if i < len(healthyRwctxs) && rwctxsGlobal[j].idx == healthyRwctxs[i].idx {
-			healthyIdx = append(healthyIdx, j)
-			i++
-		} else {
-			unhealthyIdx = append(unhealthyIdx, j)
-		}
-	}
-
-	return healthyIdx, unhealthyIdx
-}
-
-// shardAmountRemoteWriteCtx distribute time series to shards by consistent hashing.
-func shardAmountRemoteWriteCtx(tssBlock []prompbmarshal.TimeSeries, shards [][]prompbmarshal.TimeSeries, rwctxs []*remoteWriteCtx, replicas int) {
-	tmpLabels := promutil.GetLabels()
-	defer promutil.PutLabels(tmpLabels)
-
-	healthyIdx, unhealthyIdx := calculateHealthyRwctxIdx(rwctxs)
-
-	// shardsIdxMap is a map to find which the shard idx by rwctxs idx.
-	// rwctxConsistentHashGlobal will tell which the rwctxs idx a time series should be written to.
-	// And this time series should be appended to the shards by correct shard idx.
-	shardsIdxMap := make(map[int]int, len(healthyIdx))
-	for idx, rwctxsIdx := range healthyIdx {
-		shardsIdxMap[rwctxsIdx] = idx
-	}
-
+	tmpLabels := promutils.GetLabels()
 	for _, ts := range tssBlock {
 		hashLabels := ts.Labels
 		if len(shardByURLLabelsMap) > 0 {
@@ -655,25 +599,41 @@ func shardAmountRemoteWriteCtx(tssBlock []prompbmarshal.TimeSeries, shards [][]p
 			tmpLabels.Labels = hashLabels
 		}
 		h := getLabelsHash(hashLabels)
-
-		// Get the rwctxIdx through consistent hashing and then map it to the index in shards.
-		// The rwctxIdx is not always equal to the shardIdx, for example, when some rwctx are not available.
-		rwctxIdx := rwctxConsistentHashGlobal.GetNodeIdx(h, unhealthyIdx)
-		shardIdx := shardsIdxMap[rwctxIdx]
-
-		replicated := 0
+		idx := h % uint64(len(shards))
+		i := 0
 		for {
-			shards[shardIdx] = append(shards[shardIdx], ts)
-			replicated++
-			if replicated >= replicas {
+			shards[idx] = append(shards[idx], ts)
+			i++
+			if i >= replicas {
 				break
 			}
-			shardIdx++
-			if shardIdx >= len(shards) {
-				shardIdx = 0
+			idx++
+			if idx >= uint64(len(shards)) {
+				idx = 0
 			}
 		}
 	}
+	promutils.PutLabels(tmpLabels)
+
+	// Push sharded samples to remote storage systems in parallel in order to reduce
+	// the time needed for sending the data to multiple remote storage systems.
+	var wg sync.WaitGroup
+	var anyPushFailed atomic.Bool
+	for i, rwctx := range rwctxs {
+		shard := shards[i]
+		if len(shard) == 0 {
+			continue
+		}
+		wg.Add(1)
+		go func(rwctx *remoteWriteCtx, tss []prompbmarshal.TimeSeries) {
+			defer wg.Done()
+			if !rwctx.TryPush(tss, forceDropSamplesOnFailure) {
+				anyPushFailed.Store(true)
+			}
+		}(rwctx, shard)
+	}
+	wg.Wait()
+	return !anyPushFailed.Load()
 }
 
 type tssShards struct {
@@ -847,7 +807,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
+		pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{

app/vmagent/remotewrite/remotewrite_test.go

@@ -4,17 +4,12 @@ import (
 	"fmt"
 	"math"
 	"reflect"
-	"strconv"
-	"sync/atomic"
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
-
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -58,7 +53,7 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 }
 
 func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
+	f := func(streamAggrConfig, relabelConfig string, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
 		t.Helper()
 		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
 		if err != nil {
@@ -72,9 +67,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		allRelabelConfigs.Store(rcs)
 
 		pss := make([]*pendingSeries, 1)
-		isVMProto := &atomic.Bool{}
-		isVMProto.Store(true)
-		pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
+		pss[0] = newPendingSeries(nil, true, 0, 100)
 		rwctx := &remoteWriteCtx{
 			idx:                    0,
 			streamAggrKeepInput:    keepInput,
@@ -84,15 +77,12 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, enableWindows, dedupInterval, nil, "dedup-global")
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
 		}
 
 		if streamAggrConfig != "" {
 			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-			opts := streamaggr.Options{
-				EnableWindows: enableWindows,
-			}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, &opts, "global")
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
@@ -101,7 +91,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		}
 
 		offsetMsecs := time.Now().UnixMilli()
-		inputTss := prometheus.MustParsePromMetrics(input, offsetMsecs)
+		inputTss := prompbmarshal.MustParsePromMetrics(input, offsetMsecs)
 		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
 
 		// copy inputTss to make sure it is not mutated during TryPush call
@@ -124,13 +114,13 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, false, 0, false, false, `
+`, 0, false, false, `
 metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
 metric{env="bar"} 25
 `)
-	f(``, ``, true, time.Hour, false, false, `
+	f(``, ``, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="foo"} 20
 metric{env="dev"} 15
@@ -140,7 +130,7 @@ metric{env="foo"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, true, time.Hour, false, false, `
+`, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
@@ -150,7 +140,7 @@ metric{env="bar"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, true, time.Hour, true, false, `
+`, time.Hour, true, false, `
 metric{env="test"} 10
 metric{env="dev"} 20
 metric{env="foo"} 15
@@ -160,7 +150,7 @@ metric{env="dev"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, true, time.Hour, false, true, `
+`, time.Hour, false, true, `
 metric{env="foo"} 10
 metric{env="dev"} 20
 metric{env="foo"} 15
@@ -170,180 +160,10 @@ metric{env="dev"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, true, time.Hour, true, true, `
+`, time.Hour, true, true, `
 metric{env="dev"} 10
 metric{env="test"} 20
 metric{env="dev"} 15
 metric{env="bar"} 25
 `)
 }
-
-func TestShardAmountRemoteWriteCtx(t *testing.T) {
-	// 1. distribute 100000 series to n nodes.
-	// 2. remove the last node from healthy list.
-	// 3. distribute the same 10000 series to (n-1) node again.
-	// 4. check active time series change rate:
-	// change rate must < (3/total nodes). e.g. +30% if 10 you have 10 nodes.
-
-	f := func(remoteWriteCount int, healthyIdx []int, replicas int) {
-		t.Helper()
-		defer func() {
-			rwctxsGlobal = nil
-			rwctxsGlobalIdx = nil
-			rwctxConsistentHashGlobal = nil
-		}()
-
-		rwctxsGlobal = make([]*remoteWriteCtx, remoteWriteCount)
-		rwctxsGlobalIdx = make([]int, remoteWriteCount)
-		rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
-
-		for i := range remoteWriteCount {
-			rwCtx := &remoteWriteCtx{
-				idx: i,
-			}
-			rwctxsGlobalIdx[i] = i
-
-			if i >= len(healthyIdx) {
-				rwctxsGlobal[i] = rwCtx
-				continue
-			}
-			hIdx := healthyIdx[i]
-			if hIdx != i {
-				rwctxs = append(rwctxs, &remoteWriteCtx{
-					idx: hIdx,
-				})
-			} else {
-				rwctxs = append(rwctxs, rwCtx)
-			}
-			rwctxsGlobal[i] = rwCtx
-		}
-
-		seriesCount := 100000
-		// build 1000000 series
-		tssBlock := make([]prompbmarshal.TimeSeries, 0, seriesCount)
-		for i := 0; i < seriesCount; i++ {
-			tssBlock = append(tssBlock, prompbmarshal.TimeSeries{
-				Labels: []prompbmarshal.Label{
-					{
-						Name:  "label",
-						Value: strconv.Itoa(i),
-					},
-				},
-				Samples: []prompbmarshal.Sample{
-					{
-						Timestamp: 0,
-						Value:     0,
-					},
-				},
-			})
-		}
-
-		// build consistent hash for x remote write context
-		// build active time series set
-		nodes := make([]string, 0, remoteWriteCount)
-		activeTimeSeriesByNodes := make([]map[string]struct{}, remoteWriteCount)
-		for i := 0; i < remoteWriteCount; i++ {
-			nodes = append(nodes, fmt.Sprintf("node%d", i))
-			activeTimeSeriesByNodes[i] = make(map[string]struct{})
-		}
-		rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(nodes, 0)
-
-		// create shards
-		x := getTSSShards(len(rwctxs))
-		shards := x.shards
-
-		// execute
-		shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
-
-		for i, nodeIdx := range healthyIdx {
-			for _, ts := range shards[i] {
-				// add it to node[nodeIdx]'s active time series
-				activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
-			}
-		}
-
-		totalActiveTimeSeries := 0
-		for _, activeTimeSeries := range activeTimeSeriesByNodes {
-			totalActiveTimeSeries += len(activeTimeSeries)
-		}
-		avgActiveTimeSeries1 := totalActiveTimeSeries / remoteWriteCount
-		putTSSShards(x)
-
-		// removed last node
-		rwctxs = rwctxs[:len(rwctxs)-1]
-		healthyIdx = healthyIdx[:len(healthyIdx)-1]
-
-		x = getTSSShards(len(rwctxs))
-		shards = x.shards
-
-		// execute
-		shardAmountRemoteWriteCtx(tssBlock, shards, rwctxs, replicas)
-		for i, nodeIdx := range healthyIdx {
-			for _, ts := range shards[i] {
-				// add it to node[nodeIdx]'s active time series
-				activeTimeSeriesByNodes[nodeIdx][prompbmarshal.LabelsToString(ts.Labels)] = struct{}{}
-			}
-		}
-
-		totalActiveTimeSeries = 0
-		for _, activeTimeSeries := range activeTimeSeriesByNodes {
-			totalActiveTimeSeries += len(activeTimeSeries)
-		}
-		avgActiveTimeSeries2 := totalActiveTimeSeries / remoteWriteCount
-
-		changed := math.Abs(float64(avgActiveTimeSeries2-avgActiveTimeSeries1) / float64(avgActiveTimeSeries1))
-		threshold := 3 / float64(remoteWriteCount)
-
-		if changed >= threshold {
-			t.Fatalf("average active time series before: %d, after: %d, changed: %.2f. threshold: %.2f", avgActiveTimeSeries1, avgActiveTimeSeries2, changed, threshold)
-		}
-
-	}
-
-	f(5, []int{0, 1, 2, 3, 4}, 1)
-
-	f(5, []int{0, 1, 2, 3, 4}, 2)
-
-	f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 1)
-
-	f(10, []int{0, 1, 2, 3, 4, 5, 6, 7, 9}, 3)
-}
-
-func TestCalculateHealthyRwctxIdx(t *testing.T) {
-	f := func(total int, healthyIdx []int, unhealthyIdx []int) {
-		t.Helper()
-
-		healthyMap := make(map[int]bool)
-		for _, idx := range healthyIdx {
-			healthyMap[idx] = true
-		}
-		rwctxsGlobal = make([]*remoteWriteCtx, total)
-		rwctxsGlobalIdx = make([]int, total)
-		rwctxs := make([]*remoteWriteCtx, 0, len(healthyIdx))
-		for i := range rwctxsGlobal {
-			rwctx := &remoteWriteCtx{idx: i}
-			rwctxsGlobal[i] = rwctx
-			if healthyMap[i] {
-				rwctxs = append(rwctxs, rwctx)
-			}
-			rwctxsGlobalIdx[i] = i
-		}
-
-		gotHealthyIdx, gotUnhealthyIdx := calculateHealthyRwctxIdx(rwctxs)
-		if !reflect.DeepEqual(healthyIdx, gotHealthyIdx) {
-			t.Errorf("calculateHealthyRwctxIdx want healthyIdx = %v, got %v", healthyIdx, gotHealthyIdx)
-		}
-		if !reflect.DeepEqual(unhealthyIdx, gotUnhealthyIdx) {
-			t.Errorf("calculateHealthyRwctxIdx want unhealthyIdx = %v, got %v", unhealthyIdx, gotUnhealthyIdx)
-		}
-	}
-
-	f(5, []int{0, 1, 2, 3, 4}, nil)
-	f(5, []int{0, 1, 2, 4}, []int{3})
-	f(5, []int{2, 4}, []int{0, 1, 3})
-	f(5, []int{0, 2, 4}, []int{1, 3})
-	f(5, []int{}, []int{0, 1, 2, 3, 4})
-	f(5, []int{4}, []int{0, 1, 2, 3})
-	f(1, []int{0}, nil)
-	f(1, []int{}, []int{0})
-}

app/vmagent/remotewrite/streamaggr.go

@@ -16,55 +16,49 @@ import (
 var (
 	// Global config
 	streamAggrGlobalConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
+		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
 		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
 	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation "+
 		"with -streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
 		"aggregator before optional aggregation with -streamAggr.config . "+
-		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
+		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
 	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
 		"current aggregation interval for aggregator. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples")
+		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
 	streamAggrGlobalIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start for "+
 		"aggregator. Increase this value if you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving unordered delayed data from "+
-		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignore-aggregation-intervals-on-start")
+		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrGlobalDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples for aggregator "+
-		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels")
-	streamAggrGlobalEnableWindows = flag.Bool("streamAggr.enableWindows", false, "Enables aggregation within fixed windows for all global aggregators. "+
-		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
+		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 
 	// Per URL config
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
+		"See https://docs.victoriametrics.com/stream-aggregation/ . "+
 		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/stream-aggregation/")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples")
+		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
 	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
-		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving buffered delayed data from clients pushing data into the vmagent. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignore-aggregation-intervals-on-start")
+		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
 		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
 		"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels")
-	streamAggrEnableWindows = flagutil.NewArrayBool("remoteWrite.streamAggr.enableWindows", "Enables aggregation within fixed windows for all remote write's aggregators. "+
-		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
-		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
+		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
@@ -131,7 +125,7 @@ func reloadStreamAggrConfigGlobal() {
 func initStreamAggrConfigGlobal() {
 	sas, err := newStreamAggrConfigGlobal()
 	if err != nil {
-		logger.Fatalf("cannot initialize global stream aggregators: %s", err)
+		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
 	}
 	if sas != nil {
 		filePath := sas.FilePath()
@@ -141,7 +135,7 @@ func initStreamAggrConfigGlobal() {
 	}
 	dedupInterval := *streamAggrGlobalDedupInterval
 	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 	}
 }
 
@@ -167,7 +161,7 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
 		if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
 			dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
 		}
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, dropLabels, alias)
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, dropLabels, alias)
 	}
 }
 
@@ -213,7 +207,6 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
 		KeepInput:            *streamAggrGlobalKeepInput,
-		EnableWindows:        *streamAggrGlobalEnableWindows,
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
@@ -247,7 +240,6 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
 		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
 		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
-		EnableWindows:        streamAggrEnableWindows.GetOptionalArg(idx),
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

app/vmagent/vmimport/request_handler.go

@@ -9,8 +9,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
+	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
@@ -26,17 +26,17 @@ var (
 //
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6
 func InsertHandler(at *auth.Token, req *http.Request) error {
-	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	extraLabels, err := parserCommon.GetExtraLabels(req)
 	if err != nil {
 		return err
 	}
-	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, encoding, func(rows []vmimport.Row) error {
+	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }
 
-func insertRows(at *auth.Token, rows []vmimport.Row, extraLabels []prompbmarshal.Label) error {
+func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)
 

app/vmalert-tool/README.md

@@ -1,3 +1,3 @@
-See vmalert-tool docs [here](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/).
+See vmalert-tool docs [here](https://docs.victoriametrics.com/vmalert-tool.html).
 
-vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/vmalert-tool.md).
+vmalert-tool docs can be edited at [docs/vmalert-tool.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/vmalert-tool.md).

app/vmalert-tool/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-EXPOSE 8880
-
-ENTRYPOINT ["/vmalert-tool-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vmalert-tool-prod

app/vmalert-tool/multiarch/Dockerfile

@@ -9,5 +9,4 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica
 EXPOSE 8429
 ENTRYPOINT ["/vmalert-tool-prod"]
 ARG TARGETARCH
-ARG BINARY_SUFFIX=non-existing
-COPY vmalert-tool-linux-${TARGETARCH}-prod${BINARY_SUFFIX} ./vmalert-tool-prod
+COPY vmalert-tool-linux-${TARGETARCH}-prod ./vmalert-tool-prod