app/vminsert/netstorage: refactor snb rebuild

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -60,8 +60,8 @@ body:
   
         For VictoriaMetrics health-state issues please provide full-length screenshots
         of Grafana dashboards if possible:
-          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229)
-          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176)
+          * [Grafana dashboard for single-node VictoriaMetrics](https://grafana.com/grafana/dashboards/10229/)
+          * [Grafana dashboard for VictoriaMetrics cluster](https://grafana.com/grafana/dashboards/11176/)
         
         See how to setup monitoring here:
           * [monitoring for single-node VictoriaMetrics](https://docs.victoriametrics.com/#monitoring)

.github/workflows/main.yml

@@ -85,38 +85,9 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
 
       - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
+        run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v4
         with:
           file: ./coverage.txt
-
-  integration-test:
-    name: integration-test
-    needs: [lint, test]
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        id: go
-        uses: actions/setup-go@v5
-        with:
-          cache: false
-          go-version: stable
-
-      - name: Cache Go artifacts
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/bin
-            ~/go/pkg/mod
-          key: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-${{ matrix.scenario }}-
-
-      - name: Run integration tests
-        run: make integration-test

.gitignore

@@ -7,7 +7,6 @@
 .vscode
 *.test
 *.swp
-/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
@@ -26,4 +25,3 @@ _site
 /docs/.jekyll-metadata
 coverage.txt
 cspell.json
-*~

.wwhrd.yml

@@ -4,4 +4,3 @@ allowlist:
   - BSD-3-Clause
   - BSD-2-Clause
   - ISC
-  - MPL-2.0

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2024 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -13,7 +13,6 @@ PKG_TAG := $(BUILDINFO_TAG)
 endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
-TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -77,16 +76,6 @@ vmcluster-windows-amd64: \
 	vmselect-windows-amd64 \
 	vmstorage-windows-amd64
 
-vmcluster-darwin-amd64: \
-	vminsert-darwin-amd64 \
-	vmselect-darwin-amd64 \
-	vmstorage-darwin-amd64
-
-vmcluster-darwin-arm64: \
-	vminsert-darwin-arm64 \
-	vmselect-darwin-arm64 \
-	vmstorage-darwin-arm64
-
 crossbuild: vmcluster-crossbuild
 
 vmcluster-crossbuild:
@@ -123,9 +112,7 @@ release-vmcluster: \
 	release-vmcluster-linux-arm64 \
 	release-vmcluster-freebsd-amd64 \
 	release-vmcluster-openbsd-amd64 \
-	release-vmcluster-windows-amd64 \
-	release-vmcluster-darwin-amd64 \
-	release-vmcluster-darwin-arm64
+	release-vmcluster-windows-amd64
 
 release-vmcluster-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
@@ -142,18 +129,12 @@ release-vmcluster-openbsd-amd64:
 release-vmcluster-windows-amd64:
 	GOARCH=amd64 $(MAKE) release-vmcluster-windows-goarch
 
-release-vmcluster-darwin-amd64:
-	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmcluster-goos-goarch
-
-release-vmcluster-darwin-arm64:
-	 GOOS=darwin GOARCH=arm64 $(MAKE) release-vmcluster-goos-goarch
-
 release-vmcluster-goos-goarch: \
 	vminsert-$(GOOS)-$(GOARCH)-prod \
 	vmselect-$(GOOS)-$(GOARCH)-prod \
 	vmstorage-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vminsert-$(GOOS)-$(GOARCH)-prod \
 			vmselect-$(GOOS)-$(GOARCH)-prod \
 			vmstorage-$(GOOS)-$(GOARCH)-prod \
@@ -192,12 +173,10 @@ pprof-cpu:
 fmt:
 	gofmt -l -w -s ./lib
 	gofmt -l -w -s ./app
-	gofmt -l -w -s ./apptest
 
 vet:
 	go vet ./lib/...
 	go vet ./app/...
-	go vet ./apptest/...
 
 check-all: fmt vet golangci-lint govulncheck
 
@@ -218,9 +197,6 @@ test-full:
 test-full-386:
 	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
-integration-test: all
-	go test ./apptest/... -skip="^TestSingle.*"
-
 benchmark:
 	go test -bench=. ./lib/...
 	go test -bench=. ./app/...
@@ -230,9 +206,9 @@ benchmark-pure:
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
-	go get -u ./lib/...
-	go get -u ./app/...
-	go mod tidy -compat=1.23
+	go get -u -d ./lib/...
+	go get -u -d ./app/...
+	go mod tidy -compat=1.22
 	go mod vendor
 
 app-local:
@@ -258,7 +234,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.63.4
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`
@@ -277,3 +253,34 @@ install-wwhrd:
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
+
+copy-docs:
+# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
+# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
+	echo "---" > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "sort: ${ORDER}" >> ${DST}; \
+		echo "weight: ${ORDER}" >> ${DST}; \
+		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
+	fi
+
+	echo "title: ${TITLE}" >> ${DST}
+	@if [ ${OLD_URL} ]; then \
+		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
+	fi
+	echo "---" >> ${DST}
+	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"\//' ${DST}
+	sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"\//' ${DST}
+	sed -i='.tmp' 's/](docs\//](/' ${DST}
+	rm -rf docs/*.tmp
+
+# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
+# For ORDER=0 it adds no order tag/weight tag.
+# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
+# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
+# Images starting with <img src="docs/ are replaced with <img src="
+# Cluster docs are supposed to be ordered as 2nd.
+# The rest of docs is ordered manually.
+docs-sync:
+	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md OLD_URL='/Cluster-VictoriaMetrics.html' ORDER=2 TITLE='Cluster version' $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to <security@victoriametrics.com>
+Please report any security issues to security@victoriametrics.com

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 9428
 
 ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./victoria-logs-prod

app/victoria-logs/main.go

@@ -92,9 +92,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if vlselect.RequestHandler(w, r) {
 		return true
 	}
-	if vlstorage.RequestHandler(w, r) {
-		return true
-	}
 	return false
 }
 

app/victoria-logs/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/vlinsert/datadog/datadog.go

@@ -1,274 +0,0 @@
-package datadog
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-)
-
-var (
-	datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Datadog tags to be used as stream fields.")
-	datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Datadog tags to ignore.")
-)
-
-var parserPool fastjson.ParserPool
-
-// RequestHandler processes Datadog insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/api/v1/validate":
-		fmt.Fprintf(w, `{}`)
-		return true
-	case "/api/v2/logs":
-		return datadogLogsIngestion(w, r)
-	default:
-		return false
-	}
-}
-
-func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
-	w.Header().Add("Content-Type", "application/json")
-	startTime := time.Now()
-	v2LogsRequestsTotal.Inc()
-	reader := r.Body
-
-	var ts int64
-	if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
-		var err error
-		ts, err = strconv.ParseInt(tsValue, 10, 64)
-		if err != nil {
-			httpserver.Errorf(w, r, "could not parse dd-message-timestamp header value: %s", err)
-			return true
-		}
-		ts *= 1e6
-	} else {
-		ts = startTime.UnixNano()
-	}
-
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot read gzipped logs request: %s", err)
-			return true
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
-	}
-
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = *datadogStreamFields
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *datadogIgnoreFields
-	}
-
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return true
-	}
-
-	lmp := cp.NewLogMessageProcessor("datadog")
-	err = readLogsRequest(ts, data, lmp)
-	lmp.MustClose()
-	if err != nil {
-		logger.Warnf("cannot decode log message in /api/v2/logs request: %s, stream fields: %s", err, cp.StreamFields)
-		return true
-	}
-
-	// update v2LogsRequestDuration only for successfully parsed requests
-	// There is no need in updating v2LogsRequestDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	v2LogsRequestDuration.UpdateDuration(startTime)
-	fmt.Fprintf(w, `{}`)
-	return true
-}
-
-var (
-	v2LogsRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
-	v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
-)
-
-// datadog message field has two formats:
-//   - regular log message with string text
-//   - nested json format for serverless plugins
-//     which has folowing format:
-//     {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
-//
-// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
-func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage.Field, error) {
-	switch v.Type() {
-	case fastjson.TypeString:
-		val := v.GetStringBytes()
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: bytesutil.ToUnsafeString(val),
-		})
-	case fastjson.TypeObject:
-		var firstErr error
-		v.GetObject().Visit(func(k []byte, v *fastjson.Value) {
-			if firstErr != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "_msg",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "status":
-				val := v.GetStringBytes()
-				fields = append(fields, logstorage.Field{
-					Name:  "status",
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			case "lamdba":
-				obj, err := v.Object()
-				if err != nil {
-					firstErr = err
-					firstErr = fmt.Errorf("unexpected lambda value type for %q:%q; want object", k, v)
-					return
-				}
-				obj.Visit(func(k []byte, v *fastjson.Value) {
-					if firstErr != nil {
-						return
-					}
-					val, err := v.StringBytes()
-					if err != nil {
-						firstErr = fmt.Errorf("unexpected lambda label value type for %q:%q; want string", k, v)
-						return
-					}
-					fields = append(fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(val),
-					})
-				})
-
-			}
-		})
-	default:
-		return fields, fmt.Errorf("unsupported message type %q", v.Type().String())
-	}
-	return fields, nil
-}
-
-// readLogsRequest parses data according to DataDog logs format
-// https://docs.datadoghq.com/api/latest/logs/#send-logs
-func readLogsRequest(ts int64, data []byte, lmp insertutils.LogMessageProcessor) error {
-	p := parserPool.Get()
-	defer parserPool.Put(p)
-	v, err := p.ParseBytes(data)
-	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
-	}
-	records, err := v.Array()
-	if err != nil {
-		return fmt.Errorf("cannot extract array from parsed JSON: %w", err)
-	}
-
-	var fields []logstorage.Field
-	for _, r := range records {
-		o, err := r.Object()
-		if err != nil {
-			return fmt.Errorf("could not extract log record: %w", err)
-		}
-		o.Visit(func(k []byte, v *fastjson.Value) {
-			if err != nil {
-				return
-			}
-			switch bytesutil.ToUnsafeString(k) {
-			case "message":
-				fields, err = appendMsgFields(fields, v)
-				if err != nil {
-					return
-				}
-			case "timestamp":
-				val, e := v.Int64()
-				if e != nil {
-					err = fmt.Errorf("failed to parse timestamp for %q:%q", k, v)
-				}
-				if val > 0 {
-					ts = val * 1e6
-				}
-			case "ddtags":
-				// https://docs.datadoghq.com/getting_started/tagging/
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				var pair []byte
-				idx := 0
-				for idx >= 0 {
-					idx = bytes.IndexByte(val, ',')
-					if idx < 0 {
-						pair = val
-					} else {
-						pair = val[:idx]
-						val = val[idx+1:]
-					}
-					if len(pair) > 0 {
-						n := bytes.IndexByte(pair, ':')
-						if n < 0 {
-							// No tag value.
-							fields = append(fields, logstorage.Field{
-								Name:  bytesutil.ToUnsafeString(pair),
-								Value: "no_label_value",
-							})
-						}
-						fields = append(fields, logstorage.Field{
-							Name:  bytesutil.ToUnsafeString(pair[:n]),
-							Value: bytesutil.ToUnsafeString(pair[n+1:]),
-						})
-					}
-				}
-			default:
-				val, e := v.StringBytes()
-				if e != nil {
-					err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-					return
-				}
-				fields = append(fields, logstorage.Field{
-					Name:  bytesutil.ToUnsafeString(k),
-					Value: bytesutil.ToUnsafeString(val),
-				})
-			}
-		})
-		if err != nil {
-			return err
-		}
-		lmp.AddRow(ts, fields, nil)
-		fields = fields[:0]
-	}
-	return nil
-}

app/vlinsert/datadog/datadog_test.go

@@ -1,104 +0,0 @@
-package datadog
-
-import (
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-)
-
-func TestReadLogsRequestFailure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-
-		lmp := &insertutils.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err == nil {
-			t.Fatalf("expecting non-empty error")
-		}
-		if err := lmp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-	f("foobar")
-	f(`{}`)
-	f(`["create":{}]`)
-	f(`{"create":{}}
-foobar`)
-}
-
-func TestReadLogsRequestSuccess(t *testing.T) {
-	f := func(data string, rowsExpected int, resultExpected string) {
-		t.Helper()
-
-		ts := time.Now().UnixNano()
-		var timestampsExpected []int64
-		for i := 0; i < rowsExpected; i++ {
-			timestampsExpected = append(timestampsExpected, ts)
-		}
-		lmp := &insertutils.TestLogMessageProcessor{}
-		if err := readLogsRequest(ts, []byte(data), lmp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if err := lmp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	// Verify non-empty data
-	data := `[
-		{
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"bar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-                        "ddtags":"tag1:value1,tag2:value2",
-                        "hostname":"127.0.0.1",
-                        "message":{"message": "nested"},
-                        "service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"foobar",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"baz",
-			"service":"test"
-		}, {
-			"ddsource":"nginx",
-			"ddtags":"tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-			"message":"xyz",
-			"service":"test"
-		}, {
-			"ddsource": "nginx",
-			"ddtags":"tag1:value1,tag2:value2,",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-                }, {
-			"ddsource":"nginx",
-			"ddtags":",tag1:value1,tag2:value2",
-			"hostname":"127.0.0.1",
-                        "message":"xyz",
-                        "service":"test"
-		}
-	]`
-	rowsExpected := 7
-	resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"nested","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}
-{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}`
-	f(data, rowsExpected, resultExpected)
-}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -1,10 +1,14 @@
 package elasticsearch
 
 import (
+	"bufio"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -53,12 +57,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
-		// Return fake response for Logstash APIs requests.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
 	switch path {
 	case "/":
 		switch r.Method {
@@ -99,10 +97,9 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk")
+		lmp := cp.NewLogMessageProcessor()
 		isGzip := r.Header.Get("Content-Encoding") == "gzip"
-		streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-		n, err := readBulkRequest(streamName, r.Body, isGzip, cp.TimeField, cp.MsgFields, lmp)
+		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, lmp)
 		lmp.MustClose()
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
@@ -128,10 +125,11 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 
 var (
 	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
+	rowsIngestedTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
 	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (int, error) {
+func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string, lmp insertutils.LogMessageProcessor) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
 	if isGzip {
@@ -146,29 +144,48 @@ func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField stri
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
-	lr := insertutils.NewLineReader(streamName, wcr)
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
 
 	n := 0
+	nCheckpoint := 0
 	for {
-		ok, err := readBulkLine(lr, timeField, msgFields, lmp)
+		ok, err := readBulkLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil || !ok {
+			rowsIngestedTotal.Add(n - nCheckpoint)
 			return n, err
 		}
 		n++
+		if batchSize := n - nCheckpoint; n >= 1000 {
+			rowsIngestedTotal.Add(batchSize)
+			nCheckpoint = n
+		}
 	}
 }
 
-func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
+var lineBufferPool bytesutil.ByteBufferPool
+
+func readBulkLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 
 	// Read the command, must be "create" or "index"
 	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read "create" or "index" command, since its size exceeds -insert.maxLineSizeBytes=%d`,
+						insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
 		}
-		line = lr.Line
+		line = sc.Bytes()
 	}
 	lineStr := bytesutil.ToUnsafeString(line)
 	if !strings.Contains(lineStr, `"create"`) && !strings.Contains(lineStr, `"index"`) {
@@ -176,18 +193,16 @@ func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []stri
 	}
 
 	// Decode log message
-	if !lr.NextLine() {
-		if err := lr.Err(); err != nil {
+	if !sc.Scan() {
+		if err := sc.Err(); err != nil {
+			if errors.Is(err, bufio.ErrTooLong) {
+				return false, fmt.Errorf("cannot read log message, since its size exceeds -insert.maxLineSizeBytes=%d", insertutils.MaxLineSizeBytes.IntN())
+			}
 			return false, err
 		}
 		return false, fmt.Errorf(`missing log message after the "create" or "index" command`)
 	}
-	line = lr.Line
-	if len(line) == 0 {
-		// Special case - the line could be too long, so it was skipped.
-		// Continue parsing next lines.
-		return true, nil
-	}
+	line = sc.Bytes()
 	p := logstorage.GetJSONParser()
 	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
@@ -200,8 +215,8 @@ func readBulkLine(lr *insertutils.LineReader, timeField string, msgFields []stri
 	if ts == 0 {
 		ts = time.Now().UnixNano()
 	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
 	logstorage.PutJSONParser(p)
 
 	return true, nil
@@ -230,8 +245,19 @@ func parseElasticsearchTimestamp(s string) (int64, error) {
 		return 0, nil
 	}
 	if len(s) < len("YYYY-MM-DD") || s[len("YYYY")] != '-' {
-		// Try parsing timestamp in seconds or milliseconds
-		return insertutils.ParseUnixTimestamp(s)
+		// Try parsing timestamp in milliseconds
+		n, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse timestamp in milliseconds from %q: %w", s, err)
+		}
+		if n > int64(math.MaxInt64)/1e6 {
+			return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+		}
+		if n < int64(math.MinInt64)/1e6 {
+			return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+		}
+		n *= 1e6
+		return n, nil
 	}
 	if len(s) == len("YYYY-MM-DD") {
 		t, err := time.Parse("2006-01-02", s)

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -15,7 +15,7 @@ func TestReadBulkRequest_Failure(t *testing.T) {
 
 		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, false, "_time", []string{"_msg"}, tlp)
+		rows, err := readBulkRequest(r, false, "_time", "_msg", tlp)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -33,22 +33,21 @@ foobar`)
 }
 
 func TestReadBulkRequest_Success(t *testing.T) {
-	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
 		tlp := &insertutils.TestLogMessageProcessor{}
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, false, timeField, msgFields, tlp)
+		rows, err := readBulkRequest(r, false, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 
@@ -56,41 +55,39 @@ func TestReadBulkRequest_Success(t *testing.T) {
 		tlp = &insertutils.TestLogMessageProcessor{}
 		compressedData := compressData(data)
 		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest("test", r, true, timeField, msgFields, tlp)
+		rows, err = readBulkRequest(r, true, timeField, msgField, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if rows != len(timestampsExpected) {
-			t.Fatalf("unexpected rows read; got %d; want %d", rows, len(timestampsExpected))
+		if rows != rowsExpected {
+			t.Fatalf("unexpected rows read; got %d; want %d", rows, rowsExpected)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatalf("verification failure after compression: %s", err)
 		}
 	}
 
 	// Verify an empty data
-	f("", "_time", "_msg", nil, "")
-	f("\n", "_time", "_msg", nil, "")
-	f("\n\n", "_time", "_msg", nil, "")
+	f("", "_time", "_msg", 0, nil, "")
+	f("\n", "_time", "_msg", 0, nil, "")
+	f("\n\n", "_time", "_msg", 0, nil, "")
 
 	// Verify non-empty data
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
 {"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"create":{"_index":"filebeat-8.8.0"}}
-{"@timestamp":"2023-06-06 04:48:12.735+01:00","message":"baz"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
 {"index":{"_index":"filebeat-8.8.0"}}
-{"message":"xyz","@timestamp":"1686026893735","x":"y"}
-{"create":{"_index":"filebeat-8.8.0"}}
-{"message":"qwe rty","@timestamp":"1686026893"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}
-{"_msg":"qwe rty"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 
 func compressData(s string) string {

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -32,7 +32,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 	dataBytes := bytesutil.ToUnsafeBytes(data)
 
 	timeField := "@timestamp"
-	msgFields := []string{"message"}
+	msgField := "message"
 	blp := &insertutils.BenchmarkLogMessageProcessor{}
 
 	b.ReportAllocs()
@@ -41,7 +41,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest("test", r, isGzip, timeField, msgFields, blp)
+			_, err := readBulkRequest(r, isGzip, timeField, msgField, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutils/common_params.go

@@ -1,11 +1,7 @@
 package insertutils
 
 import (
-	"flag"
-	"fmt"
 	"net/http"
-	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -19,21 +15,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
-var (
-	defaultMsgValue = flag.String("defaultMsgValue", "missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field",
-		"Default value for _msg field if the ingested log entry doesn't contain it; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field")
-)
-
 // CommonParams contains common HTTP parameters used by log ingestion APIs.
 //
 // See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters
 type CommonParams struct {
 	TenantID     logstorage.TenantID
 	TimeField    string
-	MsgFields    []string
+	MsgField     string
 	StreamFields []string
 	IgnoreFields []string
-	ExtraFields  []logstorage.Field
 
 	Debug           bool
 	DebugRequestURI string
@@ -48,27 +38,22 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		return nil, err
 	}
 
-	timeField := "_time"
-	if tf := httputils.GetRequestValue(r, "_time_field", "VL-Time-Field"); tf != "" {
+	// Extract time field name from _time_field query arg
+	var timeField = "_time"
+	if tf := r.FormValue("_time_field"); tf != "" {
 		timeField = tf
 	}
 
-	msgFields := httputils.GetArray(r, "_msg_field", "VL-Msg-Field")
-	streamFields := httputils.GetArray(r, "_stream_fields", "VL-Stream-Fields")
-	ignoreFields := httputils.GetArray(r, "ignore_fields", "VL-Ignore-Fields")
-
-	extraFields, err := getExtraFields(r)
-	if err != nil {
-		return nil, err
+	// Extract message field name from _msg_field query arg
+	var msgField = ""
+	if msgf := r.FormValue("_msg_field"); msgf != "" {
+		msgField = msgf
 	}
 
-	debug := false
-	if dv := httputils.GetRequestValue(r, "debug", "VL-Debug"); dv != "" {
-		debug, err = strconv.ParseBool(dv)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse debug=%q: %w", dv, err)
-		}
-	}
+	streamFields := httputils.GetArray(r, "_stream_fields")
+	ignoreFields := httputils.GetArray(r, "ignore_fields")
+
+	debug := httputils.GetBool(r, "debug")
 	debugRequestURI := ""
 	debugRemoteAddr := ""
 	if debug {
@@ -79,57 +64,28 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 	cp := &CommonParams{
 		TenantID:        tenantID,
 		TimeField:       timeField,
-		MsgFields:       msgFields,
+		MsgField:        msgField,
 		StreamFields:    streamFields,
 		IgnoreFields:    ignoreFields,
-		ExtraFields:     extraFields,
 		Debug:           debug,
 		DebugRequestURI: debugRequestURI,
 		DebugRemoteAddr: debugRemoteAddr,
 	}
-
 	return cp, nil
 }
 
-func getExtraFields(r *http.Request) ([]logstorage.Field, error) {
-	efs := httputils.GetArray(r, "extra_fields", "VL-Extra-Fields")
-	if len(efs) == 0 {
-		return nil, nil
-	}
-
-	extraFields := make([]logstorage.Field, len(efs))
-	for i, ef := range efs {
-		n := strings.Index(ef, "=")
-		if n <= 0 || n == len(ef)-1 {
-			return nil, fmt.Errorf(`invalid extra_field format: %q; must be in the form "field=value"`, ef)
-		}
-		extraFields[i] = logstorage.Field{
-			Name:  ef[:n],
-			Value: ef[n+1:],
-		}
-	}
-	return extraFields, nil
-}
-
 // GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.
-func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignoreFields []string, extraFields []logstorage.Field) *CommonParams {
+func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams {
 	// See https://docs.victoriametrics.com/victorialogs/logsql/#unpack_syslog-pipe
-	if streamFields == nil {
-		streamFields = []string{
-			"hostname",
-			"app_name",
-			"proc_id",
-		}
-	}
 	cp := &CommonParams{
 		TenantID:  tenantID,
 		TimeField: "timestamp",
-		MsgFields: []string{
-			"message",
+		MsgField:  "message",
+		StreamFields: []string{
+			"hostname",
+			"app_name",
+			"proc_id",
 		},
-		StreamFields: streamFields,
-		IgnoreFields: ignoreFields,
-		ExtraFields:  extraFields,
 	}
 
 	return cp
@@ -137,12 +93,10 @@ func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignore
 
 // LogMessageProcessor is an interface for log message processors.
 type LogMessageProcessor interface {
-	// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
-	//
-	// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
+	// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
 	//
 	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
-	AddRow(timestamp int64, fields, streamFields []logstorage.Field)
+	AddRow(timestamp int64, fields []logstorage.Field)
 
 	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
 	MustClose()
@@ -156,9 +110,6 @@ type logMessageProcessor struct {
 
 	cp *CommonParams
 	lr *logstorage.LogRows
-
-	rowsIngestedTotal  *metrics.Counter
-	bytesIngestedTotal *metrics.Counter
 }
 
 func (lmp *logMessageProcessor) initPeriodicFlush() {
@@ -188,16 +139,10 @@ func (lmp *logMessageProcessor) initPeriodicFlush() {
 }
 
 // AddRow adds new log message to lmp with the given timestamp and fields.
-//
-// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
-func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
+func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
 	lmp.mu.Lock()
 	defer lmp.mu.Unlock()
 
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(fields)
-	lmp.bytesIngestedTotal.Add(n)
-
 	if len(fields) > *MaxFieldsPerLine {
 		rf := logstorage.RowFormatter(fields)
 		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
@@ -205,11 +150,11 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 		return
 	}
 
-	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields, streamFields)
+	lmp.lr.MustAdd(lmp.cp.TenantID, timestamp, fields)
 	if lmp.cp.Debug {
 		s := lmp.lr.GetRowString(0)
 		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
 		rowsDroppedTotalDebug.Inc()
 		return
 	}
@@ -238,17 +183,12 @@ func (lmp *logMessageProcessor) MustClose() {
 // NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
 //
 // MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.
-func (cp *CommonParams) NewLogMessageProcessor(protocolName string) LogMessageProcessor {
-	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.ExtraFields, *defaultMsgValue)
-	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
-	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
+func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
+	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
 	lmp := &logMessageProcessor{
 		cp: cp,
 		lr: lr,
 
-		rowsIngestedTotal:  rowsIngestedTotal,
-		bytesIngestedTotal: bytesIngestedTotal,
-
 		stopCh: make(chan struct{}),
 	}
 	lmp.initPeriodicFlush()
@@ -256,7 +196,5 @@ func (cp *CommonParams) NewLogMessageProcessor(protocolName string) LogMessagePr
 	return lmp
 }
 
-var (
-	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
-)
+var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/line_reader.go

@@ -1,146 +0,0 @@
-package insertutils
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-)
-
-// LineReader reads newline-delimited lines from the underlying reader
-type LineReader struct {
-	// Line contains the next line read after the call to NextLine
-	//
-	// The Line contents is valid until the next call to NextLine.
-	Line []byte
-
-	// name is the LineReader name
-	name string
-
-	// r is the underlying reader to read data from
-	r io.Reader
-
-	// buf is a buffer for reading the next line
-	buf []byte
-
-	// bufOffset is the offset at buf to read the next line from
-	bufOffset int
-
-	// err is the last error when reading data from r
-	err error
-
-	// eofReached is set to true when all the data is read from r
-	eofReached bool
-}
-
-// NewLineReader returns LineReader for r.
-func NewLineReader(name string, r io.Reader) *LineReader {
-	return &LineReader{
-		name: name,
-		r:    r,
-	}
-}
-
-// NextLine reads the next line from the underlying reader.
-//
-// It returns true if the next line is successfully read into Line.
-// If the line length exceeds MaxLineSizeBytes, then this line is skipped
-// and an empty line is returned instead.
-//
-// If false is returned, then no more lines left to read from r.
-// Check for Err in this case.
-func (lr *LineReader) NextLine() bool {
-	for {
-		if lr.bufOffset >= len(lr.buf) {
-			if lr.err != nil || lr.eofReached {
-				return false
-			}
-			if !lr.readMoreData() {
-				return false
-			}
-			if lr.bufOffset >= len(lr.buf) && lr.eofReached {
-				return false
-			}
-		}
-
-		buf := lr.buf[lr.bufOffset:]
-		if n := bytes.IndexByte(buf, '\n'); n >= 0 {
-			lr.Line = buf[:n]
-			lr.bufOffset += n + 1
-			return true
-		}
-		if lr.eofReached {
-			lr.Line = buf
-			lr.bufOffset += len(buf)
-			return true
-		}
-		if !lr.readMoreData() {
-			return false
-		}
-	}
-}
-
-// Err returns the last error after NextLine call.
-func (lr *LineReader) Err() error {
-	if lr.err == nil {
-		return nil
-	}
-	return fmt.Errorf("%s: %s", lr.name, lr.err)
-}
-
-func (lr *LineReader) readMoreData() bool {
-	if lr.bufOffset > 0 {
-		lr.buf = append(lr.buf[:0], lr.buf[lr.bufOffset:]...)
-		lr.bufOffset = 0
-	}
-
-	bufLen := len(lr.buf)
-	if bufLen >= MaxLineSizeBytes.IntN() {
-		logger.Warnf("%s: the line length exceeds -insert.maxLineSizeBytes=%d; skipping it; line contents=%q", lr.name, MaxLineSizeBytes.IntN(), lr.buf)
-		tooLongLinesSkipped.Inc()
-		return lr.skipUntilNextLine()
-	}
-
-	lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-	n, err := lr.r.Read(lr.buf[bufLen:])
-	lr.buf = lr.buf[:bufLen+n]
-	if err != nil {
-		if errors.Is(err, io.EOF) {
-			lr.eofReached = true
-			return true
-		}
-		lr.err = fmt.Errorf("cannot read the next line: %s", err)
-	}
-	return n > 0
-}
-
-var tooLongLinesSkipped = metrics.NewCounter("vl_too_long_lines_skipped_total")
-
-func (lr *LineReader) skipUntilNextLine() bool {
-	for {
-		lr.buf = slicesutil.SetLength(lr.buf, MaxLineSizeBytes.IntN())
-		n, err := lr.r.Read(lr.buf)
-		lr.buf = lr.buf[:n]
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				lr.eofReached = true
-				lr.buf = lr.buf[:0]
-				return true
-			}
-			lr.err = fmt.Errorf("cannot skip the current line: %s", err)
-			return false
-		}
-		if n := bytes.IndexByte(lr.buf, '\n'); n >= 0 {
-			// Include \n in the buf, so too long line is replaced with an empty line.
-			// This is needed for maintaining synchorinzation consistency between lines
-			// in protocols such as Elasticsearch bulk import.
-			lr.buf = append(lr.buf[:0], lr.buf[n:]...)
-			return true
-		}
-	}
-}

app/vlinsert/insertutils/line_reader_test.go

@@ -1,161 +0,0 @@
-package insertutils
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"reflect"
-	"testing"
-)
-
-func TestLineReader_Success(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("\n", []string{""})
-	f("\n\n", []string{"", ""})
-	f("foo", []string{"foo"})
-	f("foo\n", []string{"foo"})
-	f("\nfoo", []string{"", "foo"})
-	f("foo\n\n", []string{"foo", ""})
-	f("foo\nbar", []string{"foo", "bar"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\n\nbar\n\n", []string{"", "foo", "", "bar", ""})
-}
-
-func TestLineReader_SkipUntilNextLine(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		r := bytes.NewBufferString(data)
-		lr := NewLineReader("foo", r)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err != nil {
-			t.Fatalf("unexpected error for data=%q: %s", data, err)
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines for data=%q\ngot\n%q\nwant\n%q", data, lines, linesExpected)
-		}
-	}
-
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLineLen := MaxLineSizeBytes.IntN() + overflow
-		longLine := string(make([]byte, longLineLen))
-
-		// Single long line
-		data := longLine
-		f(data, nil)
-
-		// Multiple long lines
-		data = longLine + "\n" + longLine
-		f(data, []string{""})
-
-		data = longLine + "\n" + longLine + "\n"
-		f(data, []string{"", ""})
-
-		// Long line in the middle
-		data = "foo\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "bar"})
-
-		// Multiple long lines in the middle
-		data = "foo\n" + longLine + "\n" + longLine + "\nbar"
-		f(data, []string{"foo", "", "", "bar"})
-
-		// Long line in the end
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		// Long line in the end
-		data = "foo\n" + longLine + "\n"
-		f(data, []string{"foo", ""})
-	}
-}
-
-func TestLineReader_Failure(t *testing.T) {
-	f := func(data string, linesExpected []string) {
-		t.Helper()
-
-		fr := &failureReader{
-			r: bytes.NewBufferString(data),
-		}
-		lr := NewLineReader("foo", fr)
-		var lines []string
-		for lr.NextLine() {
-			lines = append(lines, string(lr.Line))
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if lr.NextLine() {
-			t.Fatalf("expecting error on the second call to NextLine()")
-		}
-		if err := lr.Err(); err == nil {
-			t.Fatalf("expecting non-nil error on the second call")
-		}
-		if !reflect.DeepEqual(lines, linesExpected) {
-			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
-		}
-	}
-
-	f("", nil)
-	f("foo", nil)
-	f("foo\n", []string{"foo"})
-	f("\n", []string{""})
-	f("foo\nbar", []string{"foo"})
-	f("foo\nbar\n", []string{"foo", "bar"})
-	f("\nfoo\nbar\n\n", []string{"", "foo", "bar", ""})
-
-	// long line
-	longLineLen := MaxLineSizeBytes.IntN()
-	for _, overflow := range []int{0, 100, MaxLineSizeBytes.IntN(), MaxLineSizeBytes.IntN() + 1, 2 * MaxLineSizeBytes.IntN()} {
-		longLine := string(make([]byte, longLineLen+overflow))
-
-		data := longLine
-		f(data, nil)
-
-		data = "foo\n" + longLine
-		f(data, []string{"foo"})
-
-		data = longLine + "\nfoo"
-		f(data, []string{""})
-
-		data = longLine + "\nfoo\n"
-		f(data, []string{"", "foo"})
-	}
-}
-
-type failureReader struct {
-	r io.Reader
-}
-
-func (r *failureReader) Read(p []byte) (int, error) {
-	n, _ := r.r.Read(p)
-	if n > 0 {
-		return n, nil
-	}
-	return 0, fmt.Errorf("some error")
-}

app/vlinsert/insertutils/testutils.go

@@ -15,10 +15,7 @@ type TestLogMessageProcessor struct {
 }
 
 // AddRow adds row with the given timestamp and fields to tlp
-func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("BUG: streamFields must be nil; got %v", streamFields))
-	}
+func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
 	tlp.timestamps = append(tlp.timestamps, timestamp)
 	tlp.rows = append(tlp.rows, string(logstorage.MarshalFieldsToJSON(nil, fields)))
 }
@@ -28,10 +25,10 @@ func (tlp *TestLogMessageProcessor) MustClose() {
 }
 
 // Verify verifies the number of rows, timestamps and results after AddRow calls.
-func (tlp *TestLogMessageProcessor) Verify(timestampsExpected []int64, resultExpected string) error {
+func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error {
 	result := strings.Join(tlp.rows, "\n")
-	if len(tlp.rows) != len(timestampsExpected) {
-		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), len(timestampsExpected), result, resultExpected)
+	if len(tlp.rows) != rowsExpected {
+		return fmt.Errorf("unexpected rows read; got %d; want %d;\nrows read:\n%s\nrows wanted\n%s", len(tlp.rows), rowsExpected, result, resultExpected)
 	}
 
 	if !reflect.DeepEqual(tlp.timestamps, timestampsExpected) {
@@ -48,7 +45,7 @@ func (tlp *TestLogMessageProcessor) Verify(timestampsExpected []int64, resultExp
 type BenchmarkLogMessageProcessor struct{}
 
 // AddRow implements LogMessageProcessor interface.
-func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _, _ []logstorage.Field) {
+func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field) {
 }
 
 // MustClose implements LogMessageProcessor interface.

app/vlinsert/insertutils/timestamp.go

@@ -2,8 +2,6 @@ package insertutils
 
 import (
 	"fmt"
-	"math"
-	"strconv"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -21,49 +19,15 @@ func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage
 		if f.Name != timeField {
 			continue
 		}
-		nsecs, err := parseTimestamp(f.Value)
-		if err != nil {
-			return 0, fmt.Errorf("cannot parse timestamp from field %q: %s", timeField, err)
+		if f.Value == "" || f.Value == "0" {
+			return time.Now().UnixNano(), nil
+		}
+		nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(f.Value)
+		if !ok {
+			return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp from %s=%q", timeField, f.Value)
 		}
 		f.Value = ""
-		if nsecs == 0 {
-			nsecs = time.Now().UnixNano()
-		}
 		return nsecs, nil
 	}
 	return time.Now().UnixNano(), nil
 }
-
-func parseTimestamp(s string) (int64, error) {
-	if s == "" || s == "0" {
-		return time.Now().UnixNano(), nil
-	}
-	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {
-		return ParseUnixTimestamp(s)
-	}
-	nsecs, ok := logstorage.TryParseTimestampRFC3339Nano(s)
-	if !ok {
-		return 0, fmt.Errorf("cannot unmarshal rfc3339 timestamp %q", s)
-	}
-	return nsecs, nil
-}
-
-// ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.
-func ParseUnixTimestamp(s string) (int64, error) {
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
-	}
-	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds. Convert it to milliseconds
-		n *= 1e3
-	}
-	if n > int64(math.MaxInt64)/1e6 {
-		return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
-	}
-	if n < int64(math.MinInt64)/1e6 {
-		return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
-	}
-	n *= 1e6
-	return n, nil
-}

app/vlinsert/insertutils/timestamp_test.go

@@ -27,41 +27,25 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 		}
 	}
 
-	// UTC time
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
 		{Name: "time", Value: "2024-06-18T23:37:20Z"},
 	}, 1718753840000000000)
 
-	// Time with timezone
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
 		{Name: "time", Value: "2024-06-18T23:37:20+08:00"},
 	}, 1718725040000000000)
 
-	// SQL datetime format
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "2024-06-18 23:37:20.123-05:30"},
+		{Name: "time", Value: "2024-06-18T23:37:20.123-05:30"},
 	}, 1718773640123000000)
 
-	// Time with nanosecond precision
 	f("time", []logstorage.Field{
 		{Name: "time", Value: "2024-06-18T23:37:20.123456789-05:30"},
 		{Name: "foo", Value: "bar"},
 	}, 1718773640123456789)
-
-	// Unix timestamp in milliseconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640123"},
-	}, 1718773640123000000)
-
-	// Unix timestamp in seconds
-	f("time", []logstorage.Field{
-		{Name: "foo", Value: "bar"},
-		{Name: "time", Value: "1718773640"},
-	}, 1718773640000000000)
 }
 
 func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
@@ -82,6 +66,9 @@ func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
 
 	f("foobar")
 
+	// no Z at the end
+	f("2024-06-18T23:37:20")
+
 	// incomplete time
 	f("2024-06-18")
 	f("2024-06-18T23:37")

app/vlinsert/journald/journald.go

@@ -1,250 +0,0 @@
-package journald
-
-import (
-	"bytes"
-	"encoding/binary"
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-	"slices"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-const (
-	journaldEntryMaxNameLen = 64
-)
-
-var (
-	bodyBufferPool                bytesutil.ByteBufferPool
-	allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]+`)
-)
-
-var (
-	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Journal fields to be used as stream fields. "+
-		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
-	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Journal fields to ignore. "+
-		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
-	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Journal field to be used as time field. "+
-		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
-	journaldTenantID             = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint.")
-	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
-)
-
-func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		return nil, err
-	}
-	if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 {
-		tenantID, err := logstorage.ParseTenantID(*journaldTenantID)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse -journald.tenantID=%q for journald: %w", *journaldTenantID, err)
-		}
-		cp.TenantID = tenantID
-	}
-	if cp.TimeField != "" {
-		cp.TimeField = *journaldTimeField
-	}
-	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = *journaldStreamFields
-	}
-	if len(cp.IgnoreFields) == 0 {
-		cp.IgnoreFields = *journaldIgnoreFields
-	}
-	cp.MsgFields = []string{"MESSAGE"}
-	return cp, nil
-}
-
-// RequestHandler processes Journald Export insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	case "/upload":
-		if r.Header.Get("Content-Type") != "application/vnd.fdo.journal" {
-			httpserver.Errorf(w, r, "only application/vnd.fdo.journal encoding is supported for Journald")
-			return true
-		}
-		handleJournald(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-// handleJournald parses Journal binary entries
-func handleJournald(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsJournaldTotal.Inc()
-
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	reader := r.Body
-	var err error
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
-	writeconcurrencylimiter.PutReader(wcr)
-	bb := bodyBufferPool.Get()
-	defer bodyBufferPool.Put(bb)
-	if r.Header.Get("Content-Encoding") == "zstd" {
-		bb.B, err = zstd.Decompress(bb.B[:0], data)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot decompress zstd-encoded request with length %d: %s", len(data), err)
-			return
-		}
-		data = bb.B
-	}
-	cp, err := getCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-
-	lmp := cp.NewLogMessageProcessor("journald")
-	err = parseJournaldRequest(data, lmp, cp)
-	lmp.MustClose()
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse Journald protobuf request: %s", err)
-		return
-	}
-
-	// update requestJournaldDuration only for successfully parsed requests
-	// There is no need in updating requestJournaldDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestJournaldDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsJournaldTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
-	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
-
-	requestJournaldDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
-)
-
-// See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *insertutils.CommonParams) error {
-	var fields []logstorage.Field
-	var ts int64
-	var size uint64
-	var name, value string
-	var line []byte
-
-	currentTimestamp := time.Now().UnixNano()
-
-	for len(data) > 0 {
-		idx := bytes.IndexByte(data, '\n')
-		switch {
-		case idx > 0:
-			// process fields
-			line = data[:idx]
-			data = data[idx+1:]
-		case idx == 0:
-			// next message or end of file
-			// double new line is a separator for the next message
-			if len(fields) > 0 {
-				if ts == 0 {
-					ts = currentTimestamp
-				}
-				lmp.AddRow(ts, fields, nil)
-				fields = fields[:0]
-			}
-			// skip newline separator
-			data = data[1:]
-			continue
-		case idx < 0:
-			return fmt.Errorf("missing new line separator, unread data left=%d", len(data))
-		}
-
-		idx = bytes.IndexByte(line, '=')
-		// could b either e key=value\n pair
-		// or just  key\n
-		// with binary data at the buffer
-		if idx > 0 {
-			name = bytesutil.ToUnsafeString(line[:idx])
-			value = bytesutil.ToUnsafeString(line[idx+1:])
-		} else {
-			name = bytesutil.ToUnsafeString(line)
-			if len(data) == 0 {
-				return fmt.Errorf("unexpected zero data for binary field value of key=%s", name)
-			}
-			// size of binary data encoded as le i64 at the begging
-			idx, err := binary.Decode(data, binary.LittleEndian, &size)
-			if err != nil {
-				return fmt.Errorf("failed to extract binary field %q value size: %w", name, err)
-			}
-			// skip binary data sise
-			data = data[idx:]
-			if size == 0 {
-				return fmt.Errorf("unexpected zero binary data size decoded %d", size)
-			}
-			if int(size) > len(data) {
-				return fmt.Errorf("binary data size=%d cannot exceed size of the data at buffer=%d", size, len(data))
-			}
-			value = bytesutil.ToUnsafeString(data[:size])
-			data = data[int(size):]
-			// binary data must has new line separator for the new line or next field
-			if len(data) == 0 {
-				return fmt.Errorf("unexpected empty buffer after binary field=%s read", name)
-			}
-			lastB := data[0]
-			if lastB != '\n' {
-				return fmt.Errorf("expected new line separator after binary field=%s, got=%s", name, string(lastB))
-			}
-			data = data[1:]
-		}
-		// https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
-		if len(name) > journaldEntryMaxNameLen {
-			return fmt.Errorf("journald entry name should not exceed %d symbols, got: %q", journaldEntryMaxNameLen, name)
-		}
-		if !allowedJournaldEntryNameChars.MatchString(name) {
-			return fmt.Errorf("journald entry name should consist of `A-Z0-9_` characters and must start from non-digit symbol")
-		}
-		if name == cp.TimeField {
-			n, err := strconv.ParseInt(value, 10, 64)
-			if err != nil {
-				return fmt.Errorf("failed to parse Journald timestamp, %w", err)
-			}
-			ts = n * 1e3
-			continue
-		}
-
-		if slices.Contains(cp.MsgFields, name) {
-			name = "_msg"
-		}
-
-		if *journaldIncludeEntryMetadata || !strings.HasPrefix(name, "__") {
-			fields = append(fields, logstorage.Field{
-				Name:  name,
-				Value: value,
-			})
-		}
-	}
-	if len(fields) > 0 {
-		if ts == 0 {
-			ts = currentTimestamp
-		}
-		lmp.AddRow(ts, fields, nil)
-	}
-	return nil
-}

app/vlinsert/journald/journald_test.go

@@ -1,68 +0,0 @@
-package journald
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-)
-
-func TestPushJournaldOk(t *testing.T) {
-	f := func(src string, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-		tlp := &insertutils.TestLogMessageProcessor{}
-		cp := &insertutils.CommonParams{
-			TimeField: "__REALTIME_TIMESTAMP",
-			MsgFields: []string{"MESSAGE"},
-		}
-		if err := parseJournaldRequest([]byte(src), tlp, cp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-	// Single event
-	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n",
-		[]int64{91723819283000},
-		"{\"_msg\":\"Test message\"}",
-	)
-
-	// Multiple events
-	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
-		[]int64{91723819283000, 91723819284000},
-		"{\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
-	)
-
-	// Parse binary data
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
-		[]int64{1729698775704404000},
-		"{\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
-	)
-}
-
-func TestPushJournald_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-		tlp := &insertutils.TestLogMessageProcessor{}
-		cp := &insertutils.CommonParams{
-			TimeField: "__REALTIME_TIMESTAMP",
-			MsgFields: []string{"MESSAGE"},
-		}
-		if err := parseJournaldRequest([]byte(data), tlp, cp); err == nil {
-			t.Fatalf("expected non nil error")
-		}
-	}
-	// missing new line terminator for binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda2")
-	// missing new line terminator
-	f("__REALTIME_TIMESTAMP=91723819283\n=Test message")
-	// empty field name
-	f("__REALTIME_TIMESTAMP=91723819283\n=Test message\n")
-	// field name starting with number
-	f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n")
-	// field name exceeds 64 limit
-	f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n")
-	// Only allow A-Z0-9 and '_'
-	f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n")
-}

app/vlinsert/jsonline/jsonline.go

@@ -1,6 +1,8 @@
 package jsonline
 
 import (
+	"bufio"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -8,6 +10,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -49,9 +52,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		reader = zr
 	}
 
-	lmp := cp.NewLogMessageProcessor("jsonline")
-	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
+	lmp := cp.NewLogMessageProcessor()
+	err = processStreamInternal(reader, cp.TimeField, cp.MsgField, lmp)
 	lmp.MustClose()
 
 	if err != nil {
@@ -64,15 +66,20 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) error {
+func processStreamInternal(r io.Reader, timeField, msgField string, lmp insertutils.LogMessageProcessor) error {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
-	lr := insertutils.NewLineReader(streamName, wcr)
+	lb := lineBufferPool.Get()
+	defer lineBufferPool.Put(lb)
+
+	lb.B = bytesutil.ResizeNoCopyNoOverallocate(lb.B, insertutils.MaxLineSizeBytes.IntN())
+	sc := bufio.NewScanner(wcr)
+	sc.Buffer(lb.B, len(lb.B))
 
 	n := 0
 	for {
-		ok, err := readLine(lr, timeField, msgFields, lmp)
+		ok, err := readLine(sc, timeField, msgField, lmp)
 		wcr.DecConcurrency()
 		if err != nil {
 			errorsTotal.Inc()
@@ -82,17 +89,23 @@ func processStreamInternal(streamName string, r io.Reader, timeField string, msg
 			return nil
 		}
 		n++
+		rowsIngestedTotal.Inc()
 	}
 }
 
-func readLine(lr *insertutils.LineReader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (bool, error) {
+func readLine(sc *bufio.Scanner, timeField, msgField string, lmp insertutils.LogMessageProcessor) (bool, error) {
 	var line []byte
 	for len(line) == 0 {
-		if !lr.NextLine() {
-			err := lr.Err()
-			return false, err
+		if !sc.Scan() {
+			if err := sc.Err(); err != nil {
+				if errors.Is(err, bufio.ErrTooLong) {
+					return false, fmt.Errorf(`cannot read json line, since its size exceeds -insert.maxLineSizeBytes=%d`, insertutils.MaxLineSizeBytes.IntN())
+				}
+				return false, err
+			}
+			return false, nil
 		}
-		line = lr.Line
+		line = sc.Bytes()
 	}
 
 	p := logstorage.GetJSONParser()
@@ -103,14 +116,18 @@ func readLine(lr *insertutils.LineReader, timeField string, msgFields []string,
 	if err != nil {
 		return false, fmt.Errorf("cannot get timestamp: %w", err)
 	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
+	logstorage.RenameField(p.Fields, msgField, "_msg")
+	lmp.AddRow(ts, p.Fields)
 	logstorage.PutJSONParser(p)
 
 	return true, nil
 }
 
+var lineBufferPool bytesutil.ByteBufferPool
+
 var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+
 	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
 	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
 

app/vlinsert/jsonline/jsonline_test.go

@@ -8,43 +8,32 @@ import (
 )
 
 func TestProcessStreamInternal_Success(t *testing.T) {
-	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
+	f := func(data, timeField, msgField string, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
-		msgFields := []string{msgField}
 		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, timeField, msgFields, tlp); err != nil {
+		if err := processStreamInternal(r, timeField, msgField, tlp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}
 
 	data := `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-{"message":"xyz","@timestamp":"2023-06-06 04:48:13.735Z","x":"y"}
+{"@timestamp":"2023-06-06T04:48:12.735Z","message":"baz"}
+{"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
-	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"_msg":"baz"}
-{"_msg":"xyz","x":"y"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
-
-	// Non-existing msgField
-	data = `{"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
-{"@timestamp":"2023-06-06T04:48:12.735+01:00","message":"baz"}
-`
-	timeField = "@timestamp"
-	msgField = "foobar"
-	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
-	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
-{"message":"baz"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
+	rowsExpected := 3
+	timestampsExpected := []int64{1686026891735000000, 1686026892735000000, 1686026893735000000}
+	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"@timestamp":"","_msg":"baz"}
+{"_msg":"xyz","@timestamp":"","x":"y"}`
+	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 
 func TestProcessStreamInternal_Failure(t *testing.T) {
@@ -53,7 +42,7 @@ func TestProcessStreamInternal_Failure(t *testing.T) {
 
 		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, "time", nil, tlp); err == nil {
+		if err := processStreamInternal(r, "time", "", tlp); err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
 	}

app/vlinsert/loki/loki_json.go

@@ -8,9 +8,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -18,6 +15,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/valyala/fastjson"
 )
 
 var parserPool fastjson.ParserPool
@@ -53,15 +52,16 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	lmp := cp.NewLogMessageProcessor("loki_json")
-	useDefaultStreamFields := len(cp.StreamFields) == 0
-	err = parseJSONRequest(data, lmp, useDefaultStreamFields)
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseJSONRequest(data, lmp)
 	lmp.MustClose()
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki json request: %s; data=%s", err, data)
+		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
 		return
 	}
 
+	rowsIngestedJSONTotal.Add(n)
+
 	// update requestJSONDuration only for successfully parsed requests
 	// There is no need in updating requestJSONDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
@@ -69,29 +69,31 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 }
 
 var (
-	requestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
+	requestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
+	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
+	requestJSONDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
-func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
+func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
 	v, err := p.ParseBytes(data)
 	if err != nil {
-		return fmt.Errorf("cannot parse JSON request body: %w", err)
+		return 0, fmt.Errorf("cannot parse JSON request body: %w", err)
 	}
 
 	streamsV := v.Get("streams")
 	if streamsV == nil {
-		return fmt.Errorf("missing `streams` item in the parsed JSON")
+		return 0, fmt.Errorf("missing `streams` item in the parsed JSON: %q", v)
 	}
 	streams, err := streamsV.Array()
 	if err != nil {
-		return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
+		return 0, fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
 	}
 
 	currentTimestamp := time.Now().UnixNano()
 	var commonFields []logstorage.Field
+	rowsIngested := 0
 	for _, stream := range streams {
 		// populate common labels from `stream` dict
 		commonFields = commonFields[:0]
@@ -100,11 +102,14 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 		if labelsV != nil {
 			o, err := labelsV.Object()
 			if err != nil {
-				return fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
+				return rowsIngested, fmt.Errorf("`stream` item in the parsed JSON must contain an object; got %q", labelsV)
 			}
 			labels = o
 		}
 		labels.Visit(func(k []byte, v *fastjson.Value) {
+			if err != nil {
+				return
+			}
 			vStr, errLocal := v.StringBytes()
 			if errLocal != nil {
 				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
@@ -116,37 +121,37 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 			})
 		})
 		if err != nil {
-			return fmt.Errorf("error when parsing `stream` object: %w", err)
+			return rowsIngested, fmt.Errorf("error when parsing `stream` object: %w", err)
 		}
 
 		// populate messages from `values` array
 		linesV := stream.Get("values")
 		if linesV == nil {
-			return fmt.Errorf("missing `values` item in the parsed `stream` object %q", stream)
+			return rowsIngested, fmt.Errorf("missing `values` item in the parsed JSON %q", stream)
 		}
 		lines, err := linesV.Array()
 		if err != nil {
-			return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
+			return rowsIngested, fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
 		}
 
 		fields := commonFields
 		for _, line := range lines {
 			lineA, err := line.Array()
 			if err != nil {
-				return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
+				return rowsIngested, fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
 			}
-			if len(lineA) < 2 || len(lineA) > 3 {
-				return fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2 or 3", line, len(lineA))
+			if len(lineA) != 2 {
+				return rowsIngested, fmt.Errorf("unexpected number of values in `values` item array %q; got %d want 2", line, len(lineA))
 			}
 
 			// parse timestamp
 			timestamp, err := lineA[0].StringBytes()
 			if err != nil {
-				return fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
+				return rowsIngested, fmt.Errorf("unexpected log timestamp type for %q; want string", lineA[0])
 			}
 			ts, err := parseLokiTimestamp(bytesutil.ToUnsafeString(timestamp))
 			if err != nil {
-				return fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
+				return rowsIngested, fmt.Errorf("cannot parse log timestamp %q: %w", timestamp, err)
 			}
 			if ts == 0 {
 				ts = currentTimestamp
@@ -155,46 +160,19 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 			// parse log message
 			msg, err := lineA[1].StringBytes()
 			if err != nil {
-				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
+				return rowsIngested, fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
 			}
 
 			fields = append(fields[:len(commonFields)], logstorage.Field{
 				Name:  "_msg",
 				Value: bytesutil.ToUnsafeString(msg),
 			})
-
-			// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
-			if len(lineA) > 2 {
-				structuredMetadata, err := lineA[2].Object()
-				if err != nil {
-					return fmt.Errorf("unexpected structured metadata type for %q; want JSON object", lineA[2])
-				}
-
-				structuredMetadata.Visit(func(k []byte, v *fastjson.Value) {
-					vStr, errLocal := v.StringBytes()
-					if errLocal != nil {
-						err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
-						return
-					}
-
-					fields = append(fields, logstorage.Field{
-						Name:  bytesutil.ToUnsafeString(k),
-						Value: bytesutil.ToUnsafeString(vStr),
-					})
-				})
-				if err != nil {
-					return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
-				}
-			}
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = commonFields
-			}
-			lmp.AddRow(ts, fields, streamFields)
+			lmp.AddRow(ts, fields)
 		}
+		rowsIngested += len(lines)
 	}
 
-	return nil
+	return rowsIngested, nil
 }
 
 func parseLokiTimestamp(s string) (int64, error) {

app/vlinsert/loki/loki_json_test.go

@@ -11,11 +11,12 @@ func TestParseJSONRequest_Failure(t *testing.T) {
 		t.Helper()
 
 		tlp := &insertutils.TestLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, false); err == nil {
+		n, err := parseJSONRequest([]byte(s), tlp)
+		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
-		if err := tlp.Verify(nil, ""); err != nil {
-			t.Fatalf("unexpected error: %s", err)
+		if n != 0 {
+			t.Fatalf("unexpected number of parsed lines: %d; want 0", n)
 		}
 	}
 	f(``)
@@ -44,19 +45,13 @@ func TestParseJSONRequest_Failure(t *testing.T) {
 	// Invalid length of `values` individual item
 	f(`{"streams":[{"values":[[]]}]}`)
 	f(`{"streams":[{"values":[["123"]]}]}`)
-	f(`{"streams":[{"values":[["123","456","789","8123"]]}]}`)
+	f(`{"streams":[{"values":[["123","456","789"]]}]}`)
 
 	// Invalid type for timestamp inside `values` individual item
 	f(`{"streams":[{"values":[[123,"456"]}]}`)
 
 	// Invalid type for log message
 	f(`{"streams":[{"values":[["123",1234]]}]}`)
-
-	// invalid structured metadata type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", ["metadata_1", "md_value"]]]}]}`)
-
-	// structured metadata with unexpected value type
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": 1}]] }]}`)
 }
 
 func TestParseJSONRequest_Success(t *testing.T) {
@@ -65,10 +60,11 @@ func TestParseJSONRequest_Success(t *testing.T) {
 
 		tlp := &insertutils.TestLogMessageProcessor{}
 
-		if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
+		n, err := parseJSONRequest([]byte(s), tlp)
+		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -120,8 +116,4 @@ func TestParseJSONRequest_Success(t *testing.T) {
 }`, []int64{1577836800000000001, 1577836900005000002, 1877836900005000002}, `{"foo":"bar","a":"b","_msg":"foo bar"}
 {"foo":"bar","a":"b","_msg":"abc"}
 {"x":"y","_msg":"yx"}`)
-
-	// values with metadata
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar","metadata_1":"md_value"}`)
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 }

app/vlinsert/loki/loki_json_timing_test.go

@@ -28,7 +28,8 @@ func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseJSONRequest(data, blp, false); err != nil {
+			_, err := parseJSONRequest(data, blp)
+			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}

app/vlinsert/loki/loki_protobuf.go

@@ -44,15 +44,16 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	lmp := cp.NewLogMessageProcessor("loki_protobuf")
-	useDefaultStreamFields := len(cp.StreamFields) == 0
-	err = parseProtobufRequest(data, lmp, useDefaultStreamFields)
+	lmp := cp.NewLogMessageProcessor()
+	n, err := parseProtobufRequest(data, lmp)
 	lmp.MustClose()
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
 		return
 	}
 
+	rowsIngestedProtobufTotal.Add(n)
+
 	// update requestProtobufDuration only for successfully parsed requests
 	// There is no need in updating requestProtobufDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
@@ -60,17 +61,18 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 }
 
 var (
-	requestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	requestsProtobufTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
+	requestProtobufDuration   = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
-func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
+func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
 	bb := bytesBufPool.Get()
 	defer bytesBufPool.Put(bb)
 
 	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
 	if err != nil {
-		return fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
+		return 0, fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
 	}
 	bb.B = buf
 
@@ -79,12 +81,13 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
 
 	err = req.UnmarshalProtobuf(bb.B)
 	if err != nil {
-		return fmt.Errorf("cannot parse request body: %w", err)
+		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
 	fields := getFields()
 	defer putFields(fields)
 
+	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
 	for i := range streams {
@@ -93,7 +96,7 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
 		// Labels are same for all entries in the stream.
 		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
 		if err != nil {
-			return fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
+			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
 		commonFieldsLen := len(fields.fields)
 
@@ -119,14 +122,11 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
 				ts = currentTimestamp
 			}
 
-			var streamFields []logstorage.Field
-			if useDefaultStreamFields {
-				streamFields = fields.fields[:commonFieldsLen]
-			}
-			lmp.AddRow(ts, fields.fields, streamFields)
+			lmp.AddRow(ts, fields.fields)
 		}
+		rowsIngested += len(stream.Entries)
 	}
-	return nil
+	return rowsIngested, nil
 }
 
 func getFields() *fields {

app/vlinsert/loki/loki_protobuf_test.go

@@ -15,10 +15,7 @@ type testLogMessageProcessor struct {
 	pr PushRequest
 }
 
-func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
-	if streamFields != nil {
-		panic(fmt.Errorf("unexpected non-nil streamFields: %v", streamFields))
-	}
+func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field) {
 	msg := ""
 	for _, f := range fields {
 		if f.Name == "_msg" {
@@ -53,21 +50,23 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 		t.Helper()
 
 		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
+		n, err := parseJSONRequest([]byte(s), tlp)
+		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if len(tlp.pr.Streams) != len(timestampsExpected) {
-			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
+		if n != len(tlp.pr.Streams) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
 		}
 
 		data := tlp.pr.MarshalProtobuf(nil)
 		encodedData := snappy.Encode(nil, data)
 
 		tlp2 := &insertutils.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(encodedData, tlp2, false); err != nil {
+		n, err = parseProtobufRequest(encodedData, tlp2)
+		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp2.Verify(n, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -31,7 +31,8 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseProtobufRequest(body, blp, false); err != nil {
+			_, err := parseProtobufRequest(body, blp)
+			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}

app/vlinsert/loki/pb.go

@@ -17,7 +17,7 @@ var mp easyproto.MarshalerPool
 
 // PushRequest represents Loki PushRequest
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L14
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
 type PushRequest struct {
 	Streams []Stream
 
@@ -87,7 +87,7 @@ func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []Labe
 
 // Stream represents Loki stream.
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L23
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
 type Stream struct {
 	Labels  string
 	Entries []Entry
@@ -139,7 +139,7 @@ func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair,
 
 // Entry represents Loki entry.
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L38
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
 type Entry struct {
 	Timestamp          time.Time
 	Line               string
@@ -203,7 +203,7 @@ func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]Label
 
 // LabelPair represents Loki label pair.
 //
-// See https://github.com/grafana/loki/blob/ada4b7b8713385fbe9f5984a5a0aaaddf1a7b851/pkg/push/push.proto#L33
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
 type LabelPair struct {
 	Name  string
 	Value string

app/vlinsert/main.go

@@ -1,16 +1,12 @@
 package vlinsert
 
 import (
-	"fmt"
 	"net/http"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
 )
 
@@ -27,7 +23,6 @@ func Stop() {
 // RequestHandler handles insert requests for VictoriaLogs
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
-
 	if !strings.HasPrefix(path, "/insert/") {
 		// Skip requests, which do not start with /insert/, since these aren't our requests.
 		return false
@@ -35,15 +30,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path = strings.TrimPrefix(path, "/insert")
 	path = strings.ReplaceAll(path, "//", "/")
 
-	switch path {
-	case "/jsonline":
+	if path == "/jsonline" {
 		jsonline.RequestHandler(w, r)
 		return true
-	case "/ready":
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(200)
-		fmt.Fprintf(w, `{"status":"ok"}`)
-		return true
 	}
 	switch {
 	case strings.HasPrefix(path, "/elasticsearch/"):
@@ -52,15 +41,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case strings.HasPrefix(path, "/loki/"):
 		path = strings.TrimPrefix(path, "/loki")
 		return loki.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/opentelemetry/"):
-		path = strings.TrimPrefix(path, "/opentelemetry")
-		return opentelemetry.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/journald/"):
-		path = strings.TrimPrefix(path, "/journald")
-		return journald.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/datadog/"):
-		path = strings.TrimPrefix(path, "/datadog")
-		return datadog.RequestHandler(path, w, r)
 	default:
 		return false
 	}

app/vlinsert/opentelemetry/opentelemetry.go

@@ -1,141 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-// RequestHandler processes Opentelemetry insert requests
-func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	switch path {
-	// use the same path as opentelemetry collector
-	// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
-	case "/v1/logs":
-		if r.Header.Get("Content-Type") == "application/json" {
-			httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
-			return true
-		}
-		handleProtobuf(r, w)
-		return true
-	default:
-		return false
-	}
-}
-
-func handleProtobuf(r *http.Request, w http.ResponseWriter) {
-	startTime := time.Now()
-	requestsProtobufTotal.Inc()
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
-
-	cp, err := insertutils.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
-		return
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf")
-	useDefaultStreamFields := len(cp.StreamFields) == 0
-	err = pushProtobufRequest(data, lmp, useDefaultStreamFields)
-	lmp.MustClose()
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
-		return
-	}
-
-	// update requestProtobufDuration only for successfully parsed requests
-	// There is no need in updating requestProtobufDuration for request errors,
-	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestProtobufDuration.UpdateDuration(startTime)
-}
-
-var (
-	requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-
-	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
-)
-
-func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
-	var req pb.ExportLogsServiceRequest
-	if err := req.UnmarshalProtobuf(data); err != nil {
-		errorsTotal.Inc()
-		return fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
-	}
-
-	var commonFields []logstorage.Field
-	for _, rl := range req.ResourceLogs {
-		attributes := rl.Resource.Attributes
-		commonFields = slicesutil.SetLength(commonFields, len(attributes))
-		for i, attr := range attributes {
-			commonFields[i].Name = attr.Key
-			commonFields[i].Value = attr.Value.FormatString()
-		}
-		commonFieldsLen := len(commonFields)
-		for _, sc := range rl.ScopeLogs {
-			commonFields = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp, useDefaultStreamFields)
-		}
-	}
-
-	return nil
-}
-
-func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) []logstorage.Field {
-	fields := commonFields
-	for _, lr := range sc.LogRecords {
-		fields = fields[:len(commonFields)]
-		fields = append(fields, logstorage.Field{
-			Name:  "_msg",
-			Value: lr.Body.FormatString(),
-		})
-		for _, attr := range lr.Attributes {
-			fields = append(fields, logstorage.Field{
-				Name:  attr.Key,
-				Value: attr.Value.FormatString(),
-			})
-		}
-		fields = append(fields, logstorage.Field{
-			Name:  "severity",
-			Value: lr.FormatSeverity(),
-		})
-
-		var streamFields []logstorage.Field
-		if useDefaultStreamFields {
-			streamFields = commonFields
-		}
-		lmp.AddRow(lr.ExtractTimestampNano(), fields, streamFields)
-	}
-	return fields
-}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -1,127 +0,0 @@
-package opentelemetry
-
-import (
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func TestPushProtoOk(t *testing.T) {
-	f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
-		t.Helper()
-		lr := pb.ExportLogsServiceRequest{
-			ResourceLogs: src,
-		}
-
-		pData := lr.MarshalProtobuf(nil)
-		tlp := &insertutils.TestLogMessageProcessor{}
-		if err := pushProtobufRequest(pData, tlp, false); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
-			t.Fatal(err)
-		}
-	}
-	// single line without resource attributes
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234},
-		`{"_msg":"log-line-message","severity":"Trace"}`,
-	)
-	// multi-line with resource attributes
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
-	)
-
-	// multi-scope with resource attributes and multi-line
-	f([]pb.ResourceLogs{
-		{
-			Resource: pb.Resource{
-				Attributes: []*pb.KeyValue{
-					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
-					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
-					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
-						Values: []*pb.KeyValue{
-							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
-							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
-						},
-					}}},
-				},
-			},
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
-					},
-				},
-			},
-		},
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
-						{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
-					},
-				},
-				{
-					LogRecords: []pb.LogRecord{
-						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
-						{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1235, 2345, 2346, 2347, 2348},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
-{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
-{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
-	)
-}
-
-func ptrTo[T any](s T) *T {
-	return &s
-}

app/vlinsert/opentelemetry/opentelemetry_timing_test.go

@@ -1,78 +0,0 @@
-package opentelemetry
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-)
-
-func BenchmarkParseProtobufRequest(b *testing.B) {
-	for _, scopes := range []int{1, 2} {
-		for _, rows := range []int{100, 1000} {
-			for _, attributes := range []int{5, 10} {
-				b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
-					benchmarkParseProtobufRequest(b, scopes, rows, attributes)
-				})
-			}
-		}
-	}
-}
-
-func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
-	blp := &insertutils.BenchmarkLogMessageProcessor{}
-	b.ReportAllocs()
-	b.SetBytes(int64(streams * rows))
-	b.RunParallel(func(pb *testing.PB) {
-		body := getProtobufBody(streams, rows, labels)
-		for pb.Next() {
-			if err := pushProtobufRequest(body, blp, false); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}
-
-func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
-	msg := "12345678910"
-
-	attrValues := []*pb.AnyValue{
-		{StringValue: ptrTo("string-attribute")},
-		{IntValue: ptrTo[int64](12345)},
-		{DoubleValue: ptrTo(3.14)},
-	}
-	attrs := make([]*pb.KeyValue, attributesCount)
-	for j := 0; j < attributesCount; j++ {
-		attrs[j] = &pb.KeyValue{
-			Key:   fmt.Sprintf("key-%d", j),
-			Value: attrValues[j%3],
-		}
-	}
-	entries := make([]pb.LogRecord, rowsCount)
-	for j := 0; j < rowsCount; j++ {
-		entries[j] = pb.LogRecord{
-			TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
-		}
-	}
-	scopes := make([]pb.ScopeLogs, scopesCount)
-
-	for j := 0; j < scopesCount; j++ {
-		scopes[j] = pb.ScopeLogs{
-			LogRecords: entries,
-		}
-	}
-
-	pr := pb.ExportLogsServiceRequest{
-		ResourceLogs: []pb.ResourceLogs{
-			{
-				Resource: pb.Resource{
-					Attributes: attrs,
-				},
-				ScopeLogs: scopes,
-			},
-		},
-	}
-
-	return pr.MarshalProtobuf(nil)
-}

app/vlinsert/syslog/syslog.go

@@ -3,13 +3,11 @@ package syslog
 import (
 	"bufio"
 	"crypto/tls"
-	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"net"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -37,25 +35,10 @@ var (
 	syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+
 		"For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
-	streamFieldsTCP = flagutil.NewArrayString("syslog.streamFields.tcp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-	streamFieldsUDP = flagutil.NewArrayString("syslog.streamFields.udp", "Fields to use as log stream labels for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#stream-fields`)
-
-	ignoreFieldsTCP = flagutil.NewArrayString("syslog.ignoreFields.tcp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-	ignoreFieldsUDP = flagutil.NewArrayString("syslog.ignoreFields.udp", "Fields to ignore at logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#dropping-fields`)
-
-	extraFieldsTCP = flagutil.NewArrayString("syslog.extraFields.tcp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-	extraFieldsUDP = flagutil.NewArrayString("syslog.extraFields.udp", "Fields to add to logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		`See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#adding-extra-fields`)
-
-	tenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
-	tenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
-		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multitenancy")
+	syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
+	syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
 
 	listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+
 		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/")
@@ -167,7 +150,7 @@ func runUDPListener(addr string, argIdx int) {
 		logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err)
 	}
 
-	tenantIDStr := tenantIDUDP.GetOptionalArg(argIdx)
+	tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx)
 	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
 	if err != nil {
 		logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err)
@@ -178,27 +161,9 @@ func runUDPListener(addr string, argIdx int) {
 
 	useLocalTimestamp := useLocalTimestampUDP.GetOptionalArg(argIdx)
 
-	streamFieldsStr := streamFieldsUDP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.udp=%q for -syslog.listenAddr.udp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsUDP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.udp=%q for -syslog.listenAddr.udp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsUDP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.udp=%q for -syslog.listenAddr.udp=%q: %s", extraFieldsStr, addr, err)
-	}
-
 	doneCh := make(chan struct{})
 	go func() {
-		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
+		serveUDP(ln, tenantID, compressMethod, useLocalTimestamp)
 		close(doneCh)
 	}()
 
@@ -228,7 +193,7 @@ func runTCPListener(addr string, argIdx int) {
 		logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err)
 	}
 
-	tenantIDStr := tenantIDTCP.GetOptionalArg(argIdx)
+	tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx)
 	tenantID, err := logstorage.ParseTenantID(tenantIDStr)
 	if err != nil {
 		logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err)
@@ -239,27 +204,9 @@ func runTCPListener(addr string, argIdx int) {
 
 	useLocalTimestamp := useLocalTimestampTCP.GetOptionalArg(argIdx)
 
-	streamFieldsStr := streamFieldsTCP.GetOptionalArg(argIdx)
-	streamFields, err := parseFieldsList(streamFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.streamFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", streamFieldsStr, addr, err)
-	}
-
-	ignoreFieldsStr := ignoreFieldsTCP.GetOptionalArg(argIdx)
-	ignoreFields, err := parseFieldsList(ignoreFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.ignoreFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", ignoreFieldsStr, addr, err)
-	}
-
-	extraFieldsStr := extraFieldsTCP.GetOptionalArg(argIdx)
-	extraFields, err := parseExtraFields(extraFieldsStr)
-	if err != nil {
-		logger.Fatalf("cannot parse -syslog.extraFields.tcp=%q for -syslog.listenAddr.tcp=%q: %s", extraFieldsStr, addr, err)
-	}
-
 	doneCh := make(chan struct{})
 	go func() {
-		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp, streamFields, ignoreFields, extraFields)
+		serveTCP(ln, tenantID, compressMethod, useLocalTimestamp)
 		close(doneCh)
 	}()
 
@@ -281,7 +228,7 @@ func checkCompressMethod(compressMethod, addr, protocol string) {
 	}
 }
 
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
 	gomaxprocs := cgroup.AvailableCPUs()
 	var wg sync.WaitGroup
 	localAddr := ln.LocalAddr()
@@ -289,7 +236,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
 			var bb bytesutil.ByteBuffer
 			bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024)
 			for {
@@ -314,7 +261,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 				}
 				bb.B = bb.B[:n]
 				udpRequestsTotal.Inc()
-				if err := processStream("udp", bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
+				if err := processStream(bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
 					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
 				}
 			}
@@ -323,7 +270,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 	wg.Wait()
 }
 
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool) {
 	var cm ingestserver.ConnsMap
 	cm.Init("syslog")
 
@@ -353,8 +300,8 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
 
 		wg.Add(1)
 		go func() {
-			cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
-			if err := processStream("tcp", c, compressMethod, useLocalTimestamp, cp); err != nil {
+			cp := insertutils.GetCommonParamsForSyslog(tenantID)
+			if err := processStream(c, compressMethod, useLocalTimestamp, cp); err != nil {
 				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
 			}
 
@@ -369,12 +316,12 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
 }
 
 // processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(protocol string, r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
+func processStream(r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
 	if err := vlstorage.CanWriteData(); err != nil {
 		return err
 	}
 
-	lmp := cp.NewLogMessageProcessor("syslog_" + protocol)
+	lmp := cp.NewLogMessageProcessor()
 	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
 	lmp.MustClose()
 
@@ -436,6 +383,7 @@ func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertut
 			return fmt.Errorf("cannot read line #%d: %s", n, err)
 		}
 		n++
+		rowsIngestedTotal.Inc()
 	}
 	return slr.Error()
 }
@@ -566,50 +514,18 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 		}
 		ts = nsecs
 	}
-	logstorage.RenameField(p.Fields, msgFields, "_msg")
-	lmp.AddRow(ts, p.Fields, nil)
+	logstorage.RenameField(p.Fields, "message", "_msg")
+	lmp.AddRow(ts, p.Fields)
 	logstorage.PutSyslogParser(p)
 
 	return nil
 }
 
-var msgFields = []string{"message"}
-
 var (
+	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="syslog"}`)
+
 	errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`)
 
 	udpRequestsTotal = metrics.NewCounter(`vl_udp_reqests_total{type="syslog"}`)
 	udpErrorsTotal   = metrics.NewCounter(`vl_udp_errors_total{type="syslog"}`)
 )
-
-func parseFieldsList(s string) ([]string, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var a []string
-	err := json.Unmarshal([]byte(s), &a)
-	return a, err
-}
-
-func parseExtraFields(s string) ([]logstorage.Field, error) {
-	if s == "" {
-		return nil, nil
-	}
-
-	var m map[string]string
-	if err := json.Unmarshal([]byte(s), &m); err != nil {
-		return nil, err
-	}
-	fields := make([]logstorage.Field, 0, len(m))
-	for k, v := range m {
-		fields = append(fields, logstorage.Field{
-			Name:  k,
-			Value: v,
-		})
-	}
-	sort.Slice(fields, func(i, j int) bool {
-		return fields[i].Name < fields[j].Name
-	})
-	return fields, nil
-}

app/vlinsert/syslog/syslog_test.go

@@ -75,7 +75,7 @@ func TestSyslogLineReader_Failure(t *testing.T) {
 }
 
 func TestProcessStreamInternal_Success(t *testing.T) {
-	f := func(data string, currentYear int, timestampsExpected []int64, resultExpected string) {
+	f := func(data string, currentYear, rowsExpected int, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
 		MustInit()
@@ -89,7 +89,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 		if err := processStreamInternal(r, "", false, tlp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
-		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+		if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -99,11 +99,12 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 48 <165>Jun  4 12:08:33 abcd systemd[345]: abc defg<123>1 2023-06-03T17:42:12.345Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.
 `
 	currentYear := 2023
+	rowsExpected := 3
 	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
-	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
-	f(data, currentYear, timestampsExpected, resultExpected)
+	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
 }
 
 func TestProcessStreamInternal_Failure(t *testing.T) {

app/vlogscli/Makefile

@@ -1,109 +0,0 @@
-# All these commands must run from repository root.
-
-vlogscli:
-	APP_NAME=vlogscli $(MAKE) app-local
-
-vlogscli-race:
-	APP_NAME=vlogscli RACE=-race $(MAKE) app-local
-
-vlogscli-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker
-
-vlogscli-pure-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-pure
-
-vlogscli-linux-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-amd64
-
-vlogscli-linux-arm-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm
-
-vlogscli-linux-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-arm64
-
-vlogscli-linux-ppc64le-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-ppc64le
-
-vlogscli-linux-386-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-linux-386
-
-vlogscli-darwin-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-amd64
-
-vlogscli-darwin-arm64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-darwin-arm64
-
-vlogscli-freebsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-freebsd-amd64
-
-vlogscli-openbsd-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-openbsd-amd64
-
-vlogscli-windows-amd64-prod:
-	APP_NAME=vlogscli $(MAKE) app-via-docker-windows-amd64
-
-package-vlogscli:
-	APP_NAME=vlogscli $(MAKE) package-via-docker
-
-package-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-pure
-
-package-vlogscli-amd64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-amd64
-
-package-vlogscli-arm:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm
-
-package-vlogscli-arm64:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-arm64
-
-package-vlogscli-ppc64le:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-ppc64le
-
-package-vlogscli-386:
-	APP_NAME=vlogscli $(MAKE) package-via-docker-386
-
-publish-vlogscli:
-	APP_NAME=vlogscli $(MAKE) publish-via-docker
-
-vlogscli-linux-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-ppc64le:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-s390x:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-loong64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
-vlogscli-linux-386:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-darwin-arm64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 $(MAKE) app-local-goos-goarch
-
-vlogscli-freebsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-openbsd-amd64:
-	APP_NAME=vlogscli CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 $(MAKE) app-local-goos-goarch
-
-vlogscli-windows-amd64:
-	GOARCH=amd64 APP_NAME=vlogscli $(MAKE) app-local-windows-goarch
-
-vlogscli-pure:
-	APP_NAME=vlogscli $(MAKE) app-local-pure
-
-run-vlogscli:
-	APP_NAME=vlogscli $(MAKE) run-via-docker

app/vlogscli/README.md

@@ -1,5 +0,0 @@
-# vlogscli
-
-Command-line utility for querying [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
-
-See [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/).

app/vlogscli/deployment/Dockerfile

@@ -1,6 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-ENTRYPOINT ["/vlogscli-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vlogscli-prod

app/vlogscli/json_prettifier.go

@@ -1,245 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-type outputMode int
-
-const (
-	outputModeJSONMultiline  = outputMode(0)
-	outputModeJSONSingleline = outputMode(1)
-	outputModeLogfmt         = outputMode(2)
-	outputModeCompact        = outputMode(3)
-)
-
-func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
-	switch outputMode {
-	case outputModeJSONMultiline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, true)
-		}
-	case outputModeJSONSingleline:
-		return func(w io.Writer, fields []logstorage.Field) error {
-			return writeJSONObject(w, fields, false)
-		}
-	case outputModeLogfmt:
-		return writeLogfmtObject
-	case outputModeCompact:
-		return writeCompactObject
-	default:
-		panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
-	}
-}
-
-type jsonPrettifier struct {
-	r         io.ReadCloser
-	formatter func(w io.Writer, fields []logstorage.Field) error
-
-	d *json.Decoder
-
-	pr *io.PipeReader
-	pw *io.PipeWriter
-	bw *bufio.Writer
-
-	wg sync.WaitGroup
-}
-
-func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
-	d := json.NewDecoder(r)
-	pr, pw := io.Pipe()
-	bw := bufio.NewWriter(pw)
-
-	formatter := getOutputFormatter(outputMode)
-
-	jp := &jsonPrettifier{
-		r:         r,
-		formatter: formatter,
-
-		d: d,
-
-		pr: pr,
-		pw: pw,
-		bw: bw,
-	}
-
-	jp.wg.Add(1)
-	go func() {
-		defer jp.wg.Done()
-		err := jp.prettifyJSONLines()
-		jp.closePipesWithError(err)
-	}()
-
-	return jp
-}
-
-func (jp *jsonPrettifier) closePipesWithError(err error) {
-	_ = jp.pr.CloseWithError(err)
-	_ = jp.pw.CloseWithError(err)
-}
-
-func (jp *jsonPrettifier) prettifyJSONLines() error {
-	for jp.d.More() {
-		fields, err := readNextJSONObject(jp.d)
-		if err != nil {
-			return err
-		}
-		sort.Slice(fields, func(i, j int) bool {
-			return fields[i].Name < fields[j].Name
-		})
-		if err := jp.formatter(jp.bw, fields); err != nil {
-			return err
-		}
-
-		// Flush bw after every output line in order to show results as soon as they appear.
-		if err := jp.bw.Flush(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (jp *jsonPrettifier) Close() error {
-	jp.closePipesWithError(io.ErrUnexpectedEOF)
-	err := jp.r.Close()
-	jp.wg.Wait()
-	return err
-}
-
-func (jp *jsonPrettifier) Read(p []byte) (int, error) {
-	return jp.pr.Read(p)
-}
-
-func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
-	t, err := d.Token()
-	if err != nil {
-		return nil, fmt.Errorf("cannot read '{': %w", err)
-	}
-	delim, ok := t.(json.Delim)
-	if !ok || delim.String() != "{" {
-		return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
-	}
-
-	var fields []logstorage.Field
-	for {
-		// Read object key
-		t, err := d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object key or closing brace: %w", err)
-		}
-		delim, ok := t.(json.Delim)
-		if ok {
-			if delim.String() == "}" {
-				return fields, nil
-			}
-			return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
-		}
-		key, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for object key: %v; want string or '}'", t)
-		}
-
-		// read object value
-		t, err = d.Token()
-		if err != nil {
-			return nil, fmt.Errorf("cannot read JSON object value: %w", err)
-		}
-		value, ok := t.(string)
-		if !ok {
-			return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
-		}
-
-		fields = append(fields, logstorage.Field{
-			Name:  key,
-			Value: value,
-		})
-	}
-}
-
-func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
-	data := logstorage.MarshalFieldsToLogfmt(nil, fields)
-	_, err := fmt.Fprintf(w, "%s\n", data)
-	return err
-}
-
-func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
-	if len(fields) == 1 {
-		// Just write field value as is without name
-		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
-		return err
-	}
-	if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
-		// Write _time\tfieldValue as is
-		if fields[0].Name == "_time" {
-			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)
-			return err
-		}
-		_, err := fmt.Fprintf(w, "%s\t%s\n", fields[1].Value, fields[0].Value)
-		return err
-	}
-
-	// Fall back to logfmt
-	return writeLogfmtObject(w, fields)
-}
-
-func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
-	if len(fields) == 0 {
-		fmt.Fprintf(w, "{}\n")
-		return nil
-	}
-
-	fmt.Fprintf(w, "{")
-	writeNewlineIfNeeded(w, isMultiline)
-	if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
-		return err
-	}
-	for _, f := range fields[1:] {
-		fmt.Fprintf(w, ",")
-		writeNewlineIfNeeded(w, isMultiline)
-		if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
-			return err
-		}
-	}
-	writeNewlineIfNeeded(w, isMultiline)
-	fmt.Fprintf(w, "}\n")
-	return nil
-}
-
-func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
-	if isMultiline {
-		fmt.Fprintf(w, "\n")
-	}
-}
-
-func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
-	key := getJSONString(f.Name)
-	value := getJSONString(f.Value)
-	if isMultiline {
-		_, err := fmt.Fprintf(w, "  %s: %s", key, value)
-		return err
-	}
-	_, err := fmt.Fprintf(w, "%s:%s", key, value)
-	return err
-}
-
-func getJSONString(s string) string {
-	data, err := json.Marshal(s)
-	if err != nil {
-		panic(fmt.Errorf("unexpected error when marshaling string to JSON: %w", err))
-	}
-	return jsonHTMLReplacer.Replace(string(data))
-}
-
-var jsonHTMLReplacer = strings.NewReplacer(
-	`\u003c`, "\u003c",
-	`\u003e`, "\u003e",
-	`\u0026`, "\u0026",
-)

app/vlogscli/less_wrapper.go

@@ -1,120 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-
-	"github.com/mattn/go-isatty"
-)
-
-func isTerminal() bool {
-	return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
-}
-
-func readWithLess(r io.Reader, wrapLongLines bool) error {
-	if !isTerminal() {
-		// Just write everything to stdout if no terminal is available.
-		_, err := io.Copy(os.Stdout, r)
-		if err != nil && !isErrPipe(err) {
-			return fmt.Errorf("error when forwarding data to stdout: %w", err)
-		}
-		if err := os.Stdout.Sync(); err != nil {
-			return fmt.Errorf("cannot sync data to stdout: %w", err)
-		}
-		return nil
-	}
-
-	pr, pw, err := os.Pipe()
-	if err != nil {
-		return fmt.Errorf("cannot create pipe: %w", err)
-	}
-	defer func() {
-		_ = pr.Close()
-		_ = pw.Close()
-	}()
-
-	// Ignore Ctrl+C in the current process, so 'less' could handle it properly
-	cancel := ignoreSignals(os.Interrupt)
-	defer cancel()
-
-	// Start 'less' process
-	path, err := exec.LookPath("less")
-	if err != nil {
-		return fmt.Errorf("cannot find 'less' command: %w", err)
-	}
-	opts := []string{"less", "-F", "-X"}
-	if !wrapLongLines {
-		opts = append(opts, "-S")
-	}
-	p, err := os.StartProcess(path, opts, &os.ProcAttr{
-		Env:   append(os.Environ(), "LESSCHARSET=utf-8"),
-		Files: []*os.File{pr, os.Stdout, os.Stderr},
-	})
-	if err != nil {
-		return fmt.Errorf("cannot start 'less' process: %w", err)
-	}
-
-	// Close pr after 'less' finishes in a parallel goroutine
-	// in order to unblock forwarding data to stopped 'less' below.
-	waitch := make(chan *os.ProcessState)
-	go func() {
-		// Wait for 'less' process to finish.
-		ps, err := p.Wait()
-		if err != nil {
-			fatalf("unexpected error when waiting for 'less' process: %w", err)
-		}
-		_ = pr.Close()
-		waitch <- ps
-	}()
-
-	// Forward data from r to 'less'
-	_, err = io.Copy(pw, r)
-	_ = pw.Sync()
-	_ = pw.Close()
-
-	// Wait until 'less' finished
-	ps := <-waitch
-
-	// Verify 'less' status.
-	if !ps.Success() {
-		return fmt.Errorf("'less' finished with unexpected code %d", ps.ExitCode())
-	}
-
-	if err != nil && !isErrPipe(err) {
-		return fmt.Errorf("error when forwarding data to 'less': %w", err)
-	}
-
-	return nil
-}
-
-func isErrPipe(err error) bool {
-	return errors.Is(err, syscall.EPIPE) || errors.Is(err, io.ErrClosedPipe)
-}
-
-func ignoreSignals(sigs ...os.Signal) func() {
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, sigs...)
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for {
-			_, ok := <-ch
-			if !ok {
-				return
-			}
-		}
-	}()
-	return func() {
-		signal.Stop(ch)
-		close(ch)
-		wg.Wait()
-	}
-}

app/vlogscli/main.go

@@ -1,436 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"strconv"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/ergochat/readline"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-var (
-	datasourceURL = flag.String("datasource.url", "http://localhost:9428/select/logsql/query", "URL for querying VictoriaLogs; "+
-		"see https://docs.victoriametrics.com/victorialogs/querying/#querying-logs . See also -tail.url")
-	tailURL = flag.String("tail.url", "", "URL for live tailing queries to VictoriaLogs; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing ."+
-		"The url is automatically detected from -datasource.url by replacing /query with /tail at the end if -tail.url is empty")
-	historyFile = flag.String("historyFile", "vlogscli-history", "Path to file with command history")
-	header      = flagutil.NewArrayString("header", "Optional header to pass in request -datasource.url in the form 'HeaderName: value'")
-	accountID   = flag.Int("accountID", 0, "Account ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-	projectID   = flag.Int("projectID", 0, "Project ID to query; see https://docs.victoriametrics.com/victorialogs/#multitenancy")
-)
-
-const (
-	firstLinePrompt = ";> "
-	nextLinePrompt  = ""
-)
-
-func main() {
-	// Write flags and help message to stdout, since it is easier to grep or pipe.
-	flag.CommandLine.SetOutput(os.Stdout)
-	flag.Usage = usage
-	envflag.Parse()
-	buildinfo.Init()
-	logger.InitNoLogFlags()
-
-	hes, err := parseHeaders(*header)
-	if err != nil {
-		fatalf("cannot parse -header command-line flag: %s", err)
-	}
-	headers = hes
-
-	incompleteLine := ""
-	cfg := &readline.Config{
-		Prompt:                 firstLinePrompt,
-		DisableAutoSaveHistory: true,
-		Listener: func(line []rune, pos int, _ rune) ([]rune, int, bool) {
-			incompleteLine = string(line)
-			return line, pos, false
-		},
-	}
-	rl, err := readline.NewFromConfig(cfg)
-	if err != nil {
-		fatalf("cannot initialize readline: %s", err)
-	}
-
-	fmt.Fprintf(rl, "sending queries to -datasource.url=%s\n", *datasourceURL)
-	fmt.Fprintf(rl, `type ? and press enter to see available commands`+"\n")
-	runReadlineLoop(rl, &incompleteLine)
-
-	if err := rl.Close(); err != nil {
-		fatalf("cannot close readline: %s", err)
-	}
-
-}
-
-func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
-	historyLines, err := loadFromHistory(*historyFile)
-	if err != nil {
-		fatalf("cannot load query history: %s", err)
-	}
-	for _, line := range historyLines {
-		if err := rl.SaveToHistory(line); err != nil {
-			fatalf("cannot initialize query history: %s", err)
-		}
-	}
-
-	outputMode := outputModeJSONMultiline
-	wrapLongLines := false
-	s := ""
-	for {
-		line, err := rl.ReadLine()
-		if err != nil {
-			switch err {
-			case io.EOF:
-				if s != "" {
-					// This is non-interactive query execution.
-					executeQuery(context.Background(), rl, s, outputMode, wrapLongLines)
-				}
-				return
-			case readline.ErrInterrupt:
-				if s == "" && *incompleteLine == "" {
-					fmt.Fprintf(rl, "interrupted\n")
-					os.Exit(128 + int(syscall.SIGINT))
-				}
-				// Default value for Ctrl+C - clear the prompt and store the incompletely entered line into history
-				s += *incompleteLine
-				historyLines = pushToHistory(rl, historyLines, s)
-				s = ""
-				rl.SetPrompt(firstLinePrompt)
-				continue
-			default:
-				fatalf("unexpected error in readline: %s", err)
-			}
-		}
-
-		s += line
-		if s == "" {
-			// Skip empty lines
-			continue
-		}
-
-		if isQuitCommand(s) {
-			fmt.Fprintf(rl, "bye!\n")
-			_ = pushToHistory(rl, historyLines, s)
-			return
-		}
-		if isHelpCommand(s) {
-			printCommandsHelp(rl)
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\s` {
-			fmt.Fprintf(rl, "singleline json output mode\n")
-			outputMode = outputModeJSONSingleline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\m` {
-			fmt.Fprintf(rl, "multiline json output mode\n")
-			outputMode = outputModeJSONMultiline
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\c` {
-			fmt.Fprintf(rl, "compact output mode\n")
-			outputMode = outputModeCompact
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\logfmt` {
-			fmt.Fprintf(rl, "logfmt output mode\n")
-			outputMode = outputModeLogfmt
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if s == `\wrap_long_lines` {
-			if wrapLongLines {
-				wrapLongLines = false
-				fmt.Fprintf(rl, "wrapping of long lines is disabled\n")
-			} else {
-				wrapLongLines = true
-				fmt.Fprintf(rl, "wrapping of long lines is enabled\n")
-			}
-			historyLines = pushToHistory(rl, historyLines, s)
-			s = ""
-			continue
-		}
-		if line != "" && !strings.HasSuffix(line, ";") {
-			// Assume the query is incomplete and allow the user finishing the query on the next line
-			s += "\n"
-			rl.SetPrompt(nextLinePrompt)
-			continue
-		}
-
-		// Execute the query
-		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		executeQuery(ctx, rl, s, outputMode, wrapLongLines)
-		cancel()
-
-		historyLines = pushToHistory(rl, historyLines, s)
-		s = ""
-		rl.SetPrompt(firstLinePrompt)
-	}
-}
-
-func pushToHistory(rl *readline.Instance, historyLines []string, s string) []string {
-	s = strings.TrimSpace(s)
-	if len(historyLines) == 0 || historyLines[len(historyLines)-1] != s {
-		historyLines = append(historyLines, s)
-		if len(historyLines) > 500 {
-			historyLines = historyLines[len(historyLines)-500:]
-		}
-		if err := saveToHistory(*historyFile, historyLines); err != nil {
-			fatalf("cannot save query history: %s", err)
-		}
-	}
-	if err := rl.SaveToHistory(s); err != nil {
-		fatalf("cannot update query history: %s", err)
-	}
-	return historyLines
-}
-
-func loadFromHistory(filePath string) ([]string, error) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	linesQuoted := strings.Split(string(data), "\n")
-	lines := make([]string, 0, len(linesQuoted))
-	i := 0
-	for _, lineQuoted := range linesQuoted {
-		i++
-		if lineQuoted == "" {
-			continue
-		}
-		line, err := strconv.Unquote(lineQuoted)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse line #%d at %s: %w; line: [%s]", i, filePath, err, line)
-		}
-		lines = append(lines, line)
-	}
-	return lines, nil
-}
-
-func saveToHistory(filePath string, lines []string) error {
-	linesQuoted := make([]string, len(lines))
-	for i, line := range lines {
-		lineQuoted := strconv.Quote(line)
-		linesQuoted[i] = lineQuoted
-	}
-	data := strings.Join(linesQuoted, "\n")
-	return os.WriteFile(filePath, []byte(data), 0600)
-}
-
-func isQuitCommand(s string) bool {
-	switch s {
-	case `\q`, "q", "quit", "exit":
-		return true
-	default:
-		return false
-	}
-}
-
-func isHelpCommand(s string) bool {
-	switch s {
-	case `\h`, "h", "help", "?":
-		return true
-	default:
-		return false
-	}
-}
-
-func printCommandsHelp(w io.Writer) {
-	fmt.Fprintf(w, "%s", `Available commands:
-\q - quit
-\h - show this help
-\s - singleline json output mode
-\m - multiline json output mode
-\c - compact output
-\logfmt - logfmt output mode
-\wrap_long_lines - toggles wrapping long lines
-\tail <query> - live tail <query> results
-
-See https://docs.victoriametrics.com/victorialogs/querying/vlogscli/ for more details
-`)
-}
-
-func executeQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, wrapLongLines bool) {
-	if strings.HasPrefix(qStr, `\tail `) {
-		tailQuery(ctx, output, qStr, outputMode)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, *datasourceURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if err := readWithLess(respBody, wrapLongLines); err != nil {
-		fmt.Fprintf(output, "error when reading query response: %s\n", err)
-		return
-	}
-}
-
-func tailQuery(ctx context.Context, output io.Writer, qStr string, outputMode outputMode) {
-	qStr = strings.TrimPrefix(qStr, `\tail `)
-	qURL, err := getTailURL()
-	if err != nil {
-		fmt.Fprintf(output, "%s\n", err)
-		return
-	}
-
-	respBody := getQueryResponse(ctx, output, qStr, outputMode, qURL)
-	if respBody == nil {
-		return
-	}
-	defer func() {
-		_ = respBody.Close()
-	}()
-
-	if _, err := io.Copy(output, respBody); err != nil {
-		if !errors.Is(err, context.Canceled) && !isErrPipe(err) {
-			fmt.Fprintf(output, "error when live tailing query response: %s\n", err)
-		}
-		fmt.Fprintf(output, "\n")
-		return
-	}
-}
-
-func getTailURL() (string, error) {
-	if *tailURL != "" {
-		return *tailURL, nil
-	}
-
-	u, err := url.Parse(*datasourceURL)
-	if err != nil {
-		return "", fmt.Errorf("cannot parse -datasource.url=%q: %w", *datasourceURL, err)
-	}
-	if !strings.HasSuffix(u.Path, "/query") {
-		return "", fmt.Errorf("cannot find /query suffix in -datasource.url=%q", *datasourceURL)
-	}
-	u.Path = u.Path[:len(u.Path)-len("/query")] + "/tail"
-	return u.String(), nil
-}
-
-func getQueryResponse(ctx context.Context, output io.Writer, qStr string, outputMode outputMode, qURL string) io.ReadCloser {
-	// Parse the query and convert it to canonical view.
-	qStr = strings.TrimSuffix(qStr, ";")
-	q, err := logstorage.ParseQuery(qStr)
-	if err != nil {
-		fmt.Fprintf(output, "cannot parse query: %s\n", err)
-		return nil
-	}
-	qStr = q.String()
-	fmt.Fprintf(output, "executing [%s]...", qStr)
-
-	// Prepare HTTP request for qURL
-	args := make(url.Values)
-	args.Set("query", qStr)
-	data := strings.NewReader(args.Encode())
-
-	req, err := http.NewRequestWithContext(ctx, "POST", qURL, data)
-	if err != nil {
-		panic(fmt.Errorf("BUG: cannot prepare request to server: %w", err))
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	for _, h := range headers {
-		req.Header.Set(h.Name, h.Value)
-	}
-	req.Header.Set("AccountID", strconv.Itoa(*accountID))
-	req.Header.Set("ProjectID", strconv.Itoa(*projectID))
-
-	// Execute HTTP request at qURL
-	startTime := time.Now()
-	resp, err := httpClient.Do(req)
-	queryDuration := time.Since(startTime)
-	fmt.Fprintf(output, "; duration: %.3fs\n", queryDuration.Seconds())
-	if err != nil {
-		if errors.Is(err, context.Canceled) {
-			fmt.Fprintf(output, "\n")
-		} else {
-			fmt.Fprintf(output, "cannot execute query: %s\n", err)
-		}
-		return nil
-	}
-
-	// Verify response code
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			body = []byte(fmt.Sprintf("cannot read response body: %s", err))
-		}
-		fmt.Fprintf(output, "unexpected status code: %d; response body:\n%s\n", resp.StatusCode, body)
-		return nil
-	}
-
-	// Prettify the response body
-	jp := newJSONPrettifier(resp.Body, outputMode)
-
-	return jp
-}
-
-var httpClient = &http.Client{}
-
-var headers []headerEntry
-
-type headerEntry struct {
-	Name  string
-	Value string
-}
-
-func parseHeaders(a []string) ([]headerEntry, error) {
-	hes := make([]headerEntry, len(a))
-	for i, s := range a {
-		a := strings.SplitN(s, ":", 2)
-		if len(a) != 2 {
-			return nil, fmt.Errorf("cannot parse header=%q; it must contain at least one ':'; for example, 'Cookie: foo'", s)
-		}
-		hes[i] = headerEntry{
-			Name:  strings.TrimSpace(a[0]),
-			Value: strings.TrimSpace(a[1]),
-		}
-	}
-	return hes, nil
-}
-
-func fatalf(format string, args ...any) {
-	fmt.Fprintf(os.Stderr, format+"\n", args...)
-	os.Exit(1)
-}
-
-func usage() {
-	const s = `
-vlogscli is a command-line tool for querying VictoriaLogs.
-
-See the docs at https://docs.victoriametrics.com/victorialogs/querying/vlogscli/
-`
-	flagutil.Usage(s)
-}

app/vlogscli/multiarch/Dockerfile

@@ -1,11 +0,0 @@
-# See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
-FROM $certs_image AS certs
-RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
-
-FROM $root_image
-COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-ENTRYPOINT ["/vlogscli-prod"]
-ARG TARGETARCH
-COPY vlogscli-linux-${TARGETARCH}-prod ./vlogscli-prod

app/vlogsgenerator/main.go

@@ -45,8 +45,6 @@ var (
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	u64FieldsPerLog = flag.Int("u64FieldsPerLog", 1, "The number of fields with uint64 values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
-	i64FieldsPerLog = flag.Int("i64FieldsPerLog", 1, "The number of fields with int64 values to generate per each log entry; "+
-		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	floatFieldsPerLog = flag.Int("floatFieldsPerLog", 1, "The number of fields with float64 values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	ipFieldsPerLog = flag.Int("ipFieldsPerLog", 1, "The number of fields with IPv4 values to generate per each log entry; "+
@@ -256,9 +254,6 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 		for j := 0; j < *u64FieldsPerLog; j++ {
 			fmt.Fprintf(bw, `,"u64_%d":"%d"`, j, rand.Uint64())
 		}
-		for j := 0; j < *i64FieldsPerLog; j++ {
-			fmt.Fprintf(bw, `,"i64_%d":"%d"`, j, int64(rand.Uint64()))
-		}
 		for j := 0; j < *floatFieldsPerLog; j++ {
 			fmt.Fprintf(bw, `,"float_%d":"%v"`, j, math.Round(10_000*rand.Float64())/1000)
 		}

app/vlselect/logsql/facets_response.qtpl

@@ -1,49 +0,0 @@
-{% import (
-	"slices"
-) %}
-
-{% stripspace %}
-
-{% func FacetsResponse(m map[string][]facetEntry) %}
-{
-	{% code
-		sortedKeys := make([]string, 0, len(m))
-		for k := range m {
-			sortedKeys = append(sortedKeys, k)
-		}
-		slices.Sort(sortedKeys)
-	%}
-	"facets":[
-		{% if len(sortedKeys) > 0 %}
-			{%= facetsLine(m, sortedKeys[0]) %}
-			{% for _, k := range sortedKeys[1:] %}
-				,{%= facetsLine(m, k) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetsLine(m map[string][]facetEntry, k string) %}
-{
-	"field_name":{%q= k %},
-	"values":[
-		{% code	fes := m[k] %}
-		{% if len(fes) > 0 %}
-			{%= facetLine(fes[0]) %}
-			{% for _, fe := range fes[1:] %}
-				,{%= facetLine(fe) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func facetLine(fe facetEntry) %}
-{
-	"field_value":{%q= fe.value %},
-	"hits":{%s= fe.hits %}
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/facets_response.qtpl.go

@@ -1,178 +0,0 @@
-// Code generated by qtc from "facets_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-package logsql
-
-//line app/vlselect/logsql/facets_response.qtpl:1
-import (
-	"slices"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/facets_response.qtpl:7
-func StreamFacetsResponse(qw422016 *qt422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:7
-	qw422016.N().S(`{`)
-//line app/vlselect/logsql/facets_response.qtpl:10
-	sortedKeys := make([]string, 0, len(m))
-	for k := range m {
-		sortedKeys = append(sortedKeys, k)
-	}
-	slices.Sort(sortedKeys)
-
-//line app/vlselect/logsql/facets_response.qtpl:15
-	qw422016.N().S(`"facets":[`)
-//line app/vlselect/logsql/facets_response.qtpl:17
-	if len(sortedKeys) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:18
-		streamfacetsLine(qw422016, m, sortedKeys[0])
-//line app/vlselect/logsql/facets_response.qtpl:19
-		for _, k := range sortedKeys[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:19
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:20
-			streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:21
-		}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	}
-//line app/vlselect/logsql/facets_response.qtpl:22
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func WriteFacetsResponse(qq422016 qtio422016.Writer, m map[string][]facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	StreamFacetsResponse(qw422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:25
-func FacetsResponse(m map[string][]facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:25
-	WriteFacetsResponse(qb422016, m)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:25
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:25
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:27
-func streamfacetsLine(qw422016 *qt422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:27
-	qw422016.N().S(`{"field_name":`)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().Q(k)
-//line app/vlselect/logsql/facets_response.qtpl:29
-	qw422016.N().S(`,"values":[`)
-//line app/vlselect/logsql/facets_response.qtpl:31
-	fes := m[k]
-
-//line app/vlselect/logsql/facets_response.qtpl:32
-	if len(fes) > 0 {
-//line app/vlselect/logsql/facets_response.qtpl:33
-		streamfacetLine(qw422016, fes[0])
-//line app/vlselect/logsql/facets_response.qtpl:34
-		for _, fe := range fes[1:] {
-//line app/vlselect/logsql/facets_response.qtpl:34
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/facets_response.qtpl:35
-			streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:36
-		}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	}
-//line app/vlselect/logsql/facets_response.qtpl:37
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func writefacetsLine(qq422016 qtio422016.Writer, m map[string][]facetEntry, k string) {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	streamfacetsLine(qw422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:40
-func facetsLine(m map[string][]facetEntry, k string) string {
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:40
-	writefacetsLine(qb422016, m, k)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:40
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:40
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:42
-func streamfacetLine(qw422016 *qt422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:42
-	qw422016.N().S(`{"field_value":`)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().Q(fe.value)
-//line app/vlselect/logsql/facets_response.qtpl:44
-	qw422016.N().S(`,"hits":`)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(fe.hits)
-//line app/vlselect/logsql/facets_response.qtpl:45
-	qw422016.N().S(`}`)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func writefacetLine(qq422016 qtio422016.Writer, fe facetEntry) {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	streamfacetLine(qw422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-}
-
-//line app/vlselect/logsql/facets_response.qtpl:47
-func facetLine(fe facetEntry) string {
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/facets_response.qtpl:47
-	writefacetLine(qb422016, fe)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/facets_response.qtpl:47
-	return qs422016
-//line app/vlselect/logsql/facets_response.qtpl:47
-}

app/vlselect/logsql/logsql.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"regexp"
-	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -14,7 +12,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/fastjson"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -25,82 +22,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-// ProcessFacetsRequest handles /select/logsql/facets request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-facets
-func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	limit, err := httputils.GetInt(r, "limit")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	maxValuesPerField, err := httputils.GetInt(r, "max_values_per_field")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	maxValueLen, err := httputils.GetInt(r, "max_value_len")
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	keepConstFields := httputils.GetBool(r, "keep_const_fields")
-
-	q.DropAllPipes()
-	q.AddFacetsPipe(limit, maxValuesPerField, maxValueLen, keepConstFields)
-
-	var mLock sync.Mutex
-	m := make(map[string][]facetEntry)
-	writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
-			return
-		}
-		if len(columns) != 3 {
-			logger.Panicf("BUG: expecting 3 columns; got %d columns", len(columns))
-		}
-
-		fieldNames := columns[0].Values
-		fieldValues := columns[1].Values
-		hits := columns[2].Values
-
-		bb := blockResultPool.Get()
-		for i := range fieldNames {
-			fieldName := strings.Clone(fieldNames[i])
-			fieldValue := strings.Clone(fieldValues[i])
-			hitsStr := strings.Clone(hits[i])
-
-			mLock.Lock()
-			m[fieldName] = append(m[fieldName], facetEntry{
-				value: fieldValue,
-				hits:  hitsStr,
-			})
-			mLock.Unlock()
-		}
-		blockResultPool.Put(bb)
-	}
-
-	// Execute the query
-	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
-		httpserver.Errorf(w, r, "cannot execute query [%s]: %s", q, err)
-		return
-	}
-
-	// Write response
-	w.Header().Set("Content-Type", "application/json")
-	WriteFacetsResponse(w, m)
-}
-
-type facetEntry struct {
-	value string
-	hits  string
-}
-
 // ProcessHitsRequest handles /select/logsql/hits request.
 //
 // See https://docs.victoriametrics.com/victorialogs/querying/#querying-hits-stats
@@ -123,7 +44,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 	if step <= 0 {
 		httpserver.Errorf(w, r, "'step' must be bigger than zero")
-		return
 	}
 
 	// Obtain offset
@@ -151,6 +71,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 
 	// Prepare the query for hits count.
+	q.Optimize()
 	q.DropAllPipes()
 	q.AddCountByTimePipe(int64(step), int64(offset), fields)
 
@@ -281,6 +202,7 @@ func ProcessFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *htt
 	}
 
 	// Obtain field names for the given query
+	q.Optimize()
 	fieldNames, err := vlstorage.GetFieldNames(ctx, tenantIDs, q)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain field names: %s", err)
@@ -320,6 +242,7 @@ func ProcessFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *ht
 	}
 
 	// Obtain unique values for the given field
+	q.Optimize()
 	values, err := vlstorage.GetFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain values for field %q: %s", fieldName, err)
@@ -342,6 +265,7 @@ func ProcessStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter,
 	}
 
 	// Obtain stream field names for the given query
+	q.Optimize()
 	names, err := vlstorage.GetStreamFieldNames(ctx, tenantIDs, q)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain stream field names: %s", err)
@@ -380,6 +304,7 @@ func ProcessStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter,
 	}
 
 	// Obtain stream field values for the given query and the given fieldName
+	q.Optimize()
 	values, err := vlstorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain stream field values: %s", err)
@@ -411,6 +336,7 @@ func ProcessStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http
 	}
 
 	// Obtain streamIDs for the given query
+	q.Optimize()
 	streamIDs, err := vlstorage.GetStreamIDs(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain stream_ids: %s", err)
@@ -442,6 +368,7 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 	}
 
 	// Obtain streams for the given query
+	q.Optimize()
 	streams, err := vlstorage.GetStreams(ctx, tenantIDs, q, uint64(limit))
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain streams: %s", err)
@@ -453,8 +380,6 @@ func ProcessStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.R
 }
 
 // ProcessLiveTailRequest processes live tailing request to /select/logsq/tail
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#live-tailing
 func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	liveTailRequests.Inc()
 	defer liveTailRequests.Dec()
@@ -465,10 +390,9 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 	if !q.CanLiveTail() {
-		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; "+
-			"see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
-		return
+		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
 	}
+	q.Optimize()
 
 	refreshIntervalMsecs, err := httputils.GetDuration(r, "refresh_interval", 1000)
 	if err != nil {
@@ -477,37 +401,25 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 	}
 	refreshInterval := time.Millisecond * time.Duration(refreshIntervalMsecs)
 
-	startOffsetMsecs, err := httputils.GetDuration(r, "start_offset", 5*1000)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	startOffset := startOffsetMsecs * 1e6
-
-	offsetMsecs, err := httputils.GetDuration(r, "offset", 1000)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	offset := offsetMsecs * 1e6
-
 	ctxWithCancel, cancel := context.WithCancel(ctx)
 	tp := newTailProcessor(cancel)
 
 	ticker := time.NewTicker(refreshInterval)
 	defer ticker.Stop()
 
-	end := time.Now().UnixNano() - offset
-	start := end - startOffset
+	end := time.Now().UnixNano()
 	doneCh := ctxWithCancel.Done()
 	flusher, ok := w.(http.Flusher)
 	if !ok {
 		logger.Panicf("BUG: it is expected that http.ResponseWriter (%T) supports http.Flusher interface", w)
 	}
-	qOrig := q
 	for {
-		q = qOrig.CloneWithTimeFilter(end, start, end)
-		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, tp.writeBlock); err != nil {
+		start := end - tailOffsetNsecs
+		end = time.Now().UnixNano()
+
+		qCopy := q.Clone()
+		qCopy.AddTimeFilter(start, end)
+		if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, qCopy, tp.writeBlock); err != nil {
 			httpserver.Errorf(w, r, "cannot execute tail query [%s]: %s", q, err)
 			return
 		}
@@ -525,8 +437,6 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 		case <-doneCh:
 			return
 		case <-ticker.C:
-			start = end - tailOffsetNsecs
-			end = time.Now().UnixNano() - offset
 		}
 	}
 }
@@ -650,208 +560,9 @@ func (tp *tailProcessor) getTailRows() ([][]logstorage.Field, error) {
 	return tailRows, nil
 }
 
-// ProcessStatsQueryRangeRequest handles /select/logsql/stats_query_range request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats
-func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Obtain step
-	stepStr := r.FormValue("step")
-	if stepStr == "" {
-		stepStr = "1d"
-	}
-	step, err := promutils.ParseDuration(stepStr)
-	if err != nil {
-		err = fmt.Errorf("cannot parse 'step' arg: %s", err)
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-	if step <= 0 {
-		err := fmt.Errorf("'step' must be bigger than zero")
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Obtain `by(...)` fields from the last `| stats` pipe in q.
-	// Add `_time:step` to the `by(...)` list.
-	byFields, err := q.GetStatsByFieldsAddGroupingByTime(int64(step))
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	m := make(map[string]*statsSeries)
-	var mLock sync.Mutex
-
-	timestamp := q.GetTimestamp()
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
-		for i := range timestamps {
-			labels := make([]logstorage.Field, 0, len(byFields))
-			for j, c := range columns {
-				if c.Name == "_time" {
-					nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
-					if ok {
-						timestamp = nsec
-						continue
-					}
-				}
-				if slices.Contains(byFields, c.Name) {
-					labels = append(labels, logstorage.Field{
-						Name:  clonedColumnNames[j],
-						Value: strings.Clone(c.Values[i]),
-					})
-				}
-			}
-
-			var dst []byte
-			for j, c := range columns {
-				if !slices.Contains(byFields, c.Name) {
-					name := clonedColumnNames[j]
-					dst = dst[:0]
-					dst = append(dst, name...)
-					dst = logstorage.MarshalFieldsToJSON(dst, labels)
-					key := string(dst)
-					p := statsPoint{
-						Timestamp: timestamp,
-						Value:     strings.Clone(c.Values[i]),
-					}
-
-					mLock.Lock()
-					ss := m[key]
-					if ss == nil {
-						ss = &statsSeries{
-							key:    key,
-							Name:   name,
-							Labels: labels,
-						}
-						m[key] = ss
-					}
-					ss.Points = append(ss.Points, p)
-					mLock.Unlock()
-				}
-			}
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
-		err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Sort the collected stats by time
-	rows := make([]*statsSeries, 0, len(m))
-	for _, ss := range m {
-		points := ss.Points
-		sort.Slice(points, func(i, j int) bool {
-			return points[i].Timestamp < points[j].Timestamp
-		})
-		rows = append(rows, ss)
-	}
-	sort.Slice(rows, func(i, j int) bool {
-		return rows[i].key < rows[j].key
-	})
-
-	w.Header().Set("Content-Type", "application/json")
-	WriteStatsQueryRangeResponse(w, rows)
-}
-
-type statsSeries struct {
-	key string
-
-	Name   string
-	Labels []logstorage.Field
-	Points []statsPoint
-}
-
-type statsPoint struct {
-	Timestamp int64
-	Value     string
-}
-
-// ProcessStatsQueryRequest handles /select/logsql/stats_query request.
-//
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats
-func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	q, tenantIDs, err := parseCommonArgs(r)
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	// Obtain `by(...)` fields from the last `| stats` pipe in q.
-	byFields, err := q.GetStatsByFields()
-	if err != nil {
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	var rows []statsRow
-	var rowsLock sync.Mutex
-
-	timestamp := q.GetTimestamp()
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
-		for i := range timestamps {
-			labels := make([]logstorage.Field, 0, len(byFields))
-			for j, c := range columns {
-				if slices.Contains(byFields, c.Name) {
-					labels = append(labels, logstorage.Field{
-						Name:  clonedColumnNames[j],
-						Value: strings.Clone(c.Values[i]),
-					})
-				}
-			}
-
-			for j, c := range columns {
-				if !slices.Contains(byFields, c.Name) {
-					r := statsRow{
-						Name:      clonedColumnNames[j],
-						Labels:    labels,
-						Timestamp: timestamp,
-						Value:     strings.Clone(c.Values[i]),
-					}
-
-					rowsLock.Lock()
-					rows = append(rows, r)
-					rowsLock.Unlock()
-				}
-			}
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
-		err = fmt.Errorf("cannot execute query [%s]: %s", q, err)
-		httpserver.SendPrometheusError(w, r, err)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	WriteStatsQueryResponse(w, rows)
-}
-
-type statsRow struct {
-	Name      string
-	Labels    []logstorage.Field
-	Timestamp int64
-	Value     string
-}
-
 // ProcessQueryRequest handles /select/logsql/query request.
 //
-// See https://docs.victoriametrics.com/victorialogs/querying/#querying-logs
+// See https://docs.victoriametrics.com/victorialogs/querying/#http-api
 func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	q, tenantIDs, err := parseCommonArgs(r)
 	if err != nil {
@@ -894,6 +605,7 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 
 		q.AddPipeLimit(uint64(limit))
 	}
+	q.Optimize()
 
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		if len(columns) == 0 || len(columns[0].Values) == 0 {
@@ -924,7 +636,7 @@ type row struct {
 func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
-
+	q.Optimize()
 	rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
 	if err != nil {
 		return nil, err
@@ -935,58 +647,32 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 		return rows, nil
 	}
 
-	// Slow path - adjust time range for selecting up to limitUpper rows
+	// Slow path - search for the time range containing up to limitUpper rows.
 	start, end := q.GetFilterTimeRange()
 	d := end/2 - start/2
 	start += d
 
 	qOrig := q
 	for {
-		timestamp := qOrig.GetTimestamp()
-		q = qOrig.CloneWithTimeFilter(timestamp, start, end)
+		q = qOrig.Clone()
+		q.AddTimeFilter(start, end)
 		rows, err := getQueryResultsWithLimit(ctx, tenantIDs, q, limitUpper)
 		if err != nil {
 			return nil, err
 		}
 
-		if d == 0 || start >= end {
-			// The [start ... end] time range equals one nanosecond.
-			// Just return up to limit rows.
-			if len(rows) > limit {
-				rows = rows[:limit]
-			}
-			return rows, nil
-		}
-
-		dLastBit := d & 1
-		d /= 2
-
-		if len(rows) >= limitUpper {
-			// The number of found rows on the [start ... end] time range exceeds limitUpper,
-			// so reduce the time range to [start+d ... end].
-			start += d
-			continue
-		}
-		if len(rows) >= limit {
-			// The number of found rows is in the range [limit ... limitUpper).
-			// This means that found rows contains the needed limit rows with the biggest timestamps.
+		if len(rows) >= limit && len(rows) < limitUpper || d == 0 {
 			rows = getLastNRows(rows, limit)
 			return rows, nil
 		}
 
-		// The number of found rows on [start ... end] time range is below the limit.
-		// This means the time range doesn't cover the needed logs, so it must be extended.
-
-		if len(rows) == 0 {
-			// The [start ... end] time range doesn't contain any rows, so change it to [start-d ... start).
-			end = start - 1
-			start -= d + dLastBit
-			continue
+		lastBit := d & 1
+		d /= 2
+		if len(rows) > limit {
+			start += d
+		} else {
+			start -= d + lastBit
 		}
-
-		// The number of found rows on [start ... end] time range is bigger than 0 but smaller than limit.
-		// Increase the time range to [start-d ... end].
-		start -= d + dLastBit
 	}
 }
 
@@ -1007,25 +693,20 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 	var rows []row
 	var rowsLock sync.Mutex
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
+		rowsLock.Lock()
+		defer rowsLock.Unlock()
 
 		for i, timestamp := range timestamps {
 			fields := make([]logstorage.Field, len(columns))
 			for j := range columns {
 				f := &fields[j]
-				f.Name = clonedColumnNames[j]
+				f.Name = strings.Clone(columns[j].Name)
 				f.Value = strings.Clone(columns[j].Values[i])
 			}
-
-			rowsLock.Lock()
 			rows = append(rows, row{
 				timestamp: timestamp,
 				fields:    fields,
 			})
-			rowsLock.Unlock()
 		}
 
 		if len(rows) >= limit {
@@ -1047,6 +728,13 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	}
 	tenantIDs := []logstorage.TenantID{tenantID}
 
+	// Parse query
+	qStr := r.FormValue("query")
+	q, err := logstorage.ParseQuery(qStr)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
+	}
+
 	// Parse optional start and end args
 	start, okStart, err := getTimeNsec(r, "start")
 	if err != nil {
@@ -1056,35 +744,7 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 	if err != nil {
 		return nil, nil, err
 	}
-
-	// Parse optional time arg
-	timestamp, okTime, err := getTimeNsec(r, "time")
-	if err != nil {
-		return nil, nil, err
-	}
-	if !okTime {
-		// If time arg is missing, then evaluate query either at the end timestamp (if it is set)
-		// or at the current timestamp (if end query arg isn't set)
-		if okEnd {
-			timestamp = end
-		} else {
-			timestamp = time.Now().UnixNano()
-		}
-	}
-
-	// decrease timestamp by one nanosecond in order to avoid capturing logs belonging
-	// to the first nanosecond at the next period of time (month, week, day, hour, etc.)
-	timestamp--
-
-	// Parse query
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
-	if err != nil {
-		return nil, nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
-	}
-
 	if okStart || okEnd {
-		// Add _time:[start, end] filter if start or end args were set.
 		if !okStart {
 			start = math.MinInt64
 		}
@@ -1094,22 +754,6 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
 		q.AddTimeFilter(start, end)
 	}
 
-	// Parse optional extra_filters
-	extraFiltersStr := r.FormValue("extra_filters")
-	extraFilters, err := parseExtraFilters(extraFiltersStr)
-	if err != nil {
-		return nil, nil, err
-	}
-	q.AddExtraFilters(extraFilters)
-
-	// Parse optional extra_stream_filters
-	extraStreamFiltersStr := r.FormValue("extra_stream_filters")
-	extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
-	if err != nil {
-		return nil, nil, err
-	}
-	q.AddExtraFilters(extraStreamFilters)
-
 	return q, tenantIDs, nil
 }
 
@@ -1125,115 +769,3 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
 	}
 	return nsecs, true, nil
 }
-
-func parseExtraFilters(s string) (*logstorage.Filter, error) {
-	if s == "" {
-		return nil, nil
-	}
-	if !strings.HasPrefix(s, `{"`) {
-		return logstorage.ParseFilter(s)
-	}
-
-	// Extra filters in the form {"field":"value",...}.
-	kvs, err := parseExtraFiltersJSON(s)
-	if err != nil {
-		return nil, err
-	}
-
-	filters := make([]string, len(kvs))
-	for i, kv := range kvs {
-		if len(kv.values) == 1 {
-			filters[i] = fmt.Sprintf("%q:=%q", kv.key, kv.values[0])
-		} else {
-			orValues := make([]string, len(kv.values))
-			for j, v := range kv.values {
-				orValues[j] = fmt.Sprintf("%q", v)
-			}
-			filters[i] = fmt.Sprintf("%q:in(%s)", kv.key, strings.Join(orValues, ","))
-		}
-	}
-	s = strings.Join(filters, " ")
-	return logstorage.ParseFilter(s)
-}
-
-func parseExtraStreamFilters(s string) (*logstorage.Filter, error) {
-	if s == "" {
-		return nil, nil
-	}
-	if !strings.HasPrefix(s, `{"`) {
-		return logstorage.ParseFilter(s)
-	}
-
-	// Extra stream filters in the form {"field":"value",...}.
-	kvs, err := parseExtraFiltersJSON(s)
-	if err != nil {
-		return nil, err
-	}
-
-	filters := make([]string, len(kvs))
-	for i, kv := range kvs {
-		if len(kv.values) == 1 {
-			filters[i] = fmt.Sprintf("%q=%q", kv.key, kv.values[0])
-		} else {
-			orValues := make([]string, len(kv.values))
-			for j, v := range kv.values {
-				orValues[j] = regexp.QuoteMeta(v)
-			}
-			filters[i] = fmt.Sprintf("%q=~%q", kv.key, strings.Join(orValues, "|"))
-		}
-	}
-	s = "{" + strings.Join(filters, ",") + "}"
-	return logstorage.ParseFilter(s)
-}
-
-type extraFilter struct {
-	key    string
-	values []string
-}
-
-func parseExtraFiltersJSON(s string) ([]extraFilter, error) {
-	v, err := fastjson.Parse(s)
-	if err != nil {
-		return nil, err
-	}
-	o := v.GetObject()
-
-	var errOuter error
-	var filters []extraFilter
-	o.Visit(func(k []byte, v *fastjson.Value) {
-		if errOuter != nil {
-			return
-		}
-		switch v.Type() {
-		case fastjson.TypeString:
-			filters = append(filters, extraFilter{
-				key:    string(k),
-				values: []string{string(v.GetStringBytes())},
-			})
-		case fastjson.TypeArray:
-			a := v.GetArray()
-			if len(a) == 0 {
-				return
-			}
-			orValues := make([]string, len(a))
-			for i, av := range a {
-				ov, err := av.StringBytes()
-				if err != nil {
-					errOuter = fmt.Errorf("cannot obtain string item at the array for key %q; item: %s", k, av)
-					return
-				}
-				orValues[i] = string(ov)
-			}
-			filters = append(filters, extraFilter{
-				key:    string(k),
-				values: orValues,
-			})
-		default:
-			errOuter = fmt.Errorf("unexpected type of value for key %q: %s; value: %s", k, v.Type(), v)
-		}
-	})
-	if errOuter != nil {
-		return nil, errOuter
-	}
-	return filters, nil
-}

app/vlselect/logsql/logsql_test.go

@@ -1,103 +0,0 @@
-package logsql
-
-import (
-	"testing"
-)
-
-func TestParseExtraFilters_Success(t *testing.T) {
-	f := func(s, resultExpected string) {
-		t.Helper()
-
-		f, err := parseExtraFilters(s)
-		if err != nil {
-			t.Fatalf("unexpected error in parseExtraFilters: %s", err)
-		}
-		result := f.String()
-		if result != resultExpected {
-			t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	f("", "")
-
-	// JSON string
-	f(`{"foo":"bar"}`, `foo:=bar`)
-	f(`{"foo":["bar","baz"]}`, `foo:in(bar,baz)`)
-	f(`{"z":"=b ","c":["d","e,"],"a":[],"_msg":"x"}`, `z:="=b " c:in(d,"e,") =x`)
-
-	// LogsQL filter
-	f(`foobar`, `foobar`)
-	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
-}
-
-func TestParseExtraFilters_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := parseExtraFilters(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// Invalid JSON
-	f(`{"foo"}`)
-	f(`[1,2]`)
-	f(`{"foo":[1]}`)
-
-	// Invliad LogsQL filter
-	f(`foo:(bar`)
-
-	// excess pipe
-	f(`foo | count()`)
-}
-
-func TestParseExtraStreamFilters_Success(t *testing.T) {
-	f := func(s, resultExpected string) {
-		t.Helper()
-
-		f, err := parseExtraStreamFilters(s)
-		if err != nil {
-			t.Fatalf("unexpected error in parseExtraStreamFilters: %s", err)
-		}
-		result := f.String()
-		if result != resultExpected {
-			t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
-	}
-
-	f("", "")
-
-	// JSON string
-	f(`{"foo":"bar"}`, `{foo="bar"}`)
-	f(`{"foo":["bar","baz"]}`, `{foo=~"bar|baz"}`)
-	f(`{"z":"b","c":["d","e|\""],"a":[],"_msg":"x"}`, `{z="b",c=~"d|e\\|\"",_msg="x"}`)
-
-	// LogsQL filter
-	f(`foobar`, `foobar`)
-	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
-}
-
-func TestParseExtraStreamFilters_Failure(t *testing.T) {
-	f := func(s string) {
-		t.Helper()
-
-		_, err := parseExtraStreamFilters(s)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
-
-	// Invalid JSON
-	f(`{"foo"}`)
-	f(`[1,2]`)
-	f(`{"foo":[1]}`)
-
-	// Invliad LogsQL filter
-	f(`foo:(bar`)
-
-	// excess pipe
-	f(`foo | count()`)
-}

app/vlselect/logsql/query_response.qtpl

@@ -6,31 +6,15 @@
 
 // JSONRow creates JSON row from the given fields.
 {% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-	{% code
-		i := 0
-		for i < len(columns) && columns[i].Values[rowIdx] == "" {
-			i++
-		}
-		columns = columns[i:]
-	%}
-	{% if len(columns) == 0 %}
-		{% return %}
-	{% endif %}
-	{
-	{% code	c := &columns[0] %}
+{
+	{% code c := &columns[0] %}
 	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
 	{% code columns = columns[1:] %}
 	{% for colIdx := range columns %}
-		{% code
-			c := &columns[colIdx]
-			v := c.Values[rowIdx]
-		%}
-		{% if v == "" %}
-			{% continue %}
-		{% endif %}
+		{% code c := &columns[colIdx] %}
 		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
 	{% endfor %}
-	}{% newline %}
+}{% newline %}
 {% endfunc %}
 
 // JSONRows prints formatted rows
@@ -39,11 +23,7 @@
 		{% return %}
 	{% endif %}
 	{% for _, fields := range rows %}
-		{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
-		{% if len(fields) == 0 %}
-			{% continue %}
-		{% endif %}
-		{
+	{
 		{% if len(fields) > 0 %}
 			{% code
 				f := fields[0]
@@ -51,13 +31,10 @@
 			%}
 			{%q= f.Name %}:{%q= f.Value %}
 			{% for _, f := range fields %}
-				{% if f.Value == "" %}
-					{% continue %}
-				{% endif %}
 				,{%q= f.Name %}:{%q= f.Value %}
 			{% endfor %}
 		{% endif %}
-		}{% newline %}
+	}{% newline %}
 	{% endfor %}
 {% endfunc %}
 

app/vlselect/logsql/query_response.qtpl.go

@@ -26,176 +26,141 @@ var (
 
 //line app/vlselect/logsql/query_response.qtpl:8
 func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:10
-	i := 0
-	for i < len(columns) && columns[i].Values[rowIdx] == "" {
-		i++
-	}
-	columns = columns[i:]
-
-//line app/vlselect/logsql/query_response.qtpl:16
-	if len(columns) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:17
-		return
-//line app/vlselect/logsql/query_response.qtpl:18
-	}
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:8
 	qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:20
+//line app/vlselect/logsql/query_response.qtpl:10
 	c := &columns[0]
 
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:11
 	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:11
 	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:11
 	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:22
+//line app/vlselect/logsql/query_response.qtpl:12
 	columns = columns[1:]
 
-//line app/vlselect/logsql/query_response.qtpl:23
+//line app/vlselect/logsql/query_response.qtpl:13
 	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:25
+//line app/vlselect/logsql/query_response.qtpl:14
 		c := &columns[colIdx]
-		v := c.Values[rowIdx]
 
-//line app/vlselect/logsql/query_response.qtpl:28
-		if v == "" {
-//line app/vlselect/logsql/query_response.qtpl:29
-			continue
-//line app/vlselect/logsql/query_response.qtpl:30
-		}
-//line app/vlselect/logsql/query_response.qtpl:30
+//line app/vlselect/logsql/query_response.qtpl:14
 		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:31
+//line app/vlselect/logsql/query_response.qtpl:15
 		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:31
+//line app/vlselect/logsql/query_response.qtpl:15
 		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:31
+//line app/vlselect/logsql/query_response.qtpl:15
 		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:16
 	}
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:16
 	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:33
+//line app/vlselect/logsql/query_response.qtpl:17
 	qw422016.N().S(`
 `)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 }
 
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 }
 
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:18
 }
 
 // JSONRows prints formatted rows
 
-//line app/vlselect/logsql/query_response.qtpl:37
+//line app/vlselect/logsql/query_response.qtpl:21
 func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:38
+//line app/vlselect/logsql/query_response.qtpl:22
 	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:23
 		return
-//line app/vlselect/logsql/query_response.qtpl:40
+//line app/vlselect/logsql/query_response.qtpl:24
 	}
-//line app/vlselect/logsql/query_response.qtpl:41
+//line app/vlselect/logsql/query_response.qtpl:25
 	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:42
-		fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
-
-//line app/vlselect/logsql/query_response.qtpl:43
-		if len(fields) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:44
-			continue
-//line app/vlselect/logsql/query_response.qtpl:45
-		}
-//line app/vlselect/logsql/query_response.qtpl:45
+//line app/vlselect/logsql/query_response.qtpl:25
 		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:47
+//line app/vlselect/logsql/query_response.qtpl:27
 		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:49
+//line app/vlselect/logsql/query_response.qtpl:29
 			f := fields[0]
 			fields = fields[1:]
 
-//line app/vlselect/logsql/query_response.qtpl:52
+//line app/vlselect/logsql/query_response.qtpl:32
 			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:52
+//line app/vlselect/logsql/query_response.qtpl:32
 			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:52
+//line app/vlselect/logsql/query_response.qtpl:32
 			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:53
+//line app/vlselect/logsql/query_response.qtpl:33
 			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:54
-				if f.Value == "" {
-//line app/vlselect/logsql/query_response.qtpl:55
-					continue
-//line app/vlselect/logsql/query_response.qtpl:56
-				}
-//line app/vlselect/logsql/query_response.qtpl:56
+//line app/vlselect/logsql/query_response.qtpl:33
 				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:57
+//line app/vlselect/logsql/query_response.qtpl:34
 				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:57
+//line app/vlselect/logsql/query_response.qtpl:34
 				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:57
+//line app/vlselect/logsql/query_response.qtpl:34
 				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:58
+//line app/vlselect/logsql/query_response.qtpl:35
 			}
-//line app/vlselect/logsql/query_response.qtpl:59
+//line app/vlselect/logsql/query_response.qtpl:36
 		}
-//line app/vlselect/logsql/query_response.qtpl:59
+//line app/vlselect/logsql/query_response.qtpl:36
 		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:60
+//line app/vlselect/logsql/query_response.qtpl:37
 		qw422016.N().S(`
 `)
-//line app/vlselect/logsql/query_response.qtpl:61
+//line app/vlselect/logsql/query_response.qtpl:38
 	}
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 }
 
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 }
 
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:62
+//line app/vlselect/logsql/query_response.qtpl:39
 }

app/vlselect/logsql/stats_query_range_response.qtpl

@@ -1,52 +0,0 @@
-{% stripspace %}
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-{% func StatsQueryRangeResponse(rows []*statsSeries) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"matrix",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsSeries(rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsSeries(rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsSeries(ss *statsSeries) %}
-{
-	"metric":{
-		"__name__":{%q= ss.Name %}
-		{% if len(ss.Labels) > 0 %}
-			{% for _, label := range ss.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"values":[
-		{% code points := ss.Points %}
-		{% if len(points) > 0 %}
-			{%= formatStatsPoint(&points[0]) %}
-			{% code points = points[1:] %}
-			{% for i := range points %}
-				,{%= formatStatsPoint(&points[i]) %}
-			{% endfor %}
-		{% endif %}
-	]
-}
-{% endfunc %}
-
-{% func formatStatsPoint(p *statsPoint) %}
-[
-	{%f= float64(p.Timestamp)/1e9 %},
-	{%q= p.Value %}
-]
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_range_response.qtpl.go

@@ -1,188 +0,0 @@
-// Code generated by qtc from "stats_query_range_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryRangeResponse generates response for /select/logsql/stats_query_range
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-func StreamStatsQueryRangeResponse(qw422016 *qt422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"matrix","result":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:11
-		streamformatStatsSeries(qw422016, rows[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:14
-			streamformatStatsSeries(qw422016, rows[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func WriteStatsQueryRangeResponse(qq422016 qtio422016.Writer, rows []*statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	StreamStatsQueryRangeResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-func StatsQueryRangeResponse(rows []*statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	WriteStatsQueryRangeResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-func streamformatStatsSeries(qw422016 *qt422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:25
-	qw422016.N().Q(ss.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:26
-	if len(ss.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-		for _, label := range ss.Labels {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:30
-	qw422016.N().S(`},"values":[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:33
-	points := ss.Points
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:34
-	if len(points) > 0 {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:35
-		streamformatStatsPoint(qw422016, &points[0])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:36
-		points = points[1:]
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-		for i := range points {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:37
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:38
-			streamformatStatsPoint(qw422016, &points[i])
-//line app/vlselect/logsql/stats_query_range_response.qtpl:39
-		}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	}
-//line app/vlselect/logsql/stats_query_range_response.qtpl:40
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func writeformatStatsSeries(qq422016 qtio422016.Writer, ss *statsSeries) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	streamformatStatsSeries(qw422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-func formatStatsSeries(ss *statsSeries) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	writeformatStatsSeries(qb422016, ss)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:43
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-func streamformatStatsPoint(qw422016 *qt422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:45
-	qw422016.N().S(`[`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().F(float64(p.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:47
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().Q(p.Value)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:48
-	qw422016.N().S(`]`)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func writeformatStatsPoint(qq422016 qtio422016.Writer, p *statsPoint) {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	streamformatStatsPoint(qw422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}
-
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-func formatStatsPoint(p *statsPoint) string {
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	writeformatStatsPoint(qb422016, p)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-	return qs422016
-//line app/vlselect/logsql/stats_query_range_response.qtpl:50
-}

app/vlselect/logsql/stats_query_response.qtpl

@@ -1,36 +0,0 @@
-{% stripspace %}
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-{% func StatsQueryResponse(rows []statsRow) %}
-{
-	"status":"success",
-	"data":{
-		"resultType":"vector",
-		"result":[
-			{% if len(rows) > 0 %}
-				{%= formatStatsRow(&rows[0]) %}
-				{% code rows = rows[1:] %}
-				{% for i := range rows %}
-					,{%= formatStatsRow(&rows[i]) %}
-				{% endfor %}
-			{% endif %}
-		]
-	}
-}
-{% endfunc %}
-
-{% func formatStatsRow(r *statsRow) %}
-{
-	"metric":{
-		"__name__":{%q= r.Name %}
-		{% if len(r.Labels) > 0 %}
-			{% for _, label := range r.Labels %}
-				,{%q= label.Name %}:{%q= label.Value %}
-			{% endfor %}
-		{% endif %}
-	},
-	"value":[{%f= float64(r.Timestamp)/1e9 %},{%q= r.Value %}]
-}
-{% endfunc %}
-
-{% endstripspace %}

app/vlselect/logsql/stats_query_response.qtpl.go

@@ -1,133 +0,0 @@
-// Code generated by qtc from "stats_query_response.qtpl". DO NOT EDIT.
-// See https://github.com/valyala/quicktemplate for details.
-
-// StatsQueryResponse generates response for /select/logsql/stats_query
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-package logsql
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-import (
-	qtio422016 "io"
-
-	qt422016 "github.com/valyala/quicktemplate"
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-var (
-	_ = qtio422016.Copy
-	_ = qt422016.AcquireByteBuffer
-)
-
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-func StreamStatsQueryResponse(qw422016 *qt422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:4
-	qw422016.N().S(`{"status":"success","data":{"resultType":"vector","result":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:10
-	if len(rows) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:11
-		streamformatStatsRow(qw422016, &rows[0])
-//line app/vlselect/logsql/stats_query_response.qtpl:12
-		rows = rows[1:]
-
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-		for i := range rows {
-//line app/vlselect/logsql/stats_query_response.qtpl:13
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:14
-			streamformatStatsRow(qw422016, &rows[i])
-//line app/vlselect/logsql/stats_query_response.qtpl:15
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:16
-	qw422016.N().S(`]}}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func WriteStatsQueryResponse(qq422016 qtio422016.Writer, rows []statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	StreamStatsQueryResponse(qw422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-func StatsQueryResponse(rows []statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	WriteStatsQueryResponse(qb422016, rows)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:20
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-func streamformatStatsRow(qw422016 *qt422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:22
-	qw422016.N().S(`{"metric":{"__name__":`)
-//line app/vlselect/logsql/stats_query_response.qtpl:25
-	qw422016.N().Q(r.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:26
-	if len(r.Labels) > 0 {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-		for _, label := range r.Labels {
-//line app/vlselect/logsql/stats_query_response.qtpl:27
-			qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Name)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().S(`:`)
-//line app/vlselect/logsql/stats_query_response.qtpl:28
-			qw422016.N().Q(label.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:29
-		}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	}
-//line app/vlselect/logsql/stats_query_response.qtpl:30
-	qw422016.N().S(`},"value":[`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().F(float64(r.Timestamp) / 1e9)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`,`)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().Q(r.Value)
-//line app/vlselect/logsql/stats_query_response.qtpl:32
-	qw422016.N().S(`]}`)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func writeformatStatsRow(qq422016 qtio422016.Writer, r *statsRow) {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	streamformatStatsRow(qw422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}
-
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-func formatStatsRow(r *statsRow) string {
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	writeformatStatsRow(qb422016, r)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-	return qs422016
-//line app/vlselect/logsql/stats_query_response.qtpl:34
-}

app/vlselect/main.go

@@ -23,7 +23,7 @@ var (
 		"See also -search.maxQueueDuration")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the search request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden to a smaller value on a per-query basis via 'timeout' query arg")
+	maxQueryDuration = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden on a per-query basis via 'timeout' query arg")
 )
 
 func getDefaultMaxConcurrentRequests() int {
@@ -177,10 +177,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
 	httpserver.EnableCORS(w, r)
 	switch path {
-	case "/select/logsql/facets":
-		logsqlFacetsRequests.Inc()
-		logsql.ProcessFacetsRequest(ctx, w, r)
-		return true
 	case "/select/logsql/field_names":
 		logsqlFieldNamesRequests.Inc()
 		logsql.ProcessFieldNamesRequest(ctx, w, r)
@@ -197,14 +193,6 @@ func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 		logsqlQueryRequests.Inc()
 		logsql.ProcessQueryRequest(ctx, w, r)
 		return true
-	case "/select/logsql/stats_query":
-		logsqlStatsQueryRequests.Inc()
-		logsql.ProcessStatsQueryRequest(ctx, w, r)
-		return true
-	case "/select/logsql/stats_query_range":
-		logsqlStatsQueryRangeRequests.Inc()
-		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
-		return true
 	case "/select/logsql/stream_field_names":
 		logsqlStreamFieldNamesRequests.Inc()
 		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
@@ -240,13 +228,10 @@ func getMaxQueryDuration(r *http.Request) time.Duration {
 }
 
 var (
-	logsqlFacetsRequests            = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
 	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
 	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
 	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
 	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlStatsQueryRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
-	logsqlStatsQueryRangeRequests   = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
 	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
 	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
 	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)

app/vlselect/vmui/asset-manifest.json

@@ -1,12 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.02a1c6cb.css",
-    "main.js": "./static/js/main.55c8060b.js",
-    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
+    "main.css": "./static/css/main.1041c3d4.css",
+    "main.js": "./static/js/main.8451d9b9.js",
+    "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.02a1c6cb.css",
-    "static/js/main.55c8060b.js"
+    "static/css/main.1041c3d4.css",
+    "static/js/main.8451d9b9.js"
   ]
 }

app/vlselect/vmui/config.json

@@ -1,5 +0,0 @@
-{
-  "license": {
-    "type": "opensource"
-  }
-}

app/vlselect/vmui/favicon.svg

@@ -1 +0,0 @@
-<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.55c8060b.js"></script><link href="./static/css/main.02a1c6cb.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8451d9b9.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/manifest.json

@@ -1,11 +1,16 @@
 {
-  "short_name": "vmui",
-  "name": "vmui",
+  "short_name": "Victoria Metrics UI",
+  "name": "Victoria Metrics UI is a metric explorer for Victoria Metrics",
   "icons": [
     {
-      "src": "favicon.svg",
-      "sizes": "any",
-      "type": "image/svg+xml"
+      "src": "favicon-32x32.png",
+      "sizes": "32x32",
+      "type": "image/png"
+    },
+    {
+      "src": "apple-touch-icon.png",
+      "type": "image/png",
+      "sizes": "192x192"
     }
   ],
   "start_url": ".",

app/vlselect/vmui/static/js/main.8451d9b9.js.LICENSE.txt

@@ -5,7 +5,7 @@
 */
 
 /**
- * @remix-run/router v1.19.2
+ * @remix-run/router v1.17.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -16,7 +16,7 @@
  */
 
 /**
- * React Router DOM v6.26.2
+ * React Router DOM v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -27,7 +27,7 @@
  */
 
 /**
- * React Router v6.26.2
+ * React Router v6.24.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vlselect/vmui/static/media/MetricsQL.d46c42c8e891f06298c4.md

@@ -1,4 +1,5 @@
 ---
+sort: 23
 weight: 23
 title: MetricsQL
 menu:
@@ -103,7 +104,7 @@ The list of MetricsQL features on top of PromQL:
 * Trailing commas on all the lists are allowed - label filters, function args and with expressions.
   For instance, the following queries are valid: `m{foo="bar",}`, `f(a, b,)`, `WITH (x=y,) x`.
   This simplifies maintenance of multi-line queries.
-* Metric names and label names may contain any unicode letter. For example `ტემპერატურა{πόλη="Київ"}` is a valid MetricsQL expression.
+* Metric names and label names may contain any unicode letter. For example `температура{город="Київ"}` is a valid MetricsQL expression.
 * Metric names and labels names may contain escaped chars. For example, `foo\-bar{baz\=aa="b"}` is valid expression.
   It returns time series with name `foo-bar` containing label `baz=aa` with value `b`.
   Additionally, the following escape sequences are supported:
@@ -545,7 +546,7 @@ while [increase](#increase) ignores the first value in a series if it is too big
 
 This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
 
-See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
+See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
 
 #### increases_over_time
 
@@ -954,7 +955,7 @@ See also [stddev_over_time](#stddev_over_time).
 
 #### sum_eq_over_time
 
-`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -965,7 +966,7 @@ See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over
 
 #### sum_gt_over_time
 
-`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -976,7 +977,7 @@ See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over
 
 #### sum_le_over_time
 
-`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.

app/vlstorage/main.go

@@ -18,12 +18,12 @@ import (
 )
 
 var (
-	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
+	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
 		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
 		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
 	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
 		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
-	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
+	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
 		"see https://docs.victoriametrics.com/victorialogs/#retention")
 	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
 		"see https://docs.victoriametrics.com/victorialogs/#storage")
@@ -37,8 +37,6 @@ var (
 		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
 		"the storage stops accepting new data")
-
-	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 )
 
 // Init initializes vlstorage.
@@ -89,28 +87,6 @@ func Stop() {
 	strg = nil
 }
 
-// RequestHandler is a storage request handler.
-func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
-	path := r.URL.Path
-	if path == "/internal/force_merge" {
-		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-			return true
-		}
-		// Run force merge in background
-		partitionNamePrefix := r.FormValue("partition_prefix")
-		go func() {
-			activeForceMerges.Inc()
-			defer activeForceMerges.Dec()
-			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-			startTime := time.Now()
-			strg.MustForceMerge(partitionNamePrefix)
-			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-		}()
-		return true
-	}
-	return false
-}
-
 var strg *logstorage.Storage
 var storageMetrics *metrics.Set
 
@@ -229,5 +205,3 @@ func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
 	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
 }
-
-var activeForceMerges = metrics.NewCounter("vl_active_force_merges")

app/vmagent/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image=non-existing
+ARG base_image
 FROM $base_image
 
 EXPOSE 8429
 
 ENTRYPOINT ["/vmagent-prod"]
-ARG src_binary=non-existing
+ARG src_binary
 COPY $src_binary ./vmagent-prod

app/vmagent/influx/request_handler.go

@@ -36,7 +36,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, true, isGzipped, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, nil)
 	})
 }
@@ -50,12 +50,11 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	isStreamMode := req.Header.Get("Stream-Mode") == "1"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }

app/vmagent/main.go

@@ -30,7 +30,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -46,7 +45,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
 var (
@@ -79,9 +77,6 @@ var (
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 0, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
-	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
 )
 
 var (
@@ -98,15 +93,6 @@ var (
 )
 
 func main() {
-	// vmagent is optimized for reduced memory allocations,
-	// so it can run with the reduced GOGC in order to reduce the used memory,
-	// while keeping CPU usage spent in GC at low levels.
-	//
-	// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
-	// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
-	// exceeds 0.05 for extended periods of time.
-	cgroup.SetGOGC(30)
-
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
@@ -114,7 +100,6 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 
 	if promscrape.IsDryRun() {
 		if err := promscrape.CheckConfig(); err != nil {
@@ -333,10 +318,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -513,7 +494,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
 		return true
 	}
-	at, err := auth.NewTokenPossibleMultitenant(p.AuthToken)
+	at, err := auth.NewToken(p.AuthToken)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot obtain auth token: %s", err)
 		return true
@@ -525,13 +506,7 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		statusCode := http.StatusNoContent
-		if strings.HasPrefix(p.Suffix, "prometheus/api/v1/import/prometheus/metrics/job/") {
-			// Return 200 status code for pushgateway requests.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
-			statusCode = http.StatusOK
-		}
-		w.WriteHeader(statusCode)
+		w.WriteHeader(http.StatusNoContent)
 		return true
 	}
 	if strings.HasPrefix(p.Suffix, "datadog/") {
@@ -589,10 +564,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -703,8 +674,7 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
-	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/vmagent/remotewrite/client.go

@@ -7,11 +7,12 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -22,7 +23,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -34,10 +34,8 @@ var (
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
-	sendTimeout      = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxTime")
-	retryMaxTime     = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
-	proxyURL         = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
 	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
@@ -92,9 +90,6 @@ type client struct {
 	fq *persistentqueue.FastQueue
 	hc *http.Client
 
-	retryMinInterval time.Duration
-	retryMaxTime     time.Duration
-
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
@@ -148,15 +143,13 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
-		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
-		authCfg:          authCfg,
-		awsCfg:           awsCfg,
-		fq:               fq,
-		hc:               hc,
-		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
-		retryMaxTime:     retryMaxTime.GetOptionalArg(argIdx),
-		stopCh:           make(chan struct{}),
+		sanitizedURL:   sanitizedURL,
+		remoteWriteURL: remoteWriteURL,
+		authCfg:        authCfg,
+		awsCfg:         awsCfg,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
 
@@ -403,11 +396,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
 //
 // The function returns false only if c.stopCh is closed.
-// Otherwise, it tries sending the block to remote storage indefinitely.
+// Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
-	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
+	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
+	retryDuration := timeutil.AddJitterToDuration(time.Second)
 	retriesCount := 0
 
 again:
@@ -463,10 +456,10 @@ again:
 
 	// Unexpected status code returned
 	retriesCount++
-	retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
-	retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
-
-	// Handle response
+	retryDuration *= 2
+	if retryDuration > maxRetryDuration {
+		retryDuration = maxRetryDuration
+	}
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
@@ -488,49 +481,3 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
-
-// getRetryDuration returns retry duration.
-// retryAfterDuration has the highest priority.
-// If retryAfterDuration is not specified, retryDuration gets doubled.
-// retryDuration can't exceed maxRetryDuration.
-//
-// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
-func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
-	// retryAfterDuration has the highest priority duration
-	if retryAfterDuration > 0 {
-		return timeutil.AddJitterToDuration(retryAfterDuration)
-	}
-
-	// default backoff retry policy
-	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
-	}
-
-	return retryDuration
-}
-
-// parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
-// retryAfterString should be in either HTTP-date or a number of seconds.
-// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
-func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
-	if retryAfterString == "" {
-		return retryAfterDuration
-	}
-
-	defer func() {
-		v := retryAfterDuration.Seconds()
-		logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
-	}()
-
-	// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
-	if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
-		return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
-	}
-	// Retry-After could be in seconds.
-	if seconds, err := strconv.Atoi(retryAfterString); err == nil {
-		return time.Duration(seconds) * time.Second
-	}
-
-	return 0
-}

app/vmagent/remotewrite/client_test.go

@@ -1,99 +0,0 @@
-package remotewrite
-
-import (
-	"math"
-	"net/http"
-	"testing"
-	"time"
-)
-
-func TestCalculateRetryDuration(t *testing.T) {
-	// `testFunc` call `calculateRetryDuration` for `n` times
-	// and evaluate if the result of `calculateRetryDuration` is
-	// 1. >= expectMinDuration
-	// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
-	f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
-		t.Helper()
-
-		for i := 0; i < n; i++ {
-			retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
-		}
-
-		expectMaxDuration := helper(expectMinDuration)
-		expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
-
-		if !(retryDuration >= expectMinDuration && retryDuration <= expectMaxDuration) {
-			t.Fatalf(
-				"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
-				expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
-				retryDuration.Milliseconds(),
-			)
-		}
-	}
-
-	// Call calculateRetryDuration for 1 time.
-	{
-		// default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-		// default backoff policy exceed max limit"
-		f(0, 10*time.Minute, 1, time.Minute)
-
-		// retry after > default backoff policy
-		f(10*time.Second, 1*time.Second, 1, 10*time.Second)
-		// retry after < default backoff policy
-		f(1*time.Second, 10*time.Second, 1, 1*time.Second)
-		// retry after invalid and < default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-
-	}
-
-	// Call calculateRetryDuration for multiple times.
-	{
-		// default backoff policy 2 times
-		f(0, time.Second, 2, 4*time.Second)
-		// default backoff policy 3 times
-		f(0, time.Second, 3, 8*time.Second)
-		// default backoff policy N times exceed max limit
-		f(0, time.Second, 10, time.Minute)
-
-		// retry after 120s 1 times
-		f(120*time.Second, time.Second, 1, 120*time.Second)
-		// retry after 120s 2 times
-		f(120*time.Second, time.Second, 2, 120*time.Second)
-	}
-}
-
-func TestParseRetryAfterHeader(t *testing.T) {
-	f := func(retryAfterString string, expectResult time.Duration) {
-		t.Helper()
-
-		result := parseRetryAfterHeader(retryAfterString)
-		// expect `expectResult == result` when retryAfterString is in seconds or invalid
-		// expect the difference between result and expectResult to be lower than 10%
-		if !(expectResult == result || math.Abs(float64(expectResult-result))/float64(expectResult) < 0.10) {
-			t.Fatalf(
-				"incorrect retry after duration, want (ms): %d, got (ms): %d",
-				expectResult.Milliseconds(), result.Milliseconds(),
-			)
-		}
-	}
-
-	// retry after header in seconds
-	f("10", 10*time.Second)
-	// retry after header in date time
-	f(time.Now().Add(30*time.Second).UTC().Format(http.TimeFormat), 30*time.Second)
-	// retry after header invalid
-	f("invalid-retry-after", 0)
-	// retry after header not in GMT
-	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
-}
-
-// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
-func helper(d time.Duration) time.Duration {
-	dv := d / 10
-	if dv > 10*time.Second {
-		dv = 10 * time.Second
-	}
-
-	return d + dv
-}

app/vmagent/remotewrite/remotewrite.go

@@ -7,10 +7,13 @@ import (
 	"net/url"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -18,7 +21,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
@@ -28,7 +30,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
 )
@@ -98,6 +99,9 @@ var (
 	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
 	rwctxsGlobal []*remoteWriteCtx
 
+	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
+	defaultAuthToken = &auth.Token{}
+
 	// ErrQueueFullHTTPRetry must be returned when TryPush() returns false.
 	ErrQueueFullHTTPRetry = &httpserver.ErrorWithStatusCode{
 		Err: fmt.Errorf("remote storage systems cannot keep up with the data ingestion rate; retry the request later " +
@@ -205,7 +209,7 @@ func Init() {
 
 	initStreamAggrConfigGlobal()
 
-	rwctxsGlobal = newRemoteWriteCtxs(*remoteWriteURLs)
+	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -290,7 +294,7 @@ var (
 	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
 )
 
-func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
+func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
@@ -312,6 +316,11 @@ func newRemoteWriteCtxs(urls []string) []*remoteWriteCtx {
 			logger.Fatalf("invalid -remoteWrite.url=%q: %s", remoteWriteURL, err)
 		}
 		sanitizedURL := fmt.Sprintf("%d:secret-url", i+1)
+		if at != nil {
+			// Construct full remote_write url for the given tenant according to https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+			remoteWriteURL.Path = fmt.Sprintf("%s/insert/%d:%d/prometheus/api/v1/write", remoteWriteURL.Path, at.AccountID, at.ProjectID)
+			sanitizedURL = fmt.Sprintf("%s:%d:%d", sanitizedURL, at.AccountID, at.ProjectID)
+		}
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
@@ -402,6 +411,11 @@ func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
 	tss := wr.Timeseries
 
+	if at == nil && MultitenancyEnabled() {
+		// Write data to default tenant if at isn't set when multitenancy is enabled.
+		at = defaultAuthToken
+	}
+
 	var tenantRctx *relabelCtx
 	if at != nil {
 		// Convert at to (vm_account_id, vm_project_id) labels.
@@ -427,7 +441,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
+	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}
@@ -471,15 +485,6 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 			rowsCountAfterRelabel := getRowsCount(tssBlock)
 			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
-		if timeserieslimits.Enabled() {
-			tmpBlock := tssBlock[:0]
-			for _, ts := range tssBlock {
-				if !timeserieslimits.IsExceeding(ts.Labels) {
-					tmpBlock = append(tmpBlock, ts)
-				}
-			}
-			tssBlock = tmpBlock
-		}
 		sortLabelsIfNeeded(tssBlock)
 		tssBlock = limitSeriesCardinality(tssBlock)
 		if sas.IsEnabled() {
@@ -489,8 +494,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 				tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
 			}
 			matchIdxsPool.Put(matchIdxs)
-		}
-		if deduplicatorGlobal != nil {
+		} else if deduplicatorGlobal != nil {
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
@@ -724,14 +728,29 @@ func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue i
 	select {
 	case <-logSkippedSeriesTicker.C:
 		// Do not use logger.WithThrottler() here, since this will increase CPU usage
-		// because every call to logSkippedSeries will result to a call to prompbmarshal.LabelsToString.
-		logger.Warnf("skip series %s because %s=%d reached", prompbmarshal.LabelsToString(labels), flagName, flagValue)
+		// because every call to logSkippedSeries will result to a call to labelsToString.
+		logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue)
 	default:
 	}
 }
 
 var logSkippedSeriesTicker = time.NewTicker(5 * time.Second)
 
+func labelsToString(labels []prompbmarshal.Label) string {
+	var b []byte
+	b = append(b, '{')
+	for i, label := range labels {
+		b = append(b, label.Name...)
+		b = append(b, '=')
+		b = strconv.AppendQuote(b, label.Value)
+		if i+1 < len(labels) {
+			b = append(b, ',')
+		}
+	}
+	b = append(b, '}')
+	return string(b)
+}
+
 var (
 	globalRowsPushedBeforeRelabel = metrics.NewCounter("vmagent_remotewrite_global_rows_pushed_before_relabel_total")
 	rowsDroppedByGlobalRelabel    = metrics.NewCounter("vmagent_remotewrite_global_relabel_metrics_dropped_total")
@@ -903,8 +922,7 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		}
 		matchIdxsPool.Put(matchIdxs)
-	}
-	if rwctx.deduplicator != nil {
+	} else if rwctx.deduplicator != nil {
 		rwctx.deduplicator.Push(tss)
 		return true
 	}

app/vmagent/remotewrite/streamaggr.go

@@ -3,7 +3,6 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
-	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -24,7 +23,7 @@ var (
 	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
+	streamAggrGlobalDedupInterval = flagutil.NewDuration("streamAggr.dedupInterval", "0s", "Input samples are de-duplicated with this interval on "+
 		"aggregator before optional aggregation with -streamAggr.config . "+
 		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
 	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
@@ -51,13 +50,12 @@ var (
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
 		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
 		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
-		"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
 )
 
@@ -132,10 +130,11 @@ func initStreamAggrConfigGlobal() {
 		sasGlobal.Store(sas)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	}
-	dedupInterval := *streamAggrGlobalDedupInterval
-	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
+	} else {
+		dedupInterval := streamAggrGlobalDedupInterval.Duration()
+		if dedupInterval > 0 {
+			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
+		}
 	}
 }
 
@@ -153,15 +152,12 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
 		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	}
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-	if dedupInterval > 0 {
-		alias := fmt.Sprintf("dedup-%d", idx+1)
-		var dropLabels []string
-		if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-			dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
+	} else {
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+		if dedupInterval > 0 {
+			alias := fmt.Sprintf("dedup-%d", idx+1)
+			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
 		}
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, dropLabels, alias)
 	}
 }
 
@@ -202,11 +198,10 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 	}
 
 	opts := &streamaggr.Options{
-		DedupInterval:        *streamAggrGlobalDedupInterval,
+		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
 		DropInputLabels:      *streamAggrGlobalDropInputLabels,
 		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
-		KeepInput:            *streamAggrGlobalKeepInput,
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
@@ -230,16 +225,11 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
 	if *showRemoteWriteURL {
 		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
 	}
-	var dropLabels []string
-	if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-		dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
-	}
 	opts := &streamaggr.Options{
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
-		DropInputLabels:      dropLabels,
+		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
-		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
+		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

app/vmalert-tool/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image=non-existing
-ARG root_image=non-existing
+ARG certs_image
+ARG root_image
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 

app/vmalert/config/config.go

@@ -1,25 +1,19 @@
 package config
 
 import (
-	"bytes"
 	"crypto/md5"
-	"flag"
 	"fmt"
 	"hash/fnv"
-	"io"
 	"net/url"
 	"sort"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"gopkg.in/yaml.v2"
-)
-
-var (
-	defaultRuleType = flag.String("rule.defaultRuleType", "prometheus", `Default type for rule expressions, can be overridden via "type" parameter on the group level, see https://docs.victoriametrics.com/vmalert/#groups. Supported values: "graphite", "prometheus" and "vlogs".`)
 )
 
 // Group contains list of Rules grouped into
@@ -64,9 +58,11 @@ func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
 	if err != nil {
 		return fmt.Errorf("failed to marshal group configuration for checksum: %w", err)
 	}
+	// change default value to prometheus datasource.
 	if g.Type.Get() == "" {
-		g.Type = NewRawType(*defaultRuleType)
+		g.Type.Set(NewPrometheusType())
 	}
+
 	h := md5.New()
 	h.Write(b)
 	g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
@@ -302,30 +298,16 @@ func parseConfig(data []byte) ([]Group, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot expand environment vars: %w", err)
 	}
-
-	var result []Group
-	type cfgFile struct {
+	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.
 		XXX map[string]any `yaml:",inline"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	if err != nil {
+		return nil, err
 	}
-
-	decoder := yaml.NewDecoder(bytes.NewReader(data))
-	for {
-		var cf cfgFile
-		if err = decoder.Decode(&cf); err != nil {
-			if err == io.EOF { // EOF indicates no more documents to read
-				break
-			}
-			return nil, err
-		}
-		if err = checkOverflow(cf.XXX, "config"); err != nil {
-			return nil, err
-		}
-		result = append(result, cf.Groups...)
-	}
-
-	return result, nil
+	return g.Groups, checkOverflow(g.XXX, "config")
 }
 
 func checkOverflow(m map[string]any, ctx string) error {

app/vmalert/config/config_test.go

@@ -9,14 +9,15 @@ import (
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, true); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())
@@ -39,34 +40,6 @@ groups:
 		w.Write([]byte(`
 groups:
   - name: TestGroup
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)`))
-	})
-	mux.HandleFunc("/good-multi-doc", func(w http.ResponseWriter, _ *http.Request) {
-		w.Write([]byte(`
-groups:
-  - name: foo
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)
----
-groups:
-  - name: bar
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)`))
-	})
-	mux.HandleFunc("/bad-multi-doc", func(w http.ResponseWriter, _ *http.Request) {
-		w.Write([]byte(`
-bad_field:
-  - name: foo
-    rules:
-      - record: conns
-        expr: max(vm_tcplistener_conns)
----
-groups:
-  - name: bar
     rules:
       - record: conns
         expr: max(vm_tcplistener_conns)`))
@@ -75,23 +48,13 @@ groups:
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	f := func(urls []string, expErr bool) {
-		for i, u := range urls {
-			urls[i] = srv.URL + u
-		}
-		_, err := Parse(urls, notifier.ValidateTemplates, true)
-		if err != nil && !expErr {
-			t.Fatalf("error parsing URLs %s", err)
-		}
-		if err == nil && expErr {
-			t.Fatalf("expecting error parsing URLs but got none")
-		}
+	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
+		t.Fatalf("error parsing URLs %s", err)
 	}
 
-	f([]string{"/good-alert", "/good-rr", "/good-multi-doc"}, false)
-	f([]string{"/bad"}, true)
-	f([]string{"/bad-multi-doc"}, true)
-	f([]string{"/good-alert", "/bad"}, true)
+	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
+		t.Fatalf("expected parsing error: %s", err)
+	}
 }
 
 func TestParse_Success(t *testing.T) {
@@ -122,10 +85,7 @@ func TestParse_Failure(t *testing.T) {
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
 	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
 	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
-	f([]string{"testdata/rules/vlog-rules0-bad.rules"}, "bad LogsQL expr")
 	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
-	f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields")
-	f([]string{"testdata/rules/rules-multi-doc-duplicates-bad.rules"}, "duplicate")
 	f([]string{"http://unreachable-url"}, "failed to")
 }
 
@@ -241,7 +201,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 	}, false, "duplicate")
 
 	f(&Group{
-		Name: "test graphite with prometheus expr",
+		Name: "test graphite prometheus bad expr",
 		Type: NewGraphiteType(),
 		Rules: []Rule{
 			{
@@ -268,20 +228,6 @@ func TestGroupValidate_Failure(t *testing.T) {
 		},
 	}, false, "either `record` or `alert` must be set")
 
-	f(&Group{
-		Name: "test vlogs with prometheus expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
-
 	// validate expressions
 	f(&Group{
 		Name: "test",
@@ -312,16 +258,6 @@ func TestGroupValidate_Failure(t *testing.T) {
 			}},
 		},
 	}, true, "bad graphite expr")
-
-	f(&Group{
-		Name: "test vlogs",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "stats count(*) as requests", Labels: map[string]string{
-				"description": "some-description",
-			}},
-		},
-	}, true, "bad LogsQL expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {
@@ -361,7 +297,7 @@ func TestGroupValidate_Success(t *testing.T) {
 		},
 	}, false, false)
 
-	// validate annotations
+	// validate annotiations
 	f(&Group{
 		Name: "test",
 		Rules: []Rule{
@@ -388,15 +324,6 @@ func TestGroupValidate_Success(t *testing.T) {
 			}},
 		},
 	}, false, true)
-	f(&Group{
-		Name: "test victorialogs",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: " _time: 1m | stats count(*) as requests", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, true)
 }
 
 func TestHashRule_NotEqual(t *testing.T) {

app/vmalert/config/testdata/dir/rules0-bad.rules

@@ -7,7 +7,7 @@ groups:
         labels:
           label: bar
         annotations:
-          summary: "{{ }}"
+          summary: "{{ $value }"
           description: "{{$labels}}"
       - alert: UnkownAnnotationsFunction
         for: 5m

app/vmalert/config/testdata/rules/rules-multi-doc-bad.rules

@@ -1,29 +0,0 @@
-groups:
-  - name: groupTest
-    rules:
-      - alert: VMRows
-        for: 1ms
-        expr: vm_rows > 0
-        labels:
-          label: bar
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
-invalid-field-1: invalid-value-1
-invalid-field-2: invalid-value-2
----
-groups:
-  - name: TestGroup
-    interval: 2s
-    concurrency: 2
-    type: graphite
-    rules:
-      - alert: Conns
-        expr: filterSeries(sumSeries(host.receiver.interface.cons),'last','>', 500)
-        for: 3m
-
-        annotations:
-          summary: Too high connection number for {{$labels.instance}}
-          description: "It is {{ $value }} connections for {{$labels.instance}}"
-invalid-field-2: invalid-value-2
-invalid-field-3: invalid-value-3

app/vmalert/config/testdata/rules/rules-multi-doc-duplicates-bad.rules

@@ -1,11 +0,0 @@
-groups:
-  - name: foo
-    rules:
-      - alert: VMRows
-        expr: vm_rows > 0
----
-groups:
-  - name: foo
-    rules:
-      - alert: VMRows
-        expr: vm_rows > 0

app/vmalert/config/testdata/rules/rules-multi-doc-good.rules

@@ -1,15 +0,0 @@
-
----
-groups:
-  - name: groupTest
-    rules:
-      - alert: VMRows
-        for: 1ms
-        expr: vm_rows > 0
-        labels:
-          label: bar
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
----
-groups:

app/vmalert/config/testdata/rules/rules-multi-doc2-good.rules

@@ -1,46 +0,0 @@
----
-groups:
-  - name: groupTest
-    rules:
-      - alert: VMRows
-        for: 1ms
-        expr: vm_rows > 0
-        labels:
-          label: bar
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
-  - name: groupTest-2
-    rules:
-      - alert: VMRows-2
-        for: 1ms
-        expr: vm_rows_2 > 0
-        labels:
-          label: bar2
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "\n markdown result is : \n---\n # header\n body: \n text \n----\n"
----
-groups:
-  - name: groupTest-3
-    rules:
-      - alert: VMRows-3
-        for: 1ms
-        expr: vm_rows_3 > 0
-        labels:
-          label: bar_3
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
-  - name: groupTest-4
-    rules:
-      - alert: VMRows-4
-        for: 1ms
-        expr: vm_rows_4 > 0
-        labels:
-          label: bar4
-          host: "{{ $labels.instance }}"
-        annotations:
-          summary: "{{ $value }}"
----
-groups:

app/vmalert/config/testdata/rules/vlog-rules0-bad.rules

@@ -1,10 +0,0 @@
-groups:
-  - name: InvalidStatsLogsql
-    type: vlogs
-    interval: 5m
-    rules:
-      - record: MissingFilter
-        expr: 'stats count(*) as requests'
-      - record: MissingStatsPipe
-        expr: 'service: "nginx"'
-

app/vmalert/config/testdata/rules/vlog-rules0-good.rules

@@ -1,29 +0,0 @@
-groups:
-  - name: RequestCount
-    type: vlogs
-    interval: 5m
-    rules:
-      - record: nginxRequestCount
-        expr: 'env: "test" AND service: "nginx" | stats count(*) as requests'
-        annotations:
-          description: "Service nginx on env test accepted {{$labels.requests}} requests in the last 5 minutes"
-      - record: prodRequestCount
-        expr: 'env: "prod" | stats by (service) count(*) as requests'
-        annotations:
-          description: "Service {{$labels.service}} on env prod accepted {{$labels.requests}} requests in the last 5 minutes"
-  - name: ServiceLog
-    type: vlogs
-    interval: 5m
-    rules:
-      - alert: HasErrorLog
-        expr: 'env: "prod" AND status:~"error|warn" | stats by (service) count(*) as errorLog | filter errorLog:>0'
-        annotations:
-          description: "Service {{$labels.service}} generated {{$labels.errorLog}} error logs in the last 5 minutes"
-  - name: ServiceRequest
-    type: vlogs
-    interval: 10m
-    rules:
-      - alert: TooManyFailedRequest
-        expr: '* | extract "ip=<ip> " | extract "status_code=<code>;" | stats by (ip) count() if (code:!~200) as failed, count() as total| math failed / total as failed_percentage| filter failed_percentage :> 0.01 | fields ip,failed_percentage'
-        annotations:
-          description: "Connection from address {{$labels.ip}} has {{$value}} failed requests ratio in last 10 minutes"

app/vmalert/config/types.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/metricsql"
 )
 
@@ -28,13 +27,6 @@ func NewGraphiteType() Type {
 	}
 }
 
-// NewVLogsType returns victorialogs datasource type
-func NewVLogsType() Type {
-	return Type{
-		Name: "vlogs",
-	}
-}
-
 // NewRawType returns datasource type from raw string
 // without validation.
 func NewRawType(d string) Type {
@@ -70,10 +62,6 @@ func (t *Type) ValidateExpr(expr string) error {
 		if _, err := metricsql.Parse(expr); err != nil {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
-	case "vlogs":
-		if _, err := logstorage.ParseStatsQuery(expr, 0); err != nil {
-			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
-		}
 	default:
 		return fmt.Errorf("unknown datasource type=%q", t.Name)
 	}
@@ -86,10 +74,13 @@ func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
 	if err := unmarshal(&s); err != nil {
 		return err
 	}
+	if s == "" {
+		s = "prometheus"
+	}
 	switch s {
-	case "graphite", "prometheus", "vlogs":
+	case "graphite", "prometheus":
 	default:
-		return fmt.Errorf("unknown datasource type=%q, want prometheus, graphite or vlogs", s)
+		return fmt.Errorf("unknown datasource type=%q, want %q or %q", s, "prometheus", "graphite")
 	}
 	t.Name = s
 	return nil

app/vmalert/datasource/client.go

@@ -1,333 +0,0 @@
-package datasource
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strings"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-)
-
-type datasourceType string
-
-const (
-	datasourcePrometheus datasourceType = "prometheus"
-	datasourceGraphite   datasourceType = "graphite"
-	datasourceVLogs      datasourceType = "vlogs"
-)
-
-func toDatasourceType(s string) datasourceType {
-	switch s {
-	case string(datasourcePrometheus):
-		return datasourcePrometheus
-	case string(datasourceGraphite):
-		return datasourceGraphite
-	case string(datasourceVLogs):
-		return datasourceVLogs
-	default:
-		logger.Panicf("BUG: unknown datasource type %q", s)
-	}
-	return ""
-}
-
-// Client is a datasource entity for reading data,
-// supported clients are enumerated in datasourceType.
-// WARN: when adding a new field, remember to check if Clone() method needs to be updated.
-type Client struct {
-	c                *http.Client
-	authCfg          *promauth.Config
-	datasourceURL    string
-	appendTypePrefix bool
-	queryStep        time.Duration
-	dataSourceType   datasourceType
-	// ApplyIntervalAsTimeFilter is only valid for vlogs datasource.
-	// Set to true if there is no [timeFilter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter) in the rule expression,
-	// and we will add evaluation interval as an additional timeFilter when querying.
-	applyIntervalAsTimeFilter bool
-
-	// evaluationInterval will help setting request's `step` param,
-	// or adding time filter for LogsQL expression.
-	evaluationInterval time.Duration
-	// extraParams contains params to be attached to each HTTP request
-	extraParams url.Values
-	// extraHeaders are headers to be attached to each HTTP request
-	extraHeaders []keyValue
-
-	// whether to print additional log messages
-	// for each sent request
-	debug bool
-}
-
-type keyValue struct {
-	key   string
-	value string
-}
-
-// Clone clones shared http client and other configuration to the new client.
-func (c *Client) Clone() *Client {
-	ns := &Client{
-		c:                c.c,
-		authCfg:          c.authCfg,
-		datasourceURL:    c.datasourceURL,
-		appendTypePrefix: c.appendTypePrefix,
-		queryStep:        c.queryStep,
-
-		dataSourceType:     c.dataSourceType,
-		evaluationInterval: c.evaluationInterval,
-
-		// init map so it can be populated below
-		extraParams: url.Values{},
-
-		debug: c.debug,
-	}
-	if len(c.extraHeaders) > 0 {
-		ns.extraHeaders = make([]keyValue, len(c.extraHeaders))
-		copy(ns.extraHeaders, c.extraHeaders)
-	}
-	for k, v := range c.extraParams {
-		ns.extraParams[k] = v
-	}
-
-	return ns
-}
-
-// ApplyParams - changes given querier params.
-func (c *Client) ApplyParams(params QuerierParams) *Client {
-	if params.DataSourceType != "" {
-		c.dataSourceType = toDatasourceType(params.DataSourceType)
-	}
-	c.evaluationInterval = params.EvaluationInterval
-	c.applyIntervalAsTimeFilter = params.ApplyIntervalAsTimeFilter
-	if params.QueryParams != nil {
-		if c.extraParams == nil {
-			c.extraParams = url.Values{}
-		}
-		for k, vl := range params.QueryParams {
-			// custom query params are prior to default ones
-			if c.extraParams.Has(k) {
-				c.extraParams.Del(k)
-			}
-			for _, v := range vl {
-				// don't use .Set() instead of Del/Add since it is allowed
-				// for GET params to be duplicated
-				// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4908
-				c.extraParams.Add(k, v)
-			}
-		}
-	}
-	if params.Headers != nil {
-		for key, value := range params.Headers {
-			kv := keyValue{key: key, value: value}
-			c.extraHeaders = append(c.extraHeaders, kv)
-		}
-	}
-	c.debug = params.Debug
-	return c
-}
-
-// BuildWithParams - implements interface.
-func (c *Client) BuildWithParams(params QuerierParams) Querier {
-	return c.Clone().ApplyParams(params)
-}
-
-// NewPrometheusClient returns a new prometheus datasource client.
-func NewPrometheusClient(baseURL string, authCfg *promauth.Config, appendTypePrefix bool, c *http.Client) *Client {
-	return &Client{
-		c:                c,
-		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(baseURL, "/"),
-		appendTypePrefix: appendTypePrefix,
-		queryStep:        *queryStep,
-		dataSourceType:   datasourcePrometheus,
-		extraParams:      url.Values{},
-	}
-}
-
-// Query executes the given query and returns parsed response
-func (c *Client) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
-	req, err := c.newQueryRequest(ctx, query, ts)
-	if err != nil {
-		return Result{}, nil, err
-	}
-	resp, err := c.do(req)
-	if err != nil {
-		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
-			// Return unexpected error to the caller.
-			return Result{}, nil, err
-		}
-		// Something in the middle between client and datasource might be closing
-		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = c.newQueryRequest(ctx, query, ts)
-		if err != nil {
-			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
-		}
-		resp, err = c.do(req)
-		if err != nil {
-			return Result{}, nil, fmt.Errorf("second attempt: %w", err)
-		}
-	}
-
-	// Process the received response.
-	var parseFn func(req *http.Request, resp *http.Response) (Result, error)
-	switch c.dataSourceType {
-	case datasourcePrometheus:
-		parseFn = parsePrometheusResponse
-	case datasourceGraphite:
-		parseFn = parseGraphiteResponse
-	case datasourceVLogs:
-		parseFn = parseVLogsResponse
-	default:
-		logger.Panicf("BUG: unsupported datasource type %q to parse query response", c.dataSourceType)
-	}
-	result, err := parseFn(req, resp)
-	_ = resp.Body.Close()
-	return result, req, err
-}
-
-// QueryRange executes the given query on the given time range.
-// For Prometheus type see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
-// Graphite type isn't supported.
-func (c *Client) QueryRange(ctx context.Context, query string, start, end time.Time) (res Result, err error) {
-	if c.dataSourceType == datasourceGraphite {
-		return res, fmt.Errorf("%q is not supported for QueryRange", c.dataSourceType)
-	}
-	// TODO: disable range query LogsQL with time filter now
-	if c.dataSourceType == datasourceVLogs && !c.applyIntervalAsTimeFilter {
-		return res, fmt.Errorf("range query is not supported for LogsQL expression %q because it contains time filter. Remove time filter from the expression and try again", query)
-	}
-	if start.IsZero() {
-		return res, fmt.Errorf("start param is missing")
-	}
-	if end.IsZero() {
-		return res, fmt.Errorf("end param is missing")
-	}
-	req, err := c.newQueryRangeRequest(ctx, query, start, end)
-	if err != nil {
-		return res, err
-	}
-	resp, err := c.do(req)
-	if err != nil {
-		if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) && !netutil.IsTrivialNetworkError(err) {
-			// Return unexpected error to the caller.
-			return res, err
-		}
-		// Something in the middle between client and datasource might be closing
-		// the connection. So we do a one more attempt in hope request will succeed.
-		req, err = c.newQueryRangeRequest(ctx, query, start, end)
-		if err != nil {
-			return res, fmt.Errorf("second attempt: %w", err)
-		}
-		resp, err = c.do(req)
-		if err != nil {
-			return res, fmt.Errorf("second attempt: %w", err)
-		}
-	}
-
-	// Process the received response.
-	var parseFn func(req *http.Request, resp *http.Response) (Result, error)
-	switch c.dataSourceType {
-	case datasourcePrometheus:
-		parseFn = parsePrometheusResponse
-	case datasourceVLogs:
-		parseFn = parseVLogsResponse
-	default:
-		logger.Panicf("BUG: unsupported datasource type %q to parse query range response", c.dataSourceType)
-	}
-	res, err = parseFn(req, resp)
-	_ = resp.Body.Close()
-	return res, err
-}
-
-func (c *Client) do(req *http.Request) (*http.Response, error) {
-	ru := req.URL.Redacted()
-	if *showDatasourceURL {
-		ru = req.URL.String()
-	}
-	if c.debug {
-		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, ru)
-	}
-	resp, err := c.c.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("error getting response from %s: %w", ru, err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		body, _ := io.ReadAll(resp.Body)
-		_ = resp.Body.Close()
-		return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
-	}
-	return resp, nil
-}
-
-func (c *Client) newQueryRangeRequest(ctx context.Context, query string, start, end time.Time) (*http.Request, error) {
-	req, err := c.newRequest(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create query_range request to datasource %q: %w", c.datasourceURL, err)
-	}
-	switch c.dataSourceType {
-	case datasourcePrometheus:
-		c.setPrometheusRangeReqParams(req, query, start, end)
-	case datasourceVLogs:
-		c.setVLogsRangeReqParams(req, query, start, end)
-	default:
-		logger.Panicf("BUG: unsupported datasource type %q to create range query request", c.dataSourceType)
-	}
-	return req, nil
-}
-
-func (c *Client) newQueryRequest(ctx context.Context, query string, ts time.Time) (*http.Request, error) {
-	req, err := c.newRequest(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create query request to datasource %q: %w", c.datasourceURL, err)
-	}
-	switch c.dataSourceType {
-	case datasourcePrometheus:
-		c.setPrometheusInstantReqParams(req, query, ts)
-	case datasourceGraphite:
-		c.setGraphiteReqParams(req, query)
-	case datasourceVLogs:
-		c.setVLogsInstantReqParams(req, query, ts)
-	default:
-		logger.Panicf("BUG: unsupported datasource type %q to create query request", c.dataSourceType)
-	}
-	return req, nil
-}
-
-func (c *Client) newRequest(ctx context.Context) (*http.Request, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.datasourceURL, nil)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", c.datasourceURL, err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	if c.authCfg != nil {
-		err = c.authCfg.SetHeaders(req, true)
-		if err != nil {
-			return nil, err
-		}
-	}
-	for _, h := range c.extraHeaders {
-		req.Header.Set(h.key, h.value)
-	}
-	return req, nil
-}
-
-// setReqParams adds query and other extra params for the request.
-func (c *Client) setReqParams(r *http.Request, query string) {
-	q := r.URL.Query()
-	for k, vs := range c.extraParams {
-		if q.Has(k) { // extraParams are prior to params in URL
-			q.Del(k)
-		}
-		for _, v := range vs {
-			q.Add(k, v)
-		}
-	}
-	q.Set("query", query)
-	r.URL.RawQuery = q.Encode()
-}

app/vmalert/datasource/client_vlogs.go

@@ -1,61 +0,0 @@
-package datasource
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-)
-
-func (c *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) {
-	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
-	if !*disablePathAppend {
-		r.URL.Path += "/select/logsql/stats_query"
-	}
-	q := r.URL.Query()
-	// set `time` param explicitly, it will be used as the timestamp of query results.
-	q.Set("time", timestamp.Format(time.RFC3339))
-	// set the `start` and `end` params if applyIntervalAsTimeFilter is enabled(time filter is missing in the rule expr),
-	// so the query will be executed in time range [timestamp - evaluationInterval, timestamp].
-	if c.applyIntervalAsTimeFilter && c.evaluationInterval > 0 {
-		q.Set("start", timestamp.Add(-c.evaluationInterval).Format(time.RFC3339))
-		q.Set("end", timestamp.Format(time.RFC3339))
-	}
-	r.URL.RawQuery = q.Encode()
-	c.setReqParams(r, query)
-}
-
-func (c *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) {
-	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
-	if !*disablePathAppend {
-		r.URL.Path += "/select/logsql/stats_query_range"
-	}
-	q := r.URL.Query()
-	q.Add("start", start.Format(time.RFC3339))
-	q.Add("end", end.Format(time.RFC3339))
-	// set step as evaluationInterval by default
-	if c.evaluationInterval > 0 {
-		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
-	}
-	r.URL.RawQuery = q.Encode()
-	c.setReqParams(r, query)
-}
-
-func parseVLogsResponse(req *http.Request, resp *http.Response) (res Result, err error) {
-	res, err = parsePrometheusResponse(req, resp)
-	if err != nil {
-		return Result{}, err
-	}
-	for i := range res.Data {
-		m := &res.Data[i]
-		for j := range m.Labels {
-			// reserve the stats func result name with a new label `stats_result` instead of dropping it,
-			// since there could be multiple stats results in a single query, for instance:
-			// 	_time:5m | stats quantile(0.5, request_duration_seconds) p50, quantile(0.9, request_duration_seconds) p90
-			if m.Labels[j].Name == "__name__" {
-				m.Labels[j].Name = "stats_result"
-				break
-			}
-		}
-	}
-	return
-}

app/vmalert/datasource/datasource.go

@@ -8,8 +8,6 @@ import (
 	"sort"
 	"strconv"
 	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Querier interface wraps Query and QueryRange methods
@@ -44,20 +42,16 @@ type QuerierBuilder interface {
 
 // QuerierParams params for Querier.
 type QuerierParams struct {
-	DataSourceType string
-	// ApplyIntervalAsTimeFilter is only valid for vlogs datasource.
-	// Set to true if there is no [timeFilter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter) in the rule expression,
-	// and we will add evaluation interval as an additional timeFilter when querying.
-	ApplyIntervalAsTimeFilter bool
-	EvaluationInterval        time.Duration
-	QueryParams               url.Values
-	Headers                   map[string]string
-	Debug                     bool
+	DataSourceType     string
+	EvaluationInterval time.Duration
+	QueryParams        url.Values
+	Headers            map[string]string
+	Debug              bool
 }
 
 // Metric is the basic entity which should be return by datasource
 type Metric struct {
-	Labels     []prompbmarshal.Label
+	Labels     []Label
 	Timestamps []int64
 	Values     []float64
 }
@@ -74,9 +68,22 @@ func (m *Metric) SetLabel(key, value string) {
 	m.AddLabel(key, value)
 }
 
+// SetLabels sets the given map as Metric labels
+func (m *Metric) SetLabels(ls map[string]string) {
+	var i int
+	m.Labels = make([]Label, len(ls))
+	for k, v := range ls {
+		m.Labels[i] = Label{
+			Name:  k,
+			Value: v,
+		}
+		i++
+	}
+}
+
 // AddLabel appends the given label to the label set
 func (m *Metric) AddLabel(key, value string) {
-	m.Labels = append(m.Labels, prompbmarshal.Label{Name: key, Value: value})
+	m.Labels = append(m.Labels, Label{Name: key, Value: value})
 }
 
 // DelLabel deletes the given label from the label set
@@ -99,8 +106,14 @@ func (m *Metric) Label(key string) string {
 	return ""
 }
 
+// Label represents metric's label
+type Label struct {
+	Name  string
+	Value string
+}
+
 // Labels is collection of Label
-type Labels []prompbmarshal.Label
+type Labels []Label
 
 func (ls Labels) Len() int           { return len(ls) }
 func (ls Labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
@@ -155,7 +168,7 @@ func LabelCompare(a, b Labels) int {
 // ConvertToLabels convert map to Labels
 func ConvertToLabels(m map[string]string) (labelset Labels) {
 	for k, v := range m {
-		labelset = append(labelset, prompbmarshal.Label{
+		labelset = append(labelset, Label{
 			Name:  k,
 			Value: v,
 		})

app/vmalert/datasource/init.go

@@ -11,12 +11,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
-		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
 		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
@@ -47,15 +48,22 @@ var (
 	oauth2TokenURL = flag.String("datasource.oauth2.tokenUrl", "", "Optional OAuth2 tokenURL to use for -datasource.url")
 	oauth2Scopes   = flag.String("datasource.oauth2.scopes", "", "Optional OAuth2 scopes to use for -datasource.url. Scopes must be delimited by ';'")
 
-	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries to the configured -datasource.url and -remoteRead.url. Only valid for prometheus datasource. "+
+	lookBack = flag.Duration("datasource.lookback", 0, `Deprecated: please adjust "-search.latencyOffset" at datasource side `+
+		`or specify "latency_offset" in rule group's params. Lookback defines how far into the past to look when evaluating queries. `+
+		`For example, if the datasource.lookback=5m then param "time" with value now()-5m will be added to every query.`)
+	queryStep = flag.Duration("datasource.queryStep", 5*time.Minute, "How far a value can fallback to when evaluating queries. "+
 		"For example, if -datasource.queryStep=15s then param \"step\" with value \"15s\" will be added to every query. "+
 		"If set to 0, rule's evaluation interval will be used instead.")
+	queryTimeAlignment = flag.Bool("datasource.queryTimeAlignment", true, `Deprecated: please use "eval_alignment" in rule group instead. `+
+		`Whether to align "time" parameter with evaluation interval. `+
+		"Alignment supposed to produce deterministic results despite number of vmalert replicas or time they were started. "+
+		"See more details at https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257")
 	maxIdleConnections    = flag.Int("datasource.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to each configured datasource. Consider setting this value equal to the value: groups_total * group.concurrency. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 	idleConnectionTimeout = flag.Duration("datasource.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
 	disableKeepAlive      = flag.Bool("datasource.disableKeepAlive", false, `Whether to disable long-lived connections to the datasource. `+
 		`If true, disables HTTP keep-alive and will only use the connection to the server for a single HTTP request.`)
-	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests which limits the number of digits after the decimal point in response values. `+
-		`Only valid for VictoriaMetrics as the datasource.`)
+	roundDigits = flag.Int("datasource.roundDigits", 0, `Adds "round_digits" GET param to datasource requests. `+
+		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
 // InitSecretFlags must be called after flag.Parse and before any logging
@@ -82,10 +90,16 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if *addr == "" {
 		return nil, fmt.Errorf("datasource.url is empty")
 	}
+	if !*queryTimeAlignment {
+		logger.Warnf("flag `-datasource.queryTimeAlignment` is deprecated and will be removed in next releases. Please use `eval_alignment` in rule group instead.")
+	}
+	if *lookBack != 0 {
+		logger.Warnf("flag `-datasource.lookback` is deprecated and will be removed in next releases. Please adjust `-search.latencyOffset` at datasource side or specify `latency_offset` in rule group's params. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155 for details.")
+	}
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive
@@ -119,12 +133,13 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to set request auth header to datasource %q: %w", *addr, err)
 	}
 
-	return &Client{
+	return &VMStorage{
 		c:                &http.Client{Transport: tr},
 		authCfg:          authCfg,
 		datasourceURL:    strings.TrimSuffix(*addr, "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
+		dataSourceType:   datasourcePrometheus,
 		extraParams:      extraParams,
 	}, nil
 }