replaced existing cspell make target

### Describe Your Changes

Doc updates to a patch release v1.20.1, fixing a bug in
`PeriodicScheduler` that may affect some of the customers' deployments

### Checklist

The following checks are **mandatory**:

- [ ] My change adheres [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/contributing/).

Makefile

@@ -18,7 +18,7 @@ TAR_OWNERSHIP ?= --owner=1000 --group=1000
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include cspell/Makefile
+include codespell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
@@ -567,7 +567,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.5
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

app/vlinsert/datadog/datadog.go

@@ -3,7 +3,6 @@ package datadog
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"net/http"
 	"strconv"
 	"time"
@@ -16,15 +15,17 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 )
 
 var (
-	datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Datadog tags to be used as stream fields.")
-	datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Datadog tags to ignore.")
+	datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested via DataDog protocol. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#stream-fields")
+	datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Comma-separated list of fields to ignore for logs ingested via DataDog protocol. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/#dropping-fields")
+
+	maxRequestSize = flagutil.NewBytes("datadog.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single DataDog request")
 )
 
 var parserPool fastjson.ParserPool
@@ -46,7 +47,6 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 	w.Header().Add("Content-Type", "application/json")
 	startTime := time.Now()
 	v2LogsRequestsTotal.Inc()
-	reader := r.Body
 
 	var ts int64
 	if tsValue := r.Header.Get("dd-message-timestamp"); tsValue != "" && tsValue != "0" {
@@ -61,24 +61,6 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 		ts = startTime.UnixNano()
 	}
 
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot read gzipped logs request: %s", err)
-			return true
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return true
-	}
-
 	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
@@ -97,11 +79,15 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	lmp := cp.NewLogMessageProcessor("datadog")
-	err = readLogsRequest(ts, data, lmp)
-	lmp.MustClose()
+	encoding := r.Header.Get("Content-Encoding")
+	err = common.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
+		lmp := cp.NewLogMessageProcessor("datadog")
+		err := readLogsRequest(ts, data, lmp)
+		lmp.MustClose()
+		return err
+	})
 	if err != nil {
-		logger.Warnf("cannot decode log message in /api/v2/logs request: %s, stream fields: %s", err, cp.StreamFields)
+		httpserver.Errorf(w, r, "cannot read DataDog protocol data: %s", err)
 		return true
 	}
 
@@ -121,7 +107,7 @@ var (
 // datadog message field has two formats:
 //   - regular log message with string text
 //   - nested json format for serverless plugins
-//     which has folowing format:
+//     which has the following format:
 //     {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
 //
 // See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24

app/vlinsert/elasticsearch/elasticsearch.go

@@ -100,9 +100,9 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		lmp := cp.NewLogMessageProcessor("elasticsearch_bulk")
-		isGzip := r.Header.Get("Content-Encoding") == "gzip"
+		encoding := r.Header.Get("Content-Encoding")
 		streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-		n, err := readBulkRequest(streamName, r.Body, isGzip, cp.TimeField, cp.MsgFields, lmp)
+		n, err := readBulkRequest(streamName, r.Body, encoding, cp.TimeField, cp.MsgFields, lmp)
 		lmp.MustClose()
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s, stream fields: %s", n, err, cp.StreamFields)
@@ -131,19 +131,16 @@ var (
 	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
-func readBulkRequest(streamName string, r io.Reader, isGzip bool, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (int, error) {
+func readBulkRequest(streamName string, r io.Reader, encoding string, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) (int, error) {
 	// See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 
-	if isGzip {
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return 0, fmt.Errorf("cannot read gzipped _bulk request: %w", err)
-		}
-		defer common.PutGzipReader(zr)
-		r = zr
+	reader, err := common.GetUncompressedReader(r, encoding)
+	if err != nil {
+		return 0, fmt.Errorf("cannot decode Elasticsearch protocol data: %w", err)
 	}
+	defer common.PutUncompressedReader(reader)
 
-	wcr := writeconcurrencylimiter.GetReader(r)
+	wcr := writeconcurrencylimiter.GetReader(reader)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
 	lr := insertutils.NewLineReader(streamName, wcr)

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -2,8 +2,12 @@ package elasticsearch
 
 import (
 	"bytes"
-	"compress/gzip"
 	"fmt"
+	"github.com/golang/snappy"
+	"github.com/klauspost/compress/gzip"
+	"github.com/klauspost/compress/zlib"
+	"github.com/klauspost/compress/zstd"
+	"io"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
@@ -15,7 +19,7 @@ func TestReadBulkRequest_Failure(t *testing.T) {
 
 		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, false, "_time", []string{"_msg"}, tlp)
+		rows, err := readBulkRequest("test", r, "", "_time", []string{"_msg"}, tlp)
 		if err == nil {
 			t.Fatalf("expecting non-empty error")
 		}
@@ -33,7 +37,7 @@ foobar`)
 }
 
 func TestReadBulkRequest_Success(t *testing.T) {
-	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
+	f := func(data, encoding, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
 		msgFields := []string{"non_existing_foo", msgField, "non_exiting_bar"}
@@ -41,7 +45,7 @@ func TestReadBulkRequest_Success(t *testing.T) {
 
 		// Read the request without compression
 		r := bytes.NewBufferString(data)
-		rows, err := readBulkRequest("test", r, false, timeField, msgFields, tlp)
+		rows, err := readBulkRequest("test", r, "", timeField, msgFields, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -54,9 +58,11 @@ func TestReadBulkRequest_Success(t *testing.T) {
 
 		// Read the request with compression
 		tlp = &insertutils.TestLogMessageProcessor{}
-		compressedData := compressData(data)
-		r = bytes.NewBufferString(compressedData)
-		rows, err = readBulkRequest("test", r, true, timeField, msgFields, tlp)
+		if encoding != "" {
+			data = compressData(data, encoding)
+		}
+		r = bytes.NewBufferString(data)
+		rows, err = readBulkRequest("test", r, encoding, timeField, msgFields, tlp)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -69,9 +75,9 @@ func TestReadBulkRequest_Success(t *testing.T) {
 	}
 
 	// Verify an empty data
-	f("", "_time", "_msg", nil, "")
-	f("\n", "_time", "_msg", nil, "")
-	f("\n\n", "_time", "_msg", nil, "")
+	f("", "gzip", "_time", "_msg", nil, "")
+	f("\n", "gzip", "_time", "_msg", nil, "")
+	f("\n\n", "gzip", "_time", "_msg", nil, "")
 
 	// Verify non-empty data
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
@@ -82,20 +88,35 @@ func TestReadBulkRequest_Success(t *testing.T) {
 {"message":"xyz","@timestamp":"1686026893735","x":"y"}
 {"create":{"_index":"filebeat-8.8.0"}}
 {"message":"qwe rty","@timestamp":"1686026893"}
+{"create":{"_index":"filebeat-8.8.0"}}
+{"message":"qwe rty float","@timestamp":"1686026123.62"}
 `
 	timeField := "@timestamp"
 	msgField := "message"
-	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
+	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000, 1686026123620000000}
 	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
 {"_msg":"baz"}
 {"_msg":"xyz","x":"y"}
-{"_msg":"qwe rty"}`
-	f(data, timeField, msgField, timestampsExpected, resultExpected)
+{"_msg":"qwe rty"}
+{"_msg":"qwe rty float"}`
+	f(data, "zstd", timeField, msgField, timestampsExpected, resultExpected)
 }
 
-func compressData(s string) string {
+func compressData(s string, encoding string) string {
 	var bb bytes.Buffer
-	zw := gzip.NewWriter(&bb)
+	var zw io.WriteCloser
+	switch encoding {
+	case "gzip":
+		zw = gzip.NewWriter(&bb)
+	case "zstd":
+		zw, _ = zstd.NewWriter(&bb)
+	case "snappy":
+		zw = snappy.NewBufferedWriter(&bb)
+	case "deflate":
+		zw = zlib.NewWriter(&bb)
+	default:
+		panic(fmt.Errorf("%q encoding is not supported", encoding))
+	}
 	if _, err := zw.Write([]byte(s)); err != nil {
 		panic(fmt.Errorf("unexpected error when compressing data: %w", err))
 	}

app/vlinsert/elasticsearch/elasticsearch_timing_test.go

@@ -10,15 +10,24 @@ import (
 )
 
 func BenchmarkReadBulkRequest(b *testing.B) {
-	b.Run("gzip:off", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, false)
+	b.Run("encoding:none", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, "")
 	})
-	b.Run("gzip:on", func(b *testing.B) {
-		benchmarkReadBulkRequest(b, true)
+	b.Run("encoding:gzip", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, "gzip")
+	})
+	b.Run("encoding:zstd", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, "zstd")
+	})
+	b.Run("encoding:deflate", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, "deflate")
+	})
+	b.Run("encoding:snappy", func(b *testing.B) {
+		benchmarkReadBulkRequest(b, "snappy")
 	})
 }
 
-func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
+func benchmarkReadBulkRequest(b *testing.B, encoding string) {
 	data := `{"create":{"_index":"filebeat-8.8.0"}}
 {"@timestamp":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
 {"create":{"_index":"filebeat-8.8.0"}}
@@ -26,8 +35,8 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 {"create":{"_index":"filebeat-8.8.0"}}
 {"message":"xyz","@timestamp":"2023-06-06T04:48:13.735Z","x":"y"}
 `
-	if isGzip {
-		data = compressData(data)
+	if encoding != "" {
+		data = compressData(data, encoding)
 	}
 	dataBytes := bytesutil.ToUnsafeBytes(data)
 
@@ -41,7 +50,7 @@ func benchmarkReadBulkRequest(b *testing.B, isGzip bool) {
 		r := &bytes.Reader{}
 		for pb.Next() {
 			r.Reset(dataBytes)
-			_, err := readBulkRequest("test", r, isGzip, timeField, msgFields, blp)
+			_, err := readBulkRequest("test", r, encoding, timeField, msgFields, blp)
 			if err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}

app/vlinsert/insertutils/common_params.go

@@ -141,7 +141,7 @@ type LogMessageProcessor interface {
 	//
 	// If streamFields is non-nil, then the given streamFields must be used as log stream fields instead of pre-configured fields.
 	//
-	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
+	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can reuse them.
 	AddRow(timestamp int64, fields, streamFields []logstorage.Field)
 
 	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.

app/vlinsert/insertutils/timestamp.go

@@ -2,7 +2,9 @@ package insertutils
 
 import (
 	"fmt"
+	"math"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -49,6 +51,35 @@ func parseTimestamp(s string) (int64, error) {
 
 // ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
 func ParseUnixTimestamp(s string) (int64, error) {
+	if strings.IndexByte(s, '.') >= 0 {
+		// Parse timestamp as floating-point value
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil {
+			return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
+		}
+		if f < (1<<31) && f >= (-1<<31) {
+			// The timestamp is in seconds.
+			return int64(f * 1e9), nil
+		}
+		if f < 1e3*(1<<31) && f >= 1e3*(-1<<31) {
+			// The timestamp is in milliseconds.
+			return int64(f * 1e6), nil
+		}
+		if f < 1e6*(1<<31) && f >= 1e6*(-1<<31) {
+			// The timestamp is in microseconds.
+			return int64(f * 1e3), nil
+		}
+		// The timestamp is in nanoseconds
+		if f > math.MaxInt64 {
+			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
+		}
+		if f < math.MinInt64 {
+			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
+		}
+		return int64(f), nil
+	}
+
+	// Parse timestamp as integer
 	n, err := strconv.ParseInt(s, 10, 64)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)

app/vlinsert/insertutils/timestamp_test.go

@@ -6,6 +6,63 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
+func TestParseUnixTimestamp_Success(t *testing.T) {
+	f := func(s string, timestampExpected int64) {
+		t.Helper()
+
+		timestamp, err := ParseUnixTimestamp(s)
+		if err != nil {
+			t.Fatalf("unexpected error in ParseUnixTimestamp(%q): %s", s, err)
+		}
+		if timestamp != timestampExpected {
+			t.Fatalf("unexpected timestamp returned from ParseUnixTimestamp(%q); got %d; want %d", s, timestamp, timestampExpected)
+		}
+	}
+
+	f("0", 0)
+
+	// nanoseconds
+	f("-1234567890123456789", -1234567890123456789)
+	f("1234567890123456789", 1234567890123456789)
+
+	// microseconds
+	f("-1234567890123456", -1234567890123456000)
+	f("1234567890123456", 1234567890123456000)
+	f("1234567890123456.789", 1234567890123456768)
+
+	// milliseconds
+	f("-1234567890123", -1234567890123000000)
+	f("1234567890123", 1234567890123000000)
+	f("1234567890123.456", 1234567890123456000)
+
+	// seconds
+	f("-1234567890", -1234567890000000000)
+	f("1234567890", 1234567890000000000)
+	f("-1234567890.123456", -1234567890123456000)
+}
+
+func TestParseUnixTimestamp_Failure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+
+		_, err := ParseUnixTimestamp(s)
+		if err == nil {
+			t.Fatalf("expecting non-nil error in ParseUnixTimestamp(%q)", s)
+		}
+	}
+
+	// non-numeric timestamp
+	f("")
+	f("foobar")
+	f("foo.bar")
+
+	// too big timestamp
+	f("12345678901234567890")
+	f("-12345678901234567890")
+	f("12345678901234567890.235424")
+	f("-12345678901234567890.235424")
+}
+
 func TestExtractTimestampFromFields_Success(t *testing.T) {
 	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
 		t.Helper()

app/vlinsert/journald/journald.go

@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"flag"
 	"fmt"
-	"io"
 	"net/http"
 	"regexp"
 	"slices"
@@ -16,32 +15,30 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/metrics"
 )
 
-const (
-	journaldEntryMaxNameLen = 64
-)
+// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
+const journaldEntryMaxNameLen = 64
+
+var allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
 
 var (
-	bodyBufferPool                bytesutil.ByteBufferPool
-	allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
-)
-
-var (
-	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Journal fields to be used as stream fields. "+
-		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
-	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Journal fields to ignore. "+
-		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
-	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Journal field to be used as time field. "+
-		"See the list of allowed fields at https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html.")
-	journaldTenantID             = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint.")
+	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
+	journaldIgnoreFields = flagutil.NewArrayString("journald.ignoreFields", "Comma-separated list of fields to ignore for logs ingested over journald protocol. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#dropping-fields")
+	journaldTimeField = flag.String("journald.timeField", "__REALTIME_TIMESTAMP", "Field to use as a log timestamp for logs ingested via journald protocol. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
+	journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
+		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
 	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
+
+	maxRequestSize = flagutil.NewBytes("journald.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single journald request")
 )
 
 func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
@@ -89,43 +86,29 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsJournaldTotal.Inc()
 
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	reader := r.Body
-	var err error
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
-	writeconcurrencylimiter.PutReader(wcr)
-	bb := bodyBufferPool.Get()
-	defer bodyBufferPool.Put(bb)
-	if r.Header.Get("Content-Encoding") == "zstd" {
-		bb.B, err = zstd.Decompress(bb.B[:0], data)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot decompress zstd-encoded request with length %d: %s", len(data), err)
-			return
-		}
-		data = bb.B
-	}
 	cp, err := getCommonParams(r)
 	if err != nil {
+		errorsTotal.Inc()
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
 	}
 
-	lmp := cp.NewLogMessageProcessor("journald")
-	err = parseJournaldRequest(data, lmp, cp)
-	lmp.MustClose()
+	if err := vlstorage.CanWriteData(); err != nil {
+		errorsTotal.Inc()
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	encoding := r.Header.Get("Content-Encoding")
+	err = common.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
+		lmp := cp.NewLogMessageProcessor("journald")
+		err := parseJournaldRequest(data, lmp, cp)
+		lmp.MustClose()
+		return err
+	})
 	if err != nil {
 		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse Journald protobuf request: %s", err)
+		httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
 		return
 	}
 
@@ -198,7 +181,7 @@ func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *
 			if err != nil {
 				return fmt.Errorf("failed to extract binary field %q value size: %w", name, err)
 			}
-			// skip binary data sise
+			// skip binary data size
 			data = data[idx:]
 			if size == 0 {
 				return fmt.Errorf("unexpected zero binary data size decoded %d", size)
@@ -218,7 +201,6 @@ func parseJournaldRequest(data []byte, lmp insertutils.LogMessageProcessor, cp *
 			}
 			data = data[1:]
 		}
-		// https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
 		if len(name) > journaldEntryMaxNameLen {
 			return fmt.Errorf("journald entry name should not exceed %d symbols, got: %q", journaldEntryMaxNameLen, name)
 		}

app/vlinsert/jsonline/jsonline.go

@@ -38,16 +38,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			logger.Errorf("cannot read gzipped jsonline request: %s", err)
-			return
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
+	encoding := r.Header.Get("Content-Encoding")
+	reader, err := common.GetUncompressedReader(r.Body, encoding)
+	if err != nil {
+		logger.Errorf("cannot decode jsonline request: %s", err)
+		return
 	}
+	defer common.PutUncompressedReader(reader)
 
 	lmp := cp.NewLogMessageProcessor("jsonline")
 	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)

app/vlinsert/loki/loki.go

@@ -1,12 +1,18 @@
 package loki
 
 import (
+	"flag"
+	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
+var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whether to disable automatic parsing of JSON-encoded log fields inside Loki log message into distinct log fields")
+
 // RequestHandler processes Loki insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
@@ -35,7 +41,16 @@ func handleInsert(r *http.Request, w http.ResponseWriter) {
 	}
 }
 
-func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
+type commonParams struct {
+	cp *insertutils.CommonParams
+
+	// Whether to parse JSON inside plaintext log message.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8486
+	parseMessage bool
+}
+
+func getCommonParams(r *http.Request) (*commonParams, error) {
 	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
 		return nil, err
@@ -55,5 +70,17 @@ func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) {
 
 	}
 
-	return cp, nil
+	parseMessage := !*disableMessageParsing
+	if rv := httputils.GetRequestValue(r, "disable_message_parsing", "VL-Loki-Disable-Message-Parsing"); rv != "" {
+		bv, err := strconv.ParseBool(rv)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse dusable_message_parsing=%q: %s", rv, err)
+		}
+		parseMessage = !bv
+	}
+
+	return &commonParams{
+		cp:           cp,
+		parseMessage: parseMessage,
+	}, nil
 }

app/vlinsert/loki/loki_json.go

@@ -2,10 +2,7 @@ package loki
 
 import (
 	"fmt"
-	"io"
-	"math"
 	"net/http"
-	"strconv"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -14,35 +11,19 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 )
 
+var maxRequestSize = flagutil.NewBytes("loki.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single Loki request")
+
 var parserPool fastjson.ParserPool
 
 func handleJSON(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsJSONTotal.Inc()
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
@@ -53,12 +34,17 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	lmp := cp.NewLogMessageProcessor("loki_json")
-	useDefaultStreamFields := len(cp.StreamFields) == 0
-	err = parseJSONRequest(data, lmp, useDefaultStreamFields)
-	lmp.MustClose()
+
+	encoding := r.Header.Get("Content-Encoding")
+	err = common.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
+		lmp := cp.cp.NewLogMessageProcessor("loki_json")
+		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
+		err := parseJSONRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
+		lmp.MustClose()
+		return err
+	})
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki json request: %s; data=%s", err, data)
+		httpserver.Errorf(w, r, "cannot read Loki json data: %s", err)
 		return
 	}
 
@@ -66,6 +52,9 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 	// There is no need in updating requestJSONDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	requestJSONDuration.UpdateDuration(startTime)
+
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
+	w.WriteHeader(http.StatusNoContent)
 }
 
 var (
@@ -73,9 +62,10 @@ var (
 	requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
-func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
+func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
+
 	v, err := p.ParseBytes(data)
 	if err != nil {
 		return fmt.Errorf("cannot parse JSON request body: %w", err)
@@ -90,11 +80,20 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 		return fmt.Errorf("`streams` item in the parsed JSON must contain an array; got %q", streamsV)
 	}
 
+	fields := getFields()
+	defer putFields(fields)
+
+	var msgParser *logstorage.JSONParser
+	if parseMessage {
+		msgParser = logstorage.GetJSONParser()
+		defer logstorage.PutJSONParser(msgParser)
+	}
+
 	currentTimestamp := time.Now().UnixNano()
-	var commonFields []logstorage.Field
+
 	for _, stream := range streams {
 		// populate common labels from `stream` dict
-		commonFields = commonFields[:0]
+		fields.fields = fields.fields[:0]
 		labelsV := stream.Get("stream")
 		var labels *fastjson.Object
 		if labelsV != nil {
@@ -110,7 +109,7 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 				err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
 				return
 			}
-			commonFields = append(commonFields, logstorage.Field{
+			fields.fields = append(fields.fields, logstorage.Field{
 				Name:  bytesutil.ToUnsafeString(k),
 				Value: bytesutil.ToUnsafeString(vStr),
 			})
@@ -129,8 +128,10 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 			return fmt.Errorf("`values` item in the parsed JSON must contain an array; got %q", linesV)
 		}
 
-		fields := commonFields
+		commonFieldsLen := len(fields.fields)
 		for _, line := range lines {
+			fields.fields = fields.fields[:commonFieldsLen]
+
 			lineA, err := line.Array()
 			if err != nil {
 				return fmt.Errorf("unexpected contents of `values` item; want array; got %q", line)
@@ -152,17 +153,6 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 				ts = currentTimestamp
 			}
 
-			// parse log message
-			msg, err := lineA[1].StringBytes()
-			if err != nil {
-				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
-			}
-
-			fields = append(fields[:len(commonFields)], logstorage.Field{
-				Name:  "_msg",
-				Value: bytesutil.ToUnsafeString(msg),
-			})
-
 			// parse structured metadata - see https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
 			if len(lineA) > 2 {
 				structuredMetadata, err := lineA[2].Object()
@@ -177,7 +167,7 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 						return
 					}
 
-					fields = append(fields, logstorage.Field{
+					fields.fields = append(fields.fields, logstorage.Field{
 						Name:  bytesutil.ToUnsafeString(k),
 						Value: bytesutil.ToUnsafeString(vStr),
 					})
@@ -186,39 +176,51 @@ func parseJSONRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefau
 					return fmt.Errorf("error when parsing `structuredMetadata` object: %w", err)
 				}
 			}
+
+			// parse log message
+			msg, err := lineA[1].StringBytes()
+			if err != nil {
+				return fmt.Errorf("unexpected log message type for %q; want string", lineA[1])
+			}
+			allowMsgRenaming := false
+			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, bytesutil.ToUnsafeString(msg))
+
 			var streamFields []logstorage.Field
 			if useDefaultStreamFields {
-				streamFields = commonFields
+				streamFields = fields.fields[:commonFieldsLen]
 			}
-			lmp.AddRow(ts, fields, streamFields)
+			if allowMsgRenaming {
+				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
+			}
+			lmp.AddRow(ts, fields.fields, streamFields)
 		}
 	}
 
 	return nil
 }
 
+func addMsgField(dst []logstorage.Field, msgParser *logstorage.JSONParser, msg string) ([]logstorage.Field, bool) {
+	if msgParser == nil || len(msg) < 2 || msg[0] != '{' || msg[len(msg)-1] != '}' {
+		return append(dst, logstorage.Field{
+			Name:  "_msg",
+			Value: msg,
+		}), false
+	}
+	if msgParser != nil && len(msg) >= 2 && msg[0] == '{' && msg[len(msg)-1] == '}' {
+		if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msg)); err == nil {
+			return append(dst, msgParser.Fields...), true
+		}
+	}
+	return append(dst, logstorage.Field{
+		Name:  "_msg",
+		Value: msg,
+	}), false
+}
+
 func parseLokiTimestamp(s string) (int64, error) {
 	if s == "" {
 		// Special case - an empty timestamp must be substituted with the current time by the caller.
 		return 0, nil
 	}
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		// Fall back to parsing floating-point value
-		f, err := strconv.ParseFloat(s, 64)
-		if err != nil {
-			return 0, err
-		}
-		if f > math.MaxInt64 {
-			return 0, fmt.Errorf("too big timestamp in nanoseconds: %v; mustn't exceed %v", f, int64(math.MaxInt64))
-		}
-		if f < math.MinInt64 {
-			return 0, fmt.Errorf("too small timestamp in nanoseconds: %v; must be bigger or equal to %v", f, int64(math.MinInt64))
-		}
-		n = int64(f)
-	}
-	if n < 0 {
-		return 0, fmt.Errorf("too small timestamp in nanoseconds: %d; must be bigger than 0", n)
-	}
-	return n, nil
+	return insertutils.ParseUnixTimestamp(s)
 }

app/vlinsert/loki/loki_json_test.go

@@ -11,7 +11,7 @@ func TestParseJSONRequest_Failure(t *testing.T) {
 		t.Helper()
 
 		tlp := &insertutils.TestLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, false); err == nil {
+		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
 		if err := tlp.Verify(nil, ""); err != nil {
@@ -65,7 +65,7 @@ func TestParseJSONRequest_Success(t *testing.T) {
 
 		tlp := &insertutils.TestLogMessageProcessor{}
 
-		if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
+		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
@@ -89,9 +89,9 @@ func TestParseJSONRequest_Success(t *testing.T) {
 	"label2": "value2"
 },"values":[
 	["1577836800000000001", "foo bar"],
-	["1477836900005000002", "abc"],
+	["1686026123.62", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+]}]}`, []int64{1577836800000000001, 1686026123620000000, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
 {"label1":"value1","label2":"value2","_msg":"abc"}
 {"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
@@ -122,6 +122,48 @@ func TestParseJSONRequest_Success(t *testing.T) {
 {"x":"y","_msg":"yx"}`)
 
 	// values with metadata
-	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar","metadata_1":"md_value"}`)
+	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {"metadata_1": "md_value"}]]}]}`, []int64{1577836800000000001}, `{"metadata_1":"md_value","_msg":"foo bar"}`)
 	f(`{"streams":[{"values":[["1577836800000000001", "foo bar", {}]]}]}`, []int64{1577836800000000001}, `{"_msg":"foo bar"}`)
 }
+
+func TestParseJSONRequest_ParseMessage(t *testing.T) {
+	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		tlp := &insertutils.TestLogMessageProcessor{}
+
+		if err := parseJSONRequest([]byte(s), tlp, msgFields, false, true); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	f(`{
+	"streams": [
+		{
+			"stream": {
+				"foo": "bar",
+				"a": "b"
+			},
+			"values": [
+				["1577836800000000001", "{\"user_id\":\"123\"}"],
+				["1577836900005000002", "abc", {"trace_id":"pqw"}],
+				["1577836900005000003", "{def}"]
+			]
+		},
+		{
+			"stream": {
+				"x": "y"
+			},
+			"values": [
+				["1877836900005000004", "{\"trace_id\":\"111\",\"parent_id\":\"abc\"}"]
+			]
+		}
+	]
+}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
+{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
+{"foo":"bar","a":"b","_msg":"{def}"}
+{"x":"y","_msg":"111","parent_id":"abc"}`)
+}

app/vlinsert/loki/loki_json_timing_test.go

@@ -28,7 +28,7 @@ func benchmarkParseJSONRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		data := getJSONBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseJSONRequest(data, blp, false); err != nil {
+			if err := parseJSONRequest(data, blp, nil, false, true); err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}

app/vlinsert/loki/loki_protobuf.go

@@ -2,7 +2,6 @@ package loki
 
 import (
 	"fmt"
-	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -11,29 +10,19 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 var (
-	bytesBufPool bytesutil.ByteBufferPool
 	pushReqsPool sync.Pool
 )
 
 func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsProtobufTotal.Inc()
-	wcr := writeconcurrencylimiter.GetReader(r.Body)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
 
 	cp, err := getCommonParams(r)
 	if err != nil {
@@ -44,12 +33,22 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	lmp := cp.NewLogMessageProcessor("loki_protobuf")
-	useDefaultStreamFields := len(cp.StreamFields) == 0
-	err = parseProtobufRequest(data, lmp, useDefaultStreamFields)
-	lmp.MustClose()
+
+	encoding := r.Header.Get("Content-Encoding")
+	if encoding == "" {
+		// Loki protocol uses snappy compression by default.
+		// See https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs
+		encoding = "snappy"
+	}
+	err = common.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
+		lmp := cp.cp.NewLogMessageProcessor("loki_protobuf")
+		useDefaultStreamFields := len(cp.cp.StreamFields) == 0
+		err := parseProtobufRequest(data, lmp, cp.cp.MsgFields, useDefaultStreamFields, cp.parseMessage)
+		lmp.MustClose()
+		return err
+	})
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
+		httpserver.Errorf(w, r, "cannot read Loki protobuf data: %s", err)
 		return
 	}
 
@@ -57,6 +56,9 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	// There is no need in updating requestProtobufDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
 	requestProtobufDuration.UpdateDuration(startTime)
+
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8505
+	w.WriteHeader(http.StatusNoContent)
 }
 
 var (
@@ -64,20 +66,11 @@ var (
 	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
-func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDefaultStreamFields bool) error {
-	bb := bytesBufPool.Get()
-	defer bytesBufPool.Put(bb)
-
-	buf, err := snappy.Decode(bb.B[:cap(bb.B)], data)
-	if err != nil {
-		return fmt.Errorf("cannot decode snappy-encoded request body: %w", err)
-	}
-	bb.B = buf
-
+func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err = req.UnmarshalProtobuf(bb.B)
+	err := req.UnmarshalProtobuf(data)
 	if err != nil {
 		return fmt.Errorf("cannot parse request body: %w", err)
 	}
@@ -85,8 +78,15 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
 	fields := getFields()
 	defer putFields(fields)
 
+	var msgParser *logstorage.JSONParser
+	if parseMessage {
+		msgParser = logstorage.GetJSONParser()
+		defer logstorage.PutJSONParser(msgParser)
+	}
+
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
+
 	for i := range streams {
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
@@ -109,10 +109,8 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
 				})
 			}
 
-			fields.fields = append(fields.fields, logstorage.Field{
-				Name:  "_msg",
-				Value: e.Line,
-			})
+			allowMsgRenaming := false
+			fields.fields, allowMsgRenaming = addMsgField(fields.fields, msgParser, e.Line)
 
 			ts := e.Timestamp.UnixNano()
 			if ts == 0 {
@@ -123,6 +121,9 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useD
 			if useDefaultStreamFields {
 				streamFields = fields.fields[:commonFieldsLen]
 			}
+			if allowMsgRenaming {
+				logstorage.RenameField(fields.fields[commonFieldsLen:], msgFields, "_msg")
+			}
 			lmp.AddRow(ts, fields.fields, streamFields)
 		}
 	}

app/vlinsert/loki/loki_protobuf_test.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/golang/snappy"
 )
 
 type testLogMessageProcessor struct {
@@ -53,7 +52,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 		t.Helper()
 
 		tlp := &testLogMessageProcessor{}
-		if err := parseJSONRequest([]byte(s), tlp, false); err != nil {
+		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if len(tlp.pr.Streams) != len(timestampsExpected) {
@@ -61,10 +60,9 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 		}
 
 		data := tlp.pr.MarshalProtobuf(nil)
-		encodedData := snappy.Encode(nil, data)
 
 		tlp2 := &insertutils.TestLogMessageProcessor{}
-		if err := parseProtobufRequest(encodedData, tlp2, false); err != nil {
+		if err := parseProtobufRequest(data, tlp2, nil, false, false); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
@@ -90,7 +88,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 	["1577836800000000001", "foo bar"],
 	["1477836900005000002", "abc"],
 	["147.78369e9", "foobar"]
-]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
+]}]}`, []int64{1577836800000000001, 1477836900005000002, 147783690000000000}, `{"label1":"value1","label2":"value2","_msg":"foo bar"}
 {"label1":"value1","label2":"value2","_msg":"abc"}
 {"label1":"value1","label2":"value2","_msg":"foobar"}`)
 
@@ -121,6 +119,57 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 {"x":"y","_msg":"yx"}`)
 }
 
+func TestParseProtobufRequest_ParseMessage(t *testing.T) {
+	f := func(s string, msgFields []string, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+
+		tlp := &testLogMessageProcessor{}
+		if err := parseJSONRequest([]byte(s), tlp, nil, false, false); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if len(tlp.pr.Streams) != len(timestampsExpected) {
+			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), len(timestampsExpected))
+		}
+
+		data := tlp.pr.MarshalProtobuf(nil)
+
+		tlp2 := &insertutils.TestLogMessageProcessor{}
+		if err := parseProtobufRequest(data, tlp2, msgFields, false, true); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if err := tlp2.Verify(timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	f(`{
+	"streams": [
+		{
+			"stream": {
+				"foo": "bar",
+				"a": "b"
+			},
+			"values": [
+				["1577836800000000001", "{\"user_id\":\"123\"}"],
+				["1577836900005000002", "abc", {"trace_id":"pqw"}],
+				["1577836900005000003", "{def}"]
+			]
+		},
+		{
+			"stream": {
+				"x": "y"
+			},
+			"values": [
+				["1877836900005000004", "{\"trace_id\":\"432\",\"parent_id\":\"qwerty\"}"]
+			]
+		}
+	]
+}`, []string{"a", "trace_id"}, []int64{1577836800000000001, 1577836900005000002, 1577836900005000003, 1877836900005000004}, `{"foo":"bar","a":"b","user_id":"123"}
+{"foo":"bar","a":"b","trace_id":"pqw","_msg":"abc"}
+{"foo":"bar","a":"b","_msg":"{def}"}
+{"x":"y","_msg":"432","parent_id":"qwerty"}`)
+}
+
 func TestParsePromLabels_Success(t *testing.T) {
 	f := func(s string) {
 		t.Helper()

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -6,8 +6,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang/snappy"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
@@ -31,7 +29,7 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	b.RunParallel(func(pb *testing.PB) {
 		body := getProtobufBody(streams, rows, labels)
 		for pb.Next() {
-			if err := parseProtobufRequest(body, blp, false); err != nil {
+			if err := parseProtobufRequest(body, blp, nil, false, true); err != nil {
 				panic(fmt.Errorf("unexpected error: %w", err))
 			}
 		}
@@ -78,8 +76,5 @@ func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
 		Streams: streams,
 	}
 
-	body := pr.MarshalProtobuf(nil)
-	encodedBody := snappy.Encode(nil, body)
-
-	return encodedBody
+	return pr.MarshalProtobuf(nil)
 }

app/vlinsert/opentelemetry/opentelemetry.go

@@ -2,21 +2,22 @@ package opentelemetry
 
 import (
 	"fmt"
-	"io"
 	"net/http"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
+var maxRequestSize = flagutil.NewBytes("opentelemetry.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single OpenTelemetry request")
+
 // RequestHandler processes Opentelemetry insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
@@ -37,24 +38,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
 	requestsProtobufTotal.Inc()
-	reader := r.Body
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		zr, err := common.GetGzipReader(reader)
-		if err != nil {
-			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
-			return
-		}
-		defer common.PutGzipReader(zr)
-		reader = zr
-	}
-
-	wcr := writeconcurrencylimiter.GetReader(reader)
-	data, err := io.ReadAll(wcr)
-	writeconcurrencylimiter.PutReader(wcr)
-	if err != nil {
-		httpserver.Errorf(w, r, "cannot read request body: %s", err)
-		return
-	}
 
 	cp, err := insertutils.GetCommonParams(r)
 	if err != nil {
@@ -66,12 +49,16 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		return
 	}
 
-	lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf")
-	useDefaultStreamFields := len(cp.StreamFields) == 0
-	err = pushProtobufRequest(data, lmp, useDefaultStreamFields)
-	lmp.MustClose()
+	encoding := r.Header.Get("Content-Encoding")
+	err = common.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
+		lmp := cp.NewLogMessageProcessor("opentelelemtry_protobuf")
+		useDefaultStreamFields := len(cp.StreamFields) == 0
+		err := pushProtobufRequest(data, lmp, useDefaultStreamFields)
+		lmp.MustClose()
+		return err
+	})
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
+		httpserver.Errorf(w, r, "cannot read OpenTelemetry protocol data: %s", err)
 		return
 	}
 

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -24,6 +24,7 @@ func TestPushProtoOk(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+
 	// single line without resource attributes
 	f([]pb.ResourceLogs{
 		{
@@ -39,6 +40,7 @@ func TestPushProtoOk(t *testing.T) {
 		[]int64{1234},
 		`{"_msg":"log-line-message","severity":"Trace"}`,
 	)
+
 	// multi-line with resource attributes
 	f([]pb.ResourceLogs{
 		{
@@ -106,19 +108,21 @@ func TestPushProtoOk(t *testing.T) {
 				{
 					LogRecords: []pb.LogRecord{
 						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
-						{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
+						{TraceID: "1234", SpanID: "45", ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
+						{TraceID: "4bf92f3577b34da6a3ce929d0e0e4736", SpanID: "00f067aa0ba902b7", ObservedTimeUnixNano: 3333, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-2")}},
 					},
 				},
 			},
 		},
 	},
-		[]int64{1234, 1235, 2345, 2346, 2347, 2348},
+		[]int64{1234, 1235, 2345, 2346, 2347, 2348, 3333},
 		`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
 {"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Debug"}
 {"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
-{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
+{"_msg":"log-line-resource-scope-1-1-1","trace_id":"1234","span_id":"45","severity":"Info4"}
+{"_msg":"log-line-resource-scope-1-1-2","trace_id":"4bf92f3577b34da6a3ce929d0e0e4736","span_id":"00f067aa0ba902b7","severity":"Unspecified"}`,
 	)
 }
 

app/vlinsert/syslog/syslog.go

@@ -16,8 +16,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/klauspost/compress/gzip"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -274,14 +272,14 @@ func runTCPListener(addr string, argIdx int) {
 
 func checkCompressMethod(compressMethod, addr, protocol string) {
 	switch compressMethod {
-	case "", "none", "gzip", "deflate":
+	case "", "none", "zstd", "gzip", "deflate":
 		return
 	default:
-		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
+		logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'zstd', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr)
 	}
 }
 
-func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
+func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
 	gomaxprocs := cgroup.AvailableCPUs()
 	var wg sync.WaitGroup
 	localAddr := ln.LocalAddr()
@@ -314,7 +312,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 				}
 				bb.B = bb.B[:n]
 				udpRequestsTotal.Inc()
-				if err := processStream("udp", bb.NewReader(), compressMethod, useLocalTimestamp, cp); err != nil {
+				if err := processStream("udp", bb.NewReader(), encoding, useLocalTimestamp, cp); err != nil {
 					logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err)
 				}
 			}
@@ -323,7 +321,7 @@ func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod st
 	wg.Wait()
 }
 
-func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
+func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, useLocalTimestamp bool, streamFields, ignoreFields []string, extraFields []logstorage.Field) {
 	var cm ingestserver.ConnsMap
 	cm.Init("syslog")
 
@@ -354,7 +352,7 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
 		wg.Add(1)
 		go func() {
 			cp := insertutils.GetCommonParamsForSyslog(tenantID, streamFields, ignoreFields, extraFields)
-			if err := processStream("tcp", c, compressMethod, useLocalTimestamp, cp); err != nil {
+			if err := processStream("tcp", c, encoding, useLocalTimestamp, cp); err != nil {
 				logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err)
 			}
 
@@ -369,49 +367,26 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod stri
 }
 
 // processStream parses a stream of syslog messages from r and ingests them into vlstorage.
-func processStream(protocol string, r io.Reader, compressMethod string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
+func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutils.CommonParams) error {
 	if err := vlstorage.CanWriteData(); err != nil {
 		return err
 	}
 
 	lmp := cp.NewLogMessageProcessor("syslog_" + protocol)
-	err := processStreamInternal(r, compressMethod, useLocalTimestamp, lmp)
+	err := processStreamInternal(r, encoding, useLocalTimestamp, lmp)
 	lmp.MustClose()
 
 	return err
 }
 
-func processStreamInternal(r io.Reader, compressMethod string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
-	switch compressMethod {
-	case "", "none":
-	case "gzip":
-		zr, err := common.GetGzipReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read gzipped data: %w", err)
-		}
-		r = zr
-	case "deflate":
-		zr, err := common.GetZlibReader(r)
-		if err != nil {
-			return fmt.Errorf("cannot read deflated data: %w", err)
-		}
-		r = zr
-	default:
-		logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod)
+func processStreamInternal(r io.Reader, encoding string, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {
+	reader, err := common.GetUncompressedReader(r, encoding)
+	if err != nil {
+		return fmt.Errorf("cannot decode syslog data: %w", err)
 	}
+	defer common.PutUncompressedReader(reader)
 
-	err := processUncompressedStream(r, useLocalTimestamp, lmp)
-
-	switch compressMethod {
-	case "gzip":
-		zr := r.(*gzip.Reader)
-		common.PutGzipReader(zr)
-	case "deflate":
-		zr := r.(io.ReadCloser)
-		common.PutZlibReader(zr)
-	}
-
-	return err
+	return processUncompressedStream(reader, useLocalTimestamp, lmp)
 }
 
 func processUncompressedStream(r io.Reader, useLocalTimestamp bool, lmp insertutils.LogMessageProcessor) error {

app/vlogscli/json_prettifier.go

@@ -154,7 +154,7 @@ func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
 		}
 		value, ok := t.(string)
 		if !ok {
-			return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
+			return nil, fmt.Errorf("unexpected token read for object value: %v; want string", t)
 		}
 
 		fields = append(fields, logstorage.Field{

app/vlogsgenerator/main.go

@@ -31,7 +31,7 @@ var (
 	totalStreams  = flag.Int("totalStreams", 0, "The number of total log streams; if -totalStreams > -activeStreams, then some active streams are substituted with new streams "+
 		"during data generation")
 	logsPerStream     = flag.Int64("logsPerStream", 1_000, "The number of log entries to generate per each log stream. Log entries are evenly distributed between -start and -end")
-	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constaint values to generate per each log entry; "+
+	constFieldsPerLog = flag.Int("constFieldsPerLog", 3, "The number of fields with constant values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
 	varFieldsPerLog = flag.Int("varFieldsPerLog", 1, "The number of fields with variable values to generate per each log entry; "+
 		"see https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model")
@@ -177,7 +177,10 @@ func generateAndPushLogs(cfg *workerConfig, workerID int) {
 	sw := &statWriter{
 		w: pw,
 	}
-	bw := bufio.NewWriter(sw)
+
+	// The 1MB write buffer increases data ingestion performance by reducing the number of send() syscalls
+	bw := bufio.NewWriterSize(sw, 1024*1024)
+
 	doneCh := make(chan struct{})
 	go func() {
 		generateLogs(bw, workerID, cfg.activeStreams, cfg.totalStreams)

app/vlselect/vmui/assets/MetricsQL-DGylhASK.md

@@ -2349,4 +2349,4 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
   For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
   This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+  starting from [`v1.102.0-rc2` release](https://docs.victoriametrics.com/changelog/changelog_2024/#v11020-rc2).

app/vlselect/vmui/index.html

@@ -35,10 +35,10 @@
   <meta property="og:title" content="UI for VictoriaLogs">
   <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
   <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-DuTUAk-m.js"></script>
+  <script type="module" crossorigin src="./assets/index-C68hz-qY.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-CEiptoJw.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-B_R5bdPN.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmagent/graphite/request_handler.go

@@ -21,7 +21,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, func(rows []parser.Row) error {
+	return stream.Parse(r, "", func(rows []parser.Row) error {
 		return insertRows(nil, rows)
 	})
 }

app/vmagent/influx/request_handler.go

@@ -35,8 +35,8 @@ var (
 // InsertHandlerForReader processes remote write for influx line protocol.
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
-func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, true, isGzipped, "", "", func(db string, rows []parser.Row) error {
+func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
+	return stream.Parse(r, encoding, true, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, nil)
 	})
 }
@@ -49,13 +49,13 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	isStreamMode := req.Header.Get("Stream-Mode") == "1"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	isStreamMode := req.Header.Get("Stream-Mode") == "1"
+	return stream.Parse(req.Body, encoding, isStreamMode, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }

app/vmagent/main.go

@@ -148,7 +148,7 @@ func main() {
 	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
 		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
-			return influx.InsertHandlerForReader(nil, r, false)
+			return influx.InsertHandlerForReader(nil, r, "")
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {

app/vmagent/native/request_handler.go

@@ -29,8 +29,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(block *stream.Block) error {
 		return insertRows(at, block, extraLabels)
 	})
 }

app/vmagent/newrelic/request_handler.go

@@ -28,9 +28,8 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	ce := req.Header.Get("Content-Encoding")
-	isGzip := ce == "gzip"
-	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []newrelic.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmagent/opentelemetry/request_handler.go

@@ -27,7 +27,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	encoding := req.Header.Get("Content-Encoding")
 	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
 		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
@@ -36,7 +36,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})
 }

app/vmagent/prometheusimport/request_handler.go

@@ -31,8 +31,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, defaultTimestamp, encoding, true, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vmagent/prometheusimport/request_handler_test.go

@@ -26,7 +26,7 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unxepected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {

app/vmagent/remotewrite/pendingseries.go

@@ -118,7 +118,7 @@ type writeRequest struct {
 }
 
 func (wr *writeRequest) reset() {
-	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are re-used.
+	// Do not reset lastFlushTime, fq, isVMRemoteWrite, significantFigures and roundDigits, since they are reused.
 
 	wr.wr.Timeseries = nil
 

app/vmagent/remotewrite/streamaggr.go

@@ -56,7 +56,7 @@ var (
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
 	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
-		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
+		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving buffered delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrDropInputLabels = flagutil.NewArrayString("remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
 		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
@@ -131,7 +131,7 @@ func reloadStreamAggrConfigGlobal() {
 func initStreamAggrConfigGlobal() {
 	sas, err := newStreamAggrConfigGlobal()
 	if err != nil {
-		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
+		logger.Fatalf("cannot initialize global stream aggregators: %s", err)
 	}
 	if sas != nil {
 		filePath := sas.FilePath()

app/vmagent/vmimport/request_handler.go

@@ -30,8 +30,8 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	})
 }

app/vmalert/config/config.go

@@ -88,6 +88,9 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 	if g.EvalOffset.Duration() > g.Interval.Duration() {
 		return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
 	}
+	if g.EvalOffset != nil && g.EvalDelay != nil {
+		return fmt.Errorf("eval_offset cannot be used with eval_delay")
+	}
 	if g.Limit < 0 {
 		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", g.Limit)
 	}

app/vmalert/config/testdata/dir/rules0-bad.rules

@@ -9,7 +9,7 @@ groups:
         annotations:
           summary: "{{ }}"
           description: "{{$labels}}"
-      - alert: UnkownAnnotationsFunction
+      - alert: UnknownAnnotationsFunction
         for: 5m
         expr: vm_rows > 0
         labels:

app/vmalert/config/testdata/dir/rules2-bad.rules

@@ -1,7 +1,7 @@
 groups:
   - name: group
     rules:
-      - alert: UnkownLabelFunction
+      - alert: UnknownLabelFunction
         for: 5m
         expr: vm_rows > 0
         labels:

app/vmalert/notifier/alertmanager.go

@@ -37,8 +37,9 @@ type AlertManager struct {
 type notifierMetrics struct {
 	set *metrics.Set
 
-	alertsSent       *metrics.Counter
-	alertsSendErrors *metrics.Counter
+	alertsSent         *metrics.Counter
+	alertsSendErrors   *metrics.Counter
+	alertsSendDuration *metrics.Histogram
 }
 
 func newNotifierMetrics(addr string) *notifierMetrics {
@@ -46,9 +47,10 @@ func newNotifierMetrics(addr string) *notifierMetrics {
 	metrics.RegisterSet(set)
 
 	return &notifierMetrics{
-		set:              set,
-		alertsSent:       set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
-		alertsSendErrors: set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+		set:                set,
+		alertsSent:         set.NewCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
+		alertsSendErrors:   set.NewCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+		alertsSendDuration: set.NewHistogram(fmt.Sprintf("vmalert_alerts_send_duration_seconds{addr=%q}", addr)),
 	}
 }
 
@@ -72,7 +74,9 @@ func (am AlertManager) Addr() string {
 // Send an alert or resolve message
 func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	am.metrics.alertsSent.Add(len(alerts))
+	startTime := time.Now()
 	err := am.send(ctx, alerts, headers)
+	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
 		am.metrics.alertsSendErrors.Add(len(alerts))
 	}

app/vmalert/rule/alerting_test.go

@@ -930,7 +930,7 @@ func TestGroup_Restore(t *testing.T) {
 			},
 		})
 
-	// one active alert with restore labels missmatch
+	// one active alert with restore labels mismatch
 	ts = time.Now().Truncate(time.Hour)
 	fqr.Set(`default_rollup(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
 		stateMetric("foo", ts, "env", "dev", "team", "foo"))
@@ -975,7 +975,7 @@ func TestAlertingRule_Exec_Negative(t *testing.T) {
 		t.Fatalf("expected to get err; got nil")
 	}
 	if !strings.Contains(err.Error(), expErr) {
-		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+		t.Fatalf("expected to get err %q; got %q instead", expErr, err)
 	}
 }
 

app/vmalert/rule/group.go

@@ -27,14 +27,15 @@ import (
 var (
 	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
 		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
-	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier")
+	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
 	maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maxiMum duration for automatic alert expiration, "+
 		"which by default is 4 times evaluationInterval of the parent group")
-	evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the `time` parameter for rule evaluation requests to compensate intentional data delay from the datasource."+
-		"Normally, should be equal to `-search.latencyOffset` (cmd-line flag configured for VictoriaMetrics single-node or vmselect).")
+	evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the `time` parameter for rule evaluation requests to compensate intentional data delay from the datasource. "+
+		"Normally, should be equal to `-search.latencyOffset` (cmd-line flag configured for VictoriaMetrics single-node or vmselect). "+
+		"This doesn't apply to groups with eval_offset specified.")
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
-	remoteReadLookBack     = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
-		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+	remoteReadLookBack     = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries. "+
+		"For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 )
 
 // Group is an entity for grouping rules
@@ -88,10 +89,10 @@ func newGroupMetrics(g *Group) *groupMetrics {
 	m.set = metrics.NewSet()
 
 	labels := fmt.Sprintf(`group=%q, file=%q`, g.Name, g.File)
-	m.iterationTotal = m.set.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
-	m.iterationDuration = m.set.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
-	m.iterationMissed = m.set.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
-	m.iterationInterval = m.set.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
+	m.iterationTotal = m.set.NewCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = m.set.NewSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationMissed = m.set.NewCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	m.iterationInterval = m.set.NewGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
 		g.mu.RLock()
 		i := g.Interval.Seconds()
 		g.mu.RUnlock()
@@ -375,6 +376,7 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 		}
 
 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
+		// adjust request timestamp using evalDelay and evalAlignment if necessary
 		ts = g.adjustReqTimestamp(ts)
 		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration, g.Limit)
 		for err := range errs {
@@ -468,10 +470,18 @@ func (g *Group) DeepCopy() *Group {
 	return &newG
 }
 
-// delayBeforeStart returns a duration on the interval between [ts..ts+interval].
-// delayBeforeStart accounts for `offset`, so returned duration should be always
-// bigger than the `offset`.
+// if offset is specified, delayBeforeStart returns a duration to help aligning timestamp with offset;
+// otherwise, it returns a random duration between [0..interval] based on group key.
 func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *time.Duration) time.Duration {
+	if offset != nil {
+		currentOffsetPoint := ts.Truncate(interval).Add(*offset)
+		if currentOffsetPoint.Before(ts) {
+			// wait until the next offset point
+			return currentOffsetPoint.Add(interval).Sub(ts)
+		}
+		return currentOffsetPoint.Sub(ts)
+	}
+
 	var randSleep time.Duration
 	randSleep = time.Duration(float64(interval) * (float64(key) / (1 << 64)))
 	sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())
@@ -479,15 +489,6 @@ func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *
 		randSleep += interval
 	}
 	randSleep -= sleepOffset
-	// check if `ts` after randSleep is before `offset`,
-	// if it is, add extra eval_offset to randSleep.
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3409.
-	if offset != nil {
-		tmpEvalTS := ts.Add(randSleep)
-		if tmpEvalTS.Before(tmpEvalTS.Truncate(interval).Add(*offset)) {
-			randSleep += *offset
-		}
-	}
 	return randSleep
 }
 
@@ -593,26 +594,14 @@ func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Du
 }
 
 func (g *Group) adjustReqTimestamp(timestamp time.Time) time.Time {
+	// if `eval_offset` is specified, timestamp is already aligned with offset, do nothing
 	if g.EvalOffset != nil {
-		// calculate the min timestamp on the evaluationInterval
-		intervalStart := timestamp.Truncate(g.Interval)
-		ts := intervalStart.Add(*g.EvalOffset)
-		if timestamp.Before(ts) {
-			// if passed timestamp is before the expected evaluation offset,
-			// then we should adjust it to the previous evaluation round.
-			// E.g. request with evaluationInterval=1h and evaluationOffset=30m
-			// was evaluated at 11:20. Then the timestamp should be adjusted
-			// to 10:30, to the previous evaluationInterval.
-			return ts.Add(-g.Interval)
-		}
-		// when `eval_offset` is using, ts shouldn't be effect by `eval_alignment` and `eval_delay`
-		// since it should be always aligned.
-		return ts
+		return timestamp
 	}
 
 	timestamp = timestamp.Add(-g.getEvalDelay())
 
-	// always apply the alignment as a last step
+	// apply the alignment as the last step
 	if g.evalAlignment == nil || *g.evalAlignment {
 		// align query time with interval to get similar result with grafana when plotting time series.
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5049

app/vmalert/rule/group_test.go

@@ -533,34 +533,14 @@ func TestGroupStartDelay(t *testing.T) {
 	f("2023-01-01T00:00:29.000+00:00", "2023-01-01T00:00:30.000+00:00")
 	f("2023-01-01T00:00:31.000+00:00", "2023-01-01T00:05:30.000+00:00")
 
-	// test group with offset smaller than above fixed randSleep,
-	// this way randSleep will always be enough
-	offset := 20 * time.Second
+	// test group with offset
+	offset := 3 * time.Minute
 	g.EvalOffset = &offset
 
-	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:00:30.000+00:00")
-	f("2023-01-01T00:00:29.000+00:00", "2023-01-01T00:00:30.000+00:00")
-	f("2023-01-01T00:00:31.000+00:00", "2023-01-01T00:05:30.000+00:00")
-
-	// test group with offset bigger than above fixed randSleep,
-	// this way offset will be added to delay
-	offset = 3 * time.Minute
-	g.EvalOffset = &offset
-
-	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:03:30.000+00:00")
-	f("2023-01-01T00:00:29.000+00:00", "2023-01-01T00:03:30.000+00:00")
-	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:08:30.000+00:00")
-	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:30.000+00:00")
-	f("2023-01-01T00:07:30.000+00:00", "2023-01-01T00:13:30.000+00:00")
-
-	offset = 10 * time.Minute
-	g.EvalOffset = &offset
-	// interval of 1h and key generate a static delay of 6m
-	g.Interval = time.Hour
-
-	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:16:00.000+00:00")
-	f("2023-01-01T00:05:00.000+00:00", "2023-01-01T00:16:00.000+00:00")
-	f("2023-01-01T00:30:00.000+00:00", "2023-01-01T01:16:00.000+00:00")
+	f("2023-01-01T00:00:15.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
+	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
 }
 
 func TestGetPrometheusReqTimestamp(t *testing.T) {
@@ -590,17 +570,11 @@ func TestGetPrometheusReqTimestamp(t *testing.T) {
 		evalAlignment: &disableAlign,
 	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:10:30+00:00")
 
-	// with eval_offset, find previous offset point + default evalDelay
+	// with eval_offset
 	f(&Group{
 		EvalOffset: &offset,
 		Interval:   time.Hour,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T10:30:00+00:00")
-
-	// with eval_offset + default evalDelay
-	f(&Group{
-		EvalOffset: &offset,
-		Interval:   time.Hour,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:30:00+00:00")
+	}, "2023-08-28T11:30:00+00:00", "2023-08-28T11:30:00+00:00")
 
 	// 1h interval with eval_delay
 	f(&Group{

app/vmalert/rule/recording_test.go

@@ -194,7 +194,7 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 			t.Fatalf("unexpected RecordingRule.execRange error: %s", err)
 		}
 		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries missmatch: %s", err)
+			t.Fatalf("timeseries mismatch: %s", err)
 		}
 	}
 
@@ -384,7 +384,7 @@ func TestRecordingRuleExec_Negative(t *testing.T) {
 		t.Fatalf("expected to get err; got nil")
 	}
 	if !strings.Contains(err.Error(), expErr) {
-		t.Fatalf("expected to get err %q; got %q insterad", expErr, err)
+		t.Fatalf("expected to get err %q; got %q instead", expErr, err)
 	}
 
 	fq.Reset()

app/vmalert/web.qtpl

@@ -97,7 +97,7 @@ btn-primary
                     <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                      {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
                      {% if rNoMatch[g.ID] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d rNoMatch[g.ID] %}</span> {% endif %}
-                    <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.ID] %}</span>
+                    <span class="badge bg-success" title="Number of rules with status Ok">{%d rOk[g.ID] %}</span>
                     <p class="fs-6 fw-lighter">{%s g.File %}</p>
                     {% if len(g.Params) > 0 %}
                         <div class="fs-6 fw-lighter">Extra params

app/vmalert/web.qtpl.go

@@ -355,7 +355,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, originGroups [
 			}
 //line app/vmalert/web.qtpl:99
 			qw422016.N().S(`
-                    <span class="badge bg-success" title="Number of rules withs status Ok">`)
+                    <span class="badge bg-success" title="Number of rules with status Ok">`)
 //line app/vmalert/web.qtpl:100
 			qw422016.N().D(rOk[g.ID])
 //line app/vmalert/web.qtpl:100

app/vmauth/auth_config.go

@@ -141,7 +141,7 @@ func (h *Header) UnmarshalYAML(f func(any) error) error {
 
 	n := strings.IndexByte(s, ':')
 	if n < 0 {
-		return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
+		return fmt.Errorf("missing separator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
 	}
 	h.Name = strings.TrimSpace(s[:n])
 	h.Value = strings.TrimSpace(s[n+1:])
@@ -173,7 +173,7 @@ type URLMap struct {
 	// DiscoverBackendIPs instructs discovering URLPrefix backend IPs via DNS.
 	DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"`
 
-	// HeadersConf is the config for augumenting request and response headers.
+	// HeadersConf is the config for augmenting request and response headers.
 	HeadersConf HeadersConf `yaml:",inline"`
 
 	// RetryStatusCodes is the list of response status codes used for retrying requests.
@@ -438,7 +438,7 @@ func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 		return bu
 	}
 
-	// Slow path - the first url is temporarily unavailabel. Fall back to the remaining urls.
+	// Slow path - the first url is temporarily unavailable. Fall back to the remaining urls.
 	for i := 1; i < len(bus); i++ {
 		if !bus[i].isBroken() {
 			bu = bus[i]

app/vmctl/main.go

@@ -293,7 +293,7 @@ func main() {
 						auth.WithBearer(c.String(vmNativeSrcBearerToken)),
 						auth.WithHeaders(c.String(vmNativeSrcHeaders)))
 					if err != nil {
-						return fmt.Errorf("error initilize auth config for source: %s", srcAddr)
+						return fmt.Errorf("error initialize auth config for source: %s", srcAddr)
 					}
 
 					// create TLS config
@@ -320,7 +320,7 @@ func main() {
 						auth.WithBearer(c.String(vmNativeDstBearerToken)),
 						auth.WithHeaders(c.String(vmNativeDstHeaders)))
 					if err != nil {
-						return fmt.Errorf("error initilize auth config for destination: %s", dstAddr)
+						return fmt.Errorf("error initialize auth config for destination: %s", dstAddr)
 					}
 
 					// create TLS config
@@ -384,7 +384,10 @@ func main() {
 				Action: func(c *cli.Context) error {
 					common.StartUnmarshalWorkers()
 					blockPath := c.Args().First()
-					isBlockGzipped := c.Bool("gunzip")
+					encoding := ""
+					if c.Bool("gunzip") {
+						encoding = "gzip"
+					}
 					if len(blockPath) == 0 {
 						return cli.Exit("you must provide path for exported data block", 1)
 					}
@@ -395,7 +398,7 @@ func main() {
 					}
 					defer f.Close()
 					var blocksCount atomic.Uint64
-					if err := stream.Parse(f, isBlockGzipped, func(_ *stream.Block) error {
+					if err := stream.Parse(f, encoding, func(_ *stream.Block) error {
 						blocksCount.Add(1)
 						return nil
 					}); err != nil {

app/vmctl/testdata/servers_integration_test/remote_read_server.go

@@ -442,7 +442,7 @@ func (si *mockSamplesIterator) At() (int64, float64) {
 }
 
 func (si *mockSamplesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
-	panic("BUG: musn't be called")
+	panic("BUG: mustn't be called")
 }
 
 func (si *mockSamplesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {

app/vmctl/testdata/servers_integration_test/remote_write_server.go

@@ -212,7 +212,7 @@ func (rws *RemoteWriteServer) importNativeHandler(t *testing.T) http.Handler {
 		var gotTimeSeries []vm.TimeSeries
 		var mx sync.RWMutex
 
-		err := stream.Parse(r.Body, false, func(block *stream.Block) error {
+		err := stream.Parse(r.Body, "", func(block *stream.Block) error {
 			mn := &block.MetricName
 			var timeseries vm.TimeSeries
 			timeseries.Name = string(mn.MetricGroup)

app/vminsert/graphite/request_handler.go

@@ -19,7 +19,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol
 func InsertHandler(r io.Reader) error {
-	return stream.Parse(r, false, insertRows)
+	return stream.Parse(r, "", insertRows)
 }
 
 func insertRows(rows []parser.Row) error {

app/vminsert/influx/request_handler.go

@@ -34,7 +34,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader) error {
-	return stream.Parse(r, true, false, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, "", true, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, nil)
 	})
 }
@@ -47,13 +47,13 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	isStreamMode := req.Header.Get("Stream-Mode") == "1"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	isStreamMode := req.Header.Get("Stream-Mode") == "1"
+	return stream.Parse(req.Body, encoding, isStreamMode, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, extraLabels)
 	})
 }

app/vminsert/native/request_handler.go

@@ -25,8 +25,8 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzip := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzip, func(block *stream.Block) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(block *stream.Block) error {
 		return insertRows(block, extraLabels)
 	})
 }

app/vminsert/newrelic/request_handler.go

@@ -24,9 +24,8 @@ func InsertHandlerForHTTP(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	ce := req.Header.Get("Content-Encoding")
-	isGzip := ce == "gzip"
-	return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []newrelic.Row) error {
 		return insertRows(rows, extraLabels)
 	})
 }

app/vminsert/opentelemetry/request_handler.go

@@ -24,7 +24,7 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	encoding := req.Header.Get("Content-Encoding")
 	var processBody func([]byte) ([]byte, error)
 	if req.Header.Get("Content-Type") == "application/json" {
 		if req.Header.Get("X-Amz-Firehose-Protocol-Version") != "" {
@@ -33,7 +33,7 @@ func InsertHandler(req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, isGzipped, processBody, func(tss []prompbmarshal.TimeSeries) error {
+	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(tss, extraLabels)
 	})
 }

app/vminsert/prometheusimport/request_handler.go

@@ -28,8 +28,8 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, defaultTimestamp, encoding, true, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vminsert/vmimport/request_handler.go

@@ -28,8 +28,8 @@ func InsertHandler(req *http.Request) error {
 	if err != nil {
 		return err
 	}
-	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, isGzipped, func(rows []parser.Row) error {
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	})
 }

app/vmselect/graphite/eval.go

@@ -64,7 +64,7 @@ type series struct {
 
 	expr graphiteql.Expr
 
-	// consolidateFunc is applied to raw samples in order to generate data points algined to the given step.
+	// consolidateFunc is applied to raw samples in order to generate data points aligned to the given step.
 	// see series.consolidate() function for details.
 	consolidateFunc aggrFunc
 

app/vmselect/graphite/eval_test.go

@@ -36,7 +36,7 @@ func TestExecExprSuccess(t *testing.T) {
 		if err := compareSeries(ss, expectedSeries, expr); err != nil {
 			t.Fatalf("series mismatch for query %q: %s\ngot series\n%s\nexpected series\n%s", query, err, printSeriess(ss), printSeriess(expectedSeries))
 		}
-		// make sure ec isn't changed during query exection.
+		// make sure ec isn't changed during query execution.
 		if !reflect.DeepEqual(ec, &ecCopy) {
 			t.Fatalf("unexpected ec\ngot\n%v\nwant\n%v", &ecCopy, ec)
 		}

app/vmselect/graphite/functions.json

@@ -159,7 +159,7 @@
   "asPercent": {
     "name": "asPercent",
     "function": "asPercent(seriesList, total=None, *nodes)",
-    "description": "Calculates a percentage of the total of a wildcard series. If `total` is specified,\neach series will be calculated as a percentage of that total. If `total` is not specified,\nthe sum of all points in the wildcard series will be used instead.\n\nA list of nodes can optionally be provided, if so they will be used to match series with their\ncorresponding totals following the same logic as :py:func:`groupByNodes <groupByNodes>`.\n\nWhen passing `nodes` the `total` parameter may be a series list or `None`.  If it is `None` then\nfor each series in `seriesList` the percentage of the sum of series in that group will be returned.\n\nWhen not passing `nodes`, the `total` parameter may be a single series, reference the same number\nof series as `seriesList` or be a numeric value.\n\nExample:\n\n.. code-block:: none\n\n  # Server01 connections failed and succeeded as a percentage of Server01 connections attempted\n  &target=asPercent(Server01.connections.{failed,succeeded}, Server01.connections.attempted)\n\n  # For each server, its connections failed as a percentage of its connections attempted\n  &target=asPercent(Server*.connections.failed, Server*.connections.attempted)\n\n  # For each server, its connections failed and succeeded as a percentage of its connections attemped\n  &target=asPercent(Server*.connections.{failed,succeeded}, Server*.connections.attempted, 0)\n\n  # apache01.threads.busy as a percentage of 1500\n  &target=asPercent(apache01.threads.busy,1500)\n\n  # Server01 cpu stats as a percentage of its total\n  &target=asPercent(Server01.cpu.*.jiffies)\n\n  # cpu stats for each server as a percentage of its total\n  &target=asPercent(Server*.cpu.*.jiffies, None, 0)\n\nWhen using `nodes`, any series or totals that can't be matched will create output series with\nnames like ``asPercent(someSeries,MISSING)`` or ``asPercent(MISSING,someTotalSeries)`` and all\nvalues set to None. If desired these series can be filtered out by piping the result through\n``|exclude(\"MISSING\")`` as shown below:\n\n.. code-block:: none\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)\n\n  # will produce 3 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n  # asPercent(Server2.memory.used,MISSING) [all values will be None]\n  # asPercent(MISSING,Server3.memory.total) [all values will be None]\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)|exclude(\"MISSING\")\n\n  # will produce 1 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n\nEach node may be an integer referencing a node in the series name or a string identifying a tag.\n\n.. note::\n\n  When `total` is a seriesList, specifying `nodes` to match series with the corresponding total\n  series will increase reliability.",
+    "description": "Calculates a percentage of the total of a wildcard series. If `total` is specified,\neach series will be calculated as a percentage of that total. If `total` is not specified,\nthe sum of all points in the wildcard series will be used instead.\n\nA list of nodes can optionally be provided, if so they will be used to match series with their\ncorresponding totals following the same logic as :py:func:`groupByNodes <groupByNodes>`.\n\nWhen passing `nodes` the `total` parameter may be a series list or `None`.  If it is `None` then\nfor each series in `seriesList` the percentage of the sum of series in that group will be returned.\n\nWhen not passing `nodes`, the `total` parameter may be a single series, reference the same number\nof series as `seriesList` or be a numeric value.\n\nExample:\n\n.. code-block:: none\n\n  # Server01 connections failed and succeeded as a percentage of Server01 connections attempted\n  &target=asPercent(Server01.connections.{failed,succeeded}, Server01.connections.attempted)\n\n  # For each server, its connections failed as a percentage of its connections attempted\n  &target=asPercent(Server*.connections.failed, Server*.connections.attempted)\n\n  # For each server, its connections failed and succeeded as a percentage of its connections attempted\n  &target=asPercent(Server*.connections.{failed,succeeded}, Server*.connections.attempted, 0)\n\n  # apache01.threads.busy as a percentage of 1500\n  &target=asPercent(apache01.threads.busy,1500)\n\n  # Server01 cpu stats as a percentage of its total\n  &target=asPercent(Server01.cpu.*.jiffies)\n\n  # cpu stats for each server as a percentage of its total\n  &target=asPercent(Server*.cpu.*.jiffies, None, 0)\n\nWhen using `nodes`, any series or totals that can't be matched will create output series with\nnames like ``asPercent(someSeries,MISSING)`` or ``asPercent(MISSING,someTotalSeries)`` and all\nvalues set to None. If desired these series can be filtered out by piping the result through\n``|exclude(\"MISSING\")`` as shown below:\n\n.. code-block:: none\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)\n\n  # will produce 3 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n  # asPercent(Server2.memory.used,MISSING) [all values will be None]\n  # asPercent(MISSING,Server3.memory.total) [all values will be None]\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)|exclude(\"MISSING\")\n\n  # will produce 1 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n\nEach node may be an integer referencing a node in the series name or a string identifying a tag.\n\n.. note::\n\n  When `total` is a seriesList, specifying `nodes` to match series with the corresponding total\n  series will increase reliability.",
     "module": "graphite.render.functions",
     "group": "Combine",
     "params": [
@@ -635,7 +635,7 @@
   "pct": {
     "name": "pct",
     "function": "pct(seriesList, total=None, *nodes)",
-    "description": "Calculates a percentage of the total of a wildcard series. If `total` is specified,\neach series will be calculated as a percentage of that total. If `total` is not specified,\nthe sum of all points in the wildcard series will be used instead.\n\nA list of nodes can optionally be provided, if so they will be used to match series with their\ncorresponding totals following the same logic as :py:func:`groupByNodes <groupByNodes>`.\n\nWhen passing `nodes` the `total` parameter may be a series list or `None`.  If it is `None` then\nfor each series in `seriesList` the percentage of the sum of series in that group will be returned.\n\nWhen not passing `nodes`, the `total` parameter may be a single series, reference the same number\nof series as `seriesList` or be a numeric value.\n\nExample:\n\n.. code-block:: none\n\n  # Server01 connections failed and succeeded as a percentage of Server01 connections attempted\n  &target=asPercent(Server01.connections.{failed,succeeded}, Server01.connections.attempted)\n\n  # For each server, its connections failed as a percentage of its connections attempted\n  &target=asPercent(Server*.connections.failed, Server*.connections.attempted)\n\n  # For each server, its connections failed and succeeded as a percentage of its connections attemped\n  &target=asPercent(Server*.connections.{failed,succeeded}, Server*.connections.attempted, 0)\n\n  # apache01.threads.busy as a percentage of 1500\n  &target=asPercent(apache01.threads.busy,1500)\n\n  # Server01 cpu stats as a percentage of its total\n  &target=asPercent(Server01.cpu.*.jiffies)\n\n  # cpu stats for each server as a percentage of its total\n  &target=asPercent(Server*.cpu.*.jiffies, None, 0)\n\nWhen using `nodes`, any series or totals that can't be matched will create output series with\nnames like ``asPercent(someSeries,MISSING)`` or ``asPercent(MISSING,someTotalSeries)`` and all\nvalues set to None. If desired these series can be filtered out by piping the result through\n``|exclude(\"MISSING\")`` as shown below:\n\n.. code-block:: none\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)\n\n  # will produce 3 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n  # asPercent(Server2.memory.used,MISSING) [all values will be None]\n  # asPercent(MISSING,Server3.memory.total) [all values will be None]\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)|exclude(\"MISSING\")\n\n  # will produce 1 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n\nEach node may be an integer referencing a node in the series name or a string identifying a tag.\n\n.. note::\n\n  When `total` is a seriesList, specifying `nodes` to match series with the corresponding total\n  series will increase reliability.",
+    "description": "Calculates a percentage of the total of a wildcard series. If `total` is specified,\neach series will be calculated as a percentage of that total. If `total` is not specified,\nthe sum of all points in the wildcard series will be used instead.\n\nA list of nodes can optionally be provided, if so they will be used to match series with their\ncorresponding totals following the same logic as :py:func:`groupByNodes <groupByNodes>`.\n\nWhen passing `nodes` the `total` parameter may be a series list or `None`.  If it is `None` then\nfor each series in `seriesList` the percentage of the sum of series in that group will be returned.\n\nWhen not passing `nodes`, the `total` parameter may be a single series, reference the same number\nof series as `seriesList` or be a numeric value.\n\nExample:\n\n.. code-block:: none\n\n  # Server01 connections failed and succeeded as a percentage of Server01 connections attempted\n  &target=asPercent(Server01.connections.{failed,succeeded}, Server01.connections.attempted)\n\n  # For each server, its connections failed as a percentage of its connections attempted\n  &target=asPercent(Server*.connections.failed, Server*.connections.attempted)\n\n  # For each server, its connections failed and succeeded as a percentage of its connections attempted\n  &target=asPercent(Server*.connections.{failed,succeeded}, Server*.connections.attempted, 0)\n\n  # apache01.threads.busy as a percentage of 1500\n  &target=asPercent(apache01.threads.busy,1500)\n\n  # Server01 cpu stats as a percentage of its total\n  &target=asPercent(Server01.cpu.*.jiffies)\n\n  # cpu stats for each server as a percentage of its total\n  &target=asPercent(Server*.cpu.*.jiffies, None, 0)\n\nWhen using `nodes`, any series or totals that can't be matched will create output series with\nnames like ``asPercent(someSeries,MISSING)`` or ``asPercent(MISSING,someTotalSeries)`` and all\nvalues set to None. If desired these series can be filtered out by piping the result through\n``|exclude(\"MISSING\")`` as shown below:\n\n.. code-block:: none\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)\n\n  # will produce 3 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n  # asPercent(Server2.memory.used,MISSING) [all values will be None]\n  # asPercent(MISSING,Server3.memory.total) [all values will be None]\n\n  &target=asPercent(Server{1,2}.memory.used,Server{1,3}.memory.total,0)|exclude(\"MISSING\")\n\n  # will produce 1 output series:\n  # asPercent(Server1.memory.used,Server1.memory.total) [values will be as expected]\n\nEach node may be an integer referencing a node in the series name or a string identifying a tag.\n\n.. note::\n\n  When `total` is a seriesList, specifying `nodes` to match series with the corresponding total\n  series will increase reliability.",
     "module": "graphite.render.functions",
     "group": "Combine",
     "params": [
@@ -988,7 +988,7 @@
   "integralByInterval": {
     "name": "integralByInterval",
     "function": "integralByInterval(seriesList, intervalUnit)",
-    "description": "This will do the same as integral() funcion, except resetting the total to 0\nat the given time in the parameter \"from\"\nUseful for finding totals per hour/day/week/..\n\nExample:\n\n.. code-block:: none\n\n  &target=integralByInterval(company.sales.perMinute, \"1d\")&from=midnight-10days\n\nThis would start at zero on the left side of the graph, adding the sales each\nminute, and show the evolution of sales per day during the last 10 days.",
+    "description": "This will do the same as integral() function, except resetting the total to 0\nat the given time in the parameter \"from\"\nUseful for finding totals per hour/day/week/..\n\nExample:\n\n.. code-block:: none\n\n  &target=integralByInterval(company.sales.perMinute, \"1d\")&from=midnight-10days\n\nThis would start at zero on the left side of the graph, adding the sales each\nminute, and show the evolution of sales per day during the last 10 days.",
     "module": "graphite.render.functions",
     "group": "Transform",
     "params": [
@@ -1776,7 +1776,7 @@
   "movingAverage": {
     "name": "movingAverage",
     "function": "movingAverage(seriesList, windowSize, xFilesFactor=None)",
-    "description": "Graphs the moving average of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\naverage of the preceeding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingAverage(Server.instance01.threads.busy,10)\n  &target=movingAverage(Server.instance*.threads.idle,'5min')",
+    "description": "Graphs the moving average of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\naverage of the preceding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingAverage(Server.instance01.threads.busy,10)\n  &target=movingAverage(Server.instance*.threads.idle,'5min')",
     "module": "graphite.render.functions",
     "group": "Calculate",
     "params": [
@@ -1809,7 +1809,7 @@
   "movingMax": {
     "name": "movingMax",
     "function": "movingMax(seriesList, windowSize, xFilesFactor=None)",
-    "description": "Graphs the moving maximum of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nmaximum of the preceeding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingMax(Server.instance01.requests,10)\n  &target=movingMax(Server.instance*.errors,'5min')",
+    "description": "Graphs the moving maximum of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nmaximum of the preceding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingMax(Server.instance01.requests,10)\n  &target=movingMax(Server.instance*.errors,'5min')",
     "module": "graphite.render.functions",
     "group": "Calculate",
     "params": [
@@ -1842,7 +1842,7 @@
   "movingMedian": {
     "name": "movingMedian",
     "function": "movingMedian(seriesList, windowSize, xFilesFactor=None)",
-    "description": "Graphs the moving median of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nmedian of the preceeding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingMedian(Server.instance01.threads.busy,10)\n  &target=movingMedian(Server.instance*.threads.idle,'5min')",
+    "description": "Graphs the moving median of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nmedian of the preceding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingMedian(Server.instance01.threads.busy,10)\n  &target=movingMedian(Server.instance*.threads.idle,'5min')",
     "module": "graphite.render.functions",
     "group": "Calculate",
     "params": [
@@ -1875,7 +1875,7 @@
   "movingMin": {
     "name": "movingMin",
     "function": "movingMin(seriesList, windowSize, xFilesFactor=None)",
-    "description": "Graphs the moving minimum of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nminimum of the preceeding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingMin(Server.instance01.requests,10)\n  &target=movingMin(Server.instance*.errors,'5min')",
+    "description": "Graphs the moving minimum of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nminimum of the preceding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingMin(Server.instance01.requests,10)\n  &target=movingMin(Server.instance*.errors,'5min')",
     "module": "graphite.render.functions",
     "group": "Calculate",
     "params": [
@@ -1908,7 +1908,7 @@
   "movingSum": {
     "name": "movingSum",
     "function": "movingSum(seriesList, windowSize, xFilesFactor=None)",
-    "description": "Graphs the moving sum of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nsum of the preceeding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingSum(Server.instance01.requests,10)\n  &target=movingSum(Server.instance*.errors,'5min')",
+    "description": "Graphs the moving sum of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList followed by a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), and an xFilesFactor value to specify\nhow many points in the window must be non-null for the output to be considered valid. Graphs the\nsum of the preceding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingSum(Server.instance01.requests,10)\n  &target=movingSum(Server.instance*.errors,'5min')",
     "module": "graphite.render.functions",
     "group": "Calculate",
     "params": [
@@ -1941,7 +1941,7 @@
   "movingWindow": {
     "name": "movingWindow",
     "function": "movingWindow(seriesList, windowSize, func='average', xFilesFactor=None)",
-    "description": "Graphs a moving window function of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList, a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), a function to apply to the points\nin the window to produce the output, and an xFilesFactor value to specify how many points in the\nwindow must be non-null for the output to be considered valid. Graphs the\noutput of the function for the preceeding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingWindow(Server.instance01.threads.busy,10)\n  &target=movingWindow(Server.instance*.threads.idle,'5min','median',0.5)\n\n.. note::\n\n  `xFilesFactor` follows the same semantics as in Whisper storage schemas.  Setting it to 0 (the\n  default) means that only a single value in a given interval needs to be non-null, setting it to\n  1 means that all values in the interval must be non-null.  A setting of 0.5 means that at least\n  half the values in the interval must be non-null.",
+    "description": "Graphs a moving window function of a metric (or metrics) over a fixed number of\npast points, or a time interval.\n\nTakes one metric or a wildcard seriesList, a number N of datapoints\nor a quoted string with a length of time like '1hour' or '5min' (See ``from /\nuntil`` in the :doc:`Render API <render_api>` for examples of time formats), a function to apply to the points\nin the window to produce the output, and an xFilesFactor value to specify how many points in the\nwindow must be non-null for the output to be considered valid. Graphs the\noutput of the function for the preceding datapoints for each point on the graph.\n\nExample:\n\n.. code-block:: none\n\n  &target=movingWindow(Server.instance01.threads.busy,10)\n  &target=movingWindow(Server.instance*.threads.idle,'5min','median',0.5)\n\n.. note::\n\n  `xFilesFactor` follows the same semantics as in Whisper storage schemas.  Setting it to 0 (the\n  default) means that only a single value in a given interval needs to be non-null, setting it to\n  1 means that all values in the interval must be non-null.  A setting of 0.5 means that at least\n  half the values in the interval must be non-null.",
     "module": "graphite.render.functions",
     "group": "Calculate",
     "params": [

app/vmselect/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/stats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -29,7 +30,10 @@ import (
 )
 
 var (
-	deleteAuthKey         = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries. It could be passed via authKey query arg. It overrides -httpAuth.*")
+	deleteAuthKey                = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries. It could be passed via authKey query arg. It overrides -httpAuth.*")
+	metricNamesStatsResetAuthKey = flagutil.NewPassword("metricNamesStatsResetAuthKey", "authKey for resetting metric names usage cache via /api/v1/admin/status/metric_names_stats/reset. It overrides -httpAuth.*. "+
+		"See https://docs.victoriametrics.com/#track-ingested-metrics-usage")
+
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
@@ -178,7 +182,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promql.ResetRollupResultCache()
 		return true
 	}
-
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := path[len("/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {
@@ -399,6 +402,26 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/status/metric_names_stats":
+		metricNamesStatsRequests.Inc()
+		if err := stats.MetricNamesStatsHandler(qt, w, r); err != nil {
+			metricNamesStatsErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
+	case "/api/v1/admin/status/metric_names_stats/reset":
+		metricNamesStatsResetRequests.Inc()
+		if !httpserver.CheckAuthFlag(w, r, metricNamesStatsResetAuthKey) {
+			return true
+		}
+		if err := stats.ResetMetricNamesStatsHandler(qt); err != nil {
+			metricNamesStatsResetErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	default:
 		return false
 	}
@@ -674,6 +697,12 @@ var (
 	metadataRequests       = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
 	buildInfoRequests      = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/buildinfo"}`)
 	queryExemplarsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/query_exemplars"}`)
+
+	metricNamesStatsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/metric_names_stats"}`)
+	metricNamesStatsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/status/metric_names_stats"}`)
+
+	metricNamesStatsResetRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
+	metricNamesStatsResetErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
 )
 
 func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {

app/vmselect/netstorage/netstorage.go

@@ -1367,3 +1367,18 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 //
 // See https://github.com/golang/go/blob/704401ffa06c60e059c9e6e4048045b4ff42530a/src/runtime/malloc.go#L11
 const maxFastAllocBlockSize = 32 * 1024
+
+// GetMetricNamesStats returns statistic for timeseries metric names usage.
+func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (storage.MetricNamesStatsResponse, error) {
+	qt = qt.NewChild("get metric names usage statistics with limit: %d, less or equal to: %d, match pattern=%q", limit, le, matchPattern)
+	defer qt.Done()
+	return vmstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
+}
+
+// ResetMetricNamesStats resets state of metric names usage
+func ResetMetricNamesStats(qt *querytracer.Tracer) error {
+	qt = qt.NewChild("reset metric names usage stats")
+	defer qt.Done()
+	vmstorage.ResetMetricNamesStats(qt)
+	return nil
+}

app/vmselect/prometheus/expand-with-exprs.qtpl

@@ -237,7 +237,7 @@ WITH (
 </pre>
 
 <p>
-    4. Put node_cpu_seconds_total{commonFilters} into its own varialbe with the name cpuSeconds:
+    4. Put node_cpu_seconds_total{commonFilters} into its own variable with the name cpuSeconds:
 </p>
 <pre>
 WITH (

app/vmselect/prometheus/expand-with-exprs.qtpl.go

@@ -350,7 +350,7 @@ WITH (
 </pre>
 
 <p>
-    4. Put node_cpu_seconds_total{commonFilters} into its own varialbe with the name cpuSeconds:
+    4. Put node_cpu_seconds_total{commonFilters} into its own variable with the name cpuSeconds:
 </p>
 <pre>
 WITH (

app/vmselect/prometheus/prometheus.go

@@ -53,12 +53,13 @@ var (
 
 	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage. "+
 		"When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional).")
-	maxFederateSeries   = flag.Int("search.maxFederateSeries", 1e6, "The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage")
-	maxExportSeries     = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage")
-	maxTSDBStatusSeries = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
-	maxSeriesLimit      = flag.Int("search.maxSeries", 30e3, "The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage")
-	maxDeleteSeries     = flag.Int("search.maxDeleteSeries", 1e6, "The maximum number of time series, which can be deleted using /api/v1/admin/tsdb/delete_series. This option allows limiting memory usage")
-	maxLabelsAPISeries  = flag.Int("search.maxLabelsAPISeries", 1e6, "The maximum number of time series, which could be scanned when searching for the matching time series "+
+	maxFederateSeries       = flag.Int("search.maxFederateSeries", 1e6, "The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage")
+	maxExportSeries         = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage")
+	maxTSDBStatusSeries     = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
+	maxSeriesLimit          = flag.Int("search.maxSeries", 30e3, "The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage")
+	maxDeleteSeries         = flag.Int("search.maxDeleteSeries", 1e6, "The maximum number of time series, which can be deleted using /api/v1/admin/tsdb/delete_series. This option allows limiting memory usage")
+	maxTSDBStatusTopNSeries = flag.Int("search.maxTSDBStatusTopNSeries", 1000, "The maximum value of `topN` argument that can be passed to /api/v1/status/tsdb API. This option allows limiting memory usage. See https://docs.victoriametrics.com/readme/#tsdb-stats")
+	maxLabelsAPISeries      = flag.Int("search.maxLabelsAPISeries", 1e6, "The maximum number of time series, which could be scanned when searching for the matching time series "+
 		"at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, "+
 		"-search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI")
 	maxPointsPerTimeseries = flag.Int("search.maxPointsPerTimeseries", 30e3, "The maximum points per a single timeseries returned from /api/v1/query_range. "+
@@ -537,7 +538,7 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
 	defer bufferedwriter.Put(bw)
 	WriteLabelValuesResponse(bw, labelValues, qt)
 	if err := bw.Flush(); err != nil {
-		return fmt.Errorf("canot flush label values to remote client: %w", err)
+		return fmt.Errorf("cannot flush label values to remote client: %w", err)
 	}
 	return nil
 }
@@ -584,8 +585,8 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		if n <= 0 {
 			n = 1
 		}
-		if n > 1000 {
-			n = 1000
+		if n > *maxTSDBStatusTopNSeries {
+			n = *maxTSDBStatusTopNSeries
 		}
 		topN = n
 	}
@@ -753,11 +754,11 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 			end = start
 		}
 
-		tagFilterss, err := getTagFilterssFromMatches([]string{childQuery})
+		tagFilters, err := getTagFilterssFromMatches([]string{childQuery})
 		if err != nil {
 			return err
 		}
-		filterss := searchutils.JoinTagFilterss(tagFilterss, etfs)
+		filterss := searchutils.JoinTagFilterss(tagFilters, etfs)
 
 		cp := &commonParams{
 			deadline: deadline,
@@ -985,7 +986,7 @@ func removeEmptyValuesAndTimeseries(tss []netstorage.Result) []netstorage.Result
 		// Slow path: remove NaNs.
 		srcTimestamps := ts.Timestamps
 		dstValues := ts.Values[:0]
-		// Do not re-use ts.Timestamps for dstTimestamps, since ts.Timestamps
+		// Do not reuse ts.Timestamps for dstTimestamps, since ts.Timestamps
 		// may be shared among multiple time series.
 		dstTimestamps := make([]int64, 0, len(ts.Timestamps))
 		for j, v := range ts.Values {

app/vmselect/promql/aggr.go

@@ -478,7 +478,7 @@ func aggrFuncShare(afa *aggrFuncArg) ([]*timeseries, error) {
 				}
 				sum += v
 			}
-			// Divide every non-negative value at poisition i by sum in order to get its' share.
+			// Divide every non-negative value at position i by sum in order to get its' share.
 			for _, ts := range tss {
 				v := ts.Values[i]
 				if math.IsNaN(v) || v < 0 {

app/vmselect/promql/aggr_incremental.go

@@ -161,7 +161,7 @@ func (iafc *incrementalAggrFuncContext) finalizeTimeseries() []*timeseries {
 		finalizeAggrFunc(iac)
 		tss = append(tss, iac.ts)
 	}
-	// reset iafc state, so it could be re-used
+	// reset iafc state, so it could be reused
 	iafc.resetState()
 	return tss
 }

app/vmselect/promql/binary_op.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"bytes"
 	"fmt"
 	"math"
 	"strings"
@@ -539,13 +540,41 @@ func fillLeftNaNsWithRightValues(tssLeft, tssRight []*timeseries) {
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759
 // https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640
 func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
+	if isScalar(tssRight) {
+		// fast path: if tssRight is scalar then it can be merged
+		// with tssLeft only when tssLeft is also a scalar.
+		// If tssLeft is not a scalar, then no need in comparing MetricNames.
+		// Typical case is: metric_selector or on() vector(0)
+		canBeMerged := isScalar(tssLeft)
+		valuesRight := tssRight[0].Values
+		for _, tsLeft := range tssLeft {
+			valuesLeft := tsLeft.Values
+			for i, v := range valuesLeft {
+				leftIsNaN := math.IsNaN(v)
+				valueRight := valuesRight[i]
+				if leftIsNaN && canBeMerged {
+					// fill NaNs with valueRight if labels match
+					valuesLeft[i] = valueRight
+				}
+				if !leftIsNaN || canBeMerged {
+					// set NaN to valueRight if valueLeft is not NaN
+					// or if left and right can be merged
+					valuesRight[i] = nan
+				}
+			}
+		}
+		return
+	}
+
+	nameLeft, nameRight := bbPool.Get(), bbPool.Get()
 	for _, tsLeft := range tssLeft {
 		valuesLeft := tsLeft.Values
-		nameLeft := tsLeft.MetricName.String()
+		nameLeft.B = marshalMetricNameSorted(nameLeft.B[:0], &tsLeft.MetricName)
 		for i, v := range valuesLeft {
 			leftIsNaN := math.IsNaN(v)
 			for _, tsRight := range tssRight {
-				canBeMerged := nameLeft == tsRight.MetricName.String()
+				nameRight.B = marshalMetricNameSorted(nameRight.B[:0], &tsRight.MetricName)
+				canBeMerged := bytes.Equal(nameLeft.B, nameRight.B)
 				valueRight := tsRight.Values[i]
 				if leftIsNaN && canBeMerged {
 					// fill NaNs with valueRight if labels match
@@ -559,6 +588,8 @@ func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
 			}
 		}
 	}
+	bbPool.Put(nameLeft)
+	bbPool.Put(nameRight)
 }
 
 func binaryOpIfnot(bfa *binaryOpFuncArg) ([]*timeseries, error) {

app/vmselect/promql/binary_op_timing_test.go

@@ -0,0 +1,107 @@
+package promql
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/metricsql"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+func BenchmarkBinaryOpOr(b *testing.B) {
+	mustParseMetricsQL := func(s string) *metricsql.BinaryOpExpr {
+		e, err := metricsql.Parse(s)
+		if err != nil {
+			b.Fatalf("cannot parse %q: %s", s, err)
+		}
+		be, ok := e.(*metricsql.BinaryOpExpr)
+		if !ok {
+			b.Fatalf("expected BinaryOpExpr from %q: %s", s, err)
+		}
+		return be
+	}
+	ts := func(name string) *timeseries {
+		mg := []byte(name)
+		if len(name) == 0 {
+			mg = nil
+		}
+		return &timeseries{
+			MetricName: storage.MetricName{
+				MetricGroup: mg,
+			},
+			Timestamps: []int64{1},
+			Values:     []float64{1},
+		}
+	}
+
+	b.Run("tss:1 or tss:1", func(b *testing.B) {
+		bfa := &binaryOpFuncArg{
+			be:    mustParseMetricsQL("a or b"),
+			left:  []*timeseries{ts("a")},
+			right: []*timeseries{ts("b")},
+		}
+		benchmarkBinaryOpOr(b, bfa)
+	})
+	b.Run("tss:1 or tss:1000", func(b *testing.B) {
+		right := make([]*timeseries, 1000)
+		for i := range right {
+			right[i] = ts(fmt.Sprintf(`b{foo="%d"}`, i))
+		}
+		bfa := &binaryOpFuncArg{
+			be:    mustParseMetricsQL("a or b"),
+			left:  []*timeseries{ts(`a{foo="0"}`)},
+			right: right,
+		}
+		benchmarkBinaryOpOr(b, bfa)
+	})
+	b.Run("tss:1000 or tss:1", func(b *testing.B) {
+		left := make([]*timeseries, 1000)
+		for i := range left {
+			left[i] = ts(fmt.Sprintf(`b{foo="%d"}`, i))
+		}
+		bfa := &binaryOpFuncArg{
+			be:    mustParseMetricsQL("a or b"),
+			left:  left,
+			right: []*timeseries{ts(`b{foo="0"}`)},
+		}
+		benchmarkBinaryOpOr(b, bfa)
+	})
+	b.Run("tss:1000 or tss:1000", func(b *testing.B) {
+		left, right := make([]*timeseries, 1000), make([]*timeseries, 1000)
+		for i := range left {
+			left[i] = ts(fmt.Sprintf(`a{foo="%d"}`, i))
+			right[i] = ts(fmt.Sprintf(`b{foo="%d"}`, i))
+		}
+		bfa := &binaryOpFuncArg{
+			be:    mustParseMetricsQL("a or b"),
+			left:  left,
+			right: right,
+		}
+		benchmarkBinaryOpOr(b, bfa)
+	})
+	b.Run("tss:1000 or on() vector(0)", func(b *testing.B) {
+		left := make([]*timeseries, 1000)
+		for i := range left {
+			left[i] = ts(fmt.Sprintf(`a{foo="%d"}`, i))
+		}
+		bfa := &binaryOpFuncArg{
+			be:    mustParseMetricsQL("a or on() vector(0)"),
+			left:  left,
+			right: []*timeseries{ts(``)},
+		}
+		benchmarkBinaryOpOr(b, bfa)
+	})
+}
+
+func benchmarkBinaryOpOr(b *testing.B, bfa *binaryOpFuncArg) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, err := binaryOpOr(bfa)
+			if err != nil {
+				b.Fatalf("unexpected error: %s", err)
+			}
+
+		}
+	})
+}

app/vmselect/promql/eval.go

@@ -12,6 +12,9 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -25,8 +28,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/VictoriaMetrics/metricsql"
 )
 
 var (
@@ -814,7 +815,19 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 			Err: fmt.Errorf("`@` modifier must return a single series; it returns %d series instead", len(tssAt)),
 		}
 	}
-	atTimestamp := int64(tssAt[0].Values[0] * 1000)
+	atValue := math.NaN()
+	for _, v := range tssAt[0].Values {
+		if !math.IsNaN(v) {
+			atValue = v
+			break
+		}
+	}
+	if math.IsNaN(atValue) {
+		return nil, &httpserver.UserReadableError{
+			Err: fmt.Errorf("`@` modifier must return a non-NaN value"),
+		}
+	}
+	atTimestamp := int64(atValue * 1000)
 	ecNew := copyEvalConfig(ec)
 	ecNew.Start = atTimestamp
 	ecNew.End = atTimestamp
@@ -1821,7 +1834,7 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 			samplesScannedTotal.Add(samplesScanned)
 			iafc.updateTimeseries(ts, workerID)
 
-			// ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used.
+			// ts.Timestamps points to sharedTimestamps. Zero it, so it can be reused.
 			ts.Timestamps = nil
 			ts.denyReuse = false
 		}

app/vmselect/promql/exec_test.go

@@ -10195,7 +10195,7 @@ max_over_time(cpuIdle[1h:])`)
 	f("avg(foo[5m])")
 	f("sort(foo[5m])")
 
-	// These are valid subqueries with MetricsQL extention, which allows omitting lookbehind window for rollup functions
+	// These are valid subqueries with MetricsQL extension, which allows omitting lookbehind window for rollup functions
 	f("rate(rate(http_total)[5m:1m])")
 	f("rate(sum(rate(http_total))[5m:1m])")
 	f("rate(sum(rate(http_total))[5m:1m])")

app/vmselect/promql/rollup_result_cache.go

@@ -121,7 +121,7 @@ func InitRollupResultCache(cachePath string) {
 	var c *workingsetcache.Cache
 	if len(rollupResultCachePath) > 0 {
 		if *resetRollupResultCacheOnStartup {
-			logger.Infof("removing rollupResult cache at %q becasue -search.resetRollupResultCacheOnStartup command-line flag is set", rollupResultCachePath)
+			logger.Infof("removing rollupResult cache at %q because -search.resetRollupResultCacheOnStartup command-line flag is set", rollupResultCachePath)
 			fs.MustRemoveAll(rollupResultCachePath)
 		} else {
 			logger.Infof("loading rollupResult cache from %q...", rollupResultCachePath)
@@ -474,7 +474,7 @@ func (rrc *rollupResultCache) getSeriesFromCache(qt *querytracer.Tracer, key []b
 	}
 	qt.Printf("load compressed entry from cache with size %d bytes", len(compressedResultBuf.B))
 	// Decompress into newly allocated byte slice, since tss returned from unmarshalTimeseriesFast
-	// refers to the byte slice, so it cannot be re-used.
+	// refers to the byte slice, so it cannot be reused.
 	resultBuf, err := encoding.DecompressZSTD(nil, compressedResultBuf.B)
 	if err != nil {
 		logger.Panicf("BUG: cannot decompress resultBuf from rollupResultCache: %s; it looks like it was improperly saved", err)

app/vmselect/promql/rollup_result_cache_test.go

@@ -469,7 +469,7 @@ func TestMergeSeries(t *testing.T) {
 		}
 		tss, ok := mergeSeries(nil, a, b, bStart, ec)
 		if ok {
-			t.Fatalf("expecting failre to merge series")
+			t.Fatalf("expecting failure to merge series")
 		}
 		testTimeseriesEqual(t, tss, nil)
 	})
@@ -492,7 +492,7 @@ func TestMergeSeries(t *testing.T) {
 		}
 		tss, ok := mergeSeries(nil, a, b, bStart, ec)
 		if ok {
-			t.Fatalf("expecting failre to merge series")
+			t.Fatalf("expecting failure to merge series")
 		}
 		testTimeseriesEqual(t, tss, nil)
 	})

app/vmselect/promql/timeseries.go

@@ -19,8 +19,8 @@ type timeseries struct {
 	Values     []float64
 	Timestamps []int64
 
-	// Whether the timeseries may be re-used.
-	// Timeseries may be re-used only if their members own values
+	// Whether the timeseries may be reused.
+	// Timeseries may be reused only if their members own values
 	// they refer to.
 	denyReuse bool
 }

app/vmselect/searchutils/searchutils_test.go

@@ -30,7 +30,7 @@ func TestGetExtraTagFilters(t *testing.T) {
 		}
 		got := tagFilterssToStrings(result)
 		if !reflect.DeepEqual(got, want) {
-			t.Fatalf("unxpected result for GetExtraTagFilters\ngot:  %s\nwant: %s", got, want)
+			t.Fatalf("unexpected result for GetExtraTagFilters\ngot:  %s\nwant: %s", got, want)
 		}
 	}
 	f(t, httpReqWithForm("extra_label=label=value"),
@@ -133,7 +133,7 @@ func TestJoinTagFilterss(t *testing.T) {
 		result := JoinTagFilterss(src, etfs)
 		got := tagFilterssToStrings(result)
 		if !reflect.DeepEqual(got, want) {
-			t.Fatalf("unxpected result for JoinTagFilterss\ngot:  %s\nwant: %v", got, want)
+			t.Fatalf("unexpected result for JoinTagFilterss\ngot:  %s\nwant: %v", got, want)
 		}
 	}
 	// Single tag filter
@@ -142,7 +142,7 @@ func TestJoinTagFilterss(t *testing.T) {
 	), nil, []string{
 		`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`,
 	})
-	// Miltiple tag filters
+	// Multiple tag filters
 	f(t, joinTagFilters(
 		mustParseMetricSelector(`{k1="v1",k2=~"v2",k3!="v3",k4!~"v4"}`),
 		mustParseMetricSelector(`{k5=~"v5"}`),

app/vmselect/stats/metric_names_usage_response.qtpl

@@ -0,0 +1,33 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+) %}
+
+{% stripspace %}
+MetricNamesStatsResponse generates response for /api/v1/status/metric_names_stats .
+{% func MetricNamesStatsResponse(stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) %}
+{
+	"status":"success",
+  "statsCollectedSince": {%dul= stats.CollectedSinceTs %},
+  "statsCollectedRecordsTotal": {%dul= stats.TotalRecords %},
+  "trackerMemoryMaxSizeBytes": {%dul= stats.MaxSizeBytes %},
+  "trackerCurrentMemoryUsageBytes": {%dul= stats.CurrentSizeBytes %},
+  "records":
+  [
+    {% for i, r := range stats.Records %}
+      {
+        "metricName":{%q= r.MetricName %},
+        "queryRequestsCount":{%dul= r.RequestsCount %},
+        "lastQueryRequestTimestamp":{%dul= r.LastRequestTs %}
+      }
+      {% if i+1 < len(stats.Records) %},{% endif %}
+    {% endfor %}
+  ]
+	{% code	qt.Done() %}
+	{% code	traceJSON := qt.ToJSON() %}
+	{% if traceJSON != "" %},"trace":{%s= traceJSON %}{% endif %}
+
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/stats/metric_names_usage_response.qtpl.go

@@ -0,0 +1,117 @@
+// Code generated by qtc from "metric_names_usage_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:1
+package stats
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+// MetricNamesStatsResponse generates response for /api/v1/status/metric_names_stats .
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+func StreamMetricNamesStatsResponse(qw422016 *qt422016.Writer, stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+	qw422016.N().S(`{"status":"success","statsCollectedSince":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:11
+	qw422016.N().DUL(stats.CollectedSinceTs)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:11
+	qw422016.N().S(`,"statsCollectedRecordsTotal":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:12
+	qw422016.N().DUL(stats.TotalRecords)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:12
+	qw422016.N().S(`,"trackerMemoryMaxSizeBytes":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:13
+	qw422016.N().DUL(stats.MaxSizeBytes)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:13
+	qw422016.N().S(`,"trackerCurrentMemoryUsageBytes":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:14
+	qw422016.N().DUL(stats.CurrentSizeBytes)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:14
+	qw422016.N().S(`,"records":[`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:17
+	for i, r := range stats.Records {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:17
+		qw422016.N().S(`{"metricName":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:19
+		qw422016.N().Q(r.MetricName)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:19
+		qw422016.N().S(`,"queryRequestsCount":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:20
+		qw422016.N().DUL(r.RequestsCount)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:20
+		qw422016.N().S(`,"lastQueryRequestTimestamp":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:21
+		qw422016.N().DUL(r.LastRequestTs)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:21
+		qw422016.N().S(`}`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:23
+		if i+1 < len(stats.Records) {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:23
+			qw422016.N().S(`,`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:23
+		}
+//line app/vmselect/stats/metric_names_usage_response.qtpl:24
+	}
+//line app/vmselect/stats/metric_names_usage_response.qtpl:24
+	qw422016.N().S(`]`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:26
+	qt.Done()
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:27
+	traceJSON := qt.ToJSON()
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+	if traceJSON != "" {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+		qw422016.N().S(`,"trace":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+		qw422016.N().S(traceJSON)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+	}
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+	qw422016.N().S(`}`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+}
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+func WriteMetricNamesStatsResponse(qq422016 qtio422016.Writer, stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	StreamMetricNamesStatsResponse(qw422016, stats, qt)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+}
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+func MetricNamesStatsResponse(stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) string {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	WriteMetricNamesStatsResponse(qb422016, stats, qt)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qs422016 := string(qb422016.B)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	return qs422016
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+}

app/vmselect/stats/stats.go

@@ -0,0 +1,50 @@
+package stats
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+)
+
+// MetricNamesStatsHandler returns timeseries metric names usage statistics
+func MetricNamesStatsHandler(qt *querytracer.Tracer, w http.ResponseWriter, r *http.Request) error {
+	limit := 1000
+	limitStr := r.FormValue("limit")
+	if len(limitStr) > 0 {
+		n, err := strconv.Atoi(limitStr)
+		if err != nil {
+			return fmt.Errorf("cannot parse `limit` arg %q: %w", limitStr, err)
+		}
+		if n > 0 {
+			limit = n
+		}
+	}
+	// by default display all values
+	le := -1
+	leStr := r.FormValue("le")
+	if len(leStr) > 0 {
+		n, err := strconv.Atoi(leStr)
+		if err != nil {
+			return fmt.Errorf("cannot parse `le` arg %q: %w", leStr, err)
+		}
+		le = n
+	}
+	matchPattern := r.FormValue("match_pattern")
+	stats, err := netstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
+	if err != nil {
+		return err
+	}
+	WriteMetricNamesStatsResponse(w, &stats, qt)
+	return nil
+}
+
+// ResetMetricNamesStatsHandler resets metric names usage state
+func ResetMetricNamesStatsHandler(qt *querytracer.Tracer) error {
+	if err := netstorage.ResetMetricNamesStats(qt); err != nil {
+		return err
+	}
+	return nil
+}

app/vmselect/vmui/assets/MetricsQL-DGylhASK.md

@@ -2349,4 +2349,4 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
   For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
   This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+  starting from [`v1.102.0-rc2` release](https://docs.victoriametrics.com/changelog/changelog_2024/#v11020-rc2).

app/vmselect/vmui/index.html

@@ -36,10 +36,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-DzehQsnZ.js"></script>
+  <script type="module" crossorigin src="./assets/index-C4jrb8hY.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-Cqbobgy7.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-B_R5bdPN.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -76,6 +76,11 @@ var (
 		"This may improve performance and decrease disk space usage for the use cases with fixed set of timeseries scattered across a "+
 		"big time range (for example, when loading years of historical data). "+
 		"See https://docs.victoriametrics.com/single-server-victoriametrics/#index-tuning")
+	trackMetricNamesStats = flag.Bool("storage.trackMetricNamesStats", false, "Whether to track ingest and query requests for timeseries metric names. "+
+		"This feature allows to track metric names unused at query requests. "+
+		"See https://docs.victoriametrics.com/#track-ingested-metrics-usage")
+	cacheSizeMetricNamesStats = flagutil.NewBytes("storage.cacheSizeMetricNamesStats", 0, "Overrides max size for storage/metricNamesStatsTracker cache. "+
+		"See https://docs.victoriametrics.com/single-server-victoriametrics/#cache-tuning")
 )
 
 // CheckTimeRange returns true if the given tr is denied for querying.
@@ -105,6 +110,7 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
+	storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
 	mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())
@@ -115,12 +121,12 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	logger.Infof("opening storage at %q with -retentionPeriod=%s", *DataPath, retentionPeriod)
 	startTime := time.Now()
 	WG = syncwg.WaitGroup{}
-
 	opts := storage.OpenOptions{
-		Retention:          retentionPeriod.Duration(),
-		MaxHourlySeries:    *maxHourlySeries,
-		MaxDailySeries:     *maxDailySeries,
-		DisablePerDayIndex: *disablePerDayIndex,
+		Retention:             retentionPeriod.Duration(),
+		MaxHourlySeries:       *maxHourlySeries,
+		MaxDailySeries:        *maxDailySeries,
+		DisablePerDayIndex:    *disablePerDayIndex,
+		TrackMetricNamesStats: *trackMetricNamesStats,
 	}
 	strg := storage.MustOpenStorage(*DataPath, opts)
 	Storage = strg
@@ -193,6 +199,21 @@ func DeleteSeries(qt *querytracer.Tracer, tfss []*storage.TagFilters, maxMetrics
 	return n, err
 }
 
+// GetMetricNamesStats returns metric names usage stats with give limit and lte predicate
+func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (storage.MetricNamesStatsResponse, error) {
+	WG.Add(1)
+	r := Storage.GetMetricNamesStats(qt, limit, le, matchPattern)
+	WG.Done()
+	return r, nil
+}
+
+// ResetMetricNamesStats resets state for metric names usage tracker
+func ResetMetricNamesStats(qt *querytracer.Tracer) {
+	WG.Add(1)
+	Storage.ResetMetricNamesStats(qt)
+	WG.Done()
+}
+
 // SearchMetricNames returns metric names for the given tfss on the given tr.
 func SearchMetricNames(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr storage.TimeRange, maxMetrics int, deadline uint64) ([]string, error) {
 	WG.Add(1)
@@ -657,6 +678,12 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 
 	metrics.WriteGaugeUint64(w, `vm_next_retention_seconds`, m.NextRetentionSeconds)
 
+	if *trackMetricNamesStats {
+		metrics.WriteCounterUint64(w, `vm_cache_size_bytes{type="storage/metricNamesStatsTracker"}`, m.MetricNamesUsageTrackerSizeBytes)
+		metrics.WriteCounterUint64(w, `vm_cache_size{type="storage/metricNamesStatsTracker"}`, m.MetricNamesUsageTrackerSize)
+		metrics.WriteCounterUint64(w, `vm_cache_size_max_bytes{type="storage/metricNamesStatsTracker"}`, m.MetricNamesUsageTrackerSizeMaxBytes)
+	}
+
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled`, tm.ScheduledDownsamplingPartitions)
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled_size_bytes`, tm.ScheduledDownsamplingPartitionsSize)
 }

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.24.0 AS build-web-stage
+FROM golang:1.24.1 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -2349,4 +2349,4 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
   For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
   This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+  starting from [`v1.102.0-rc2` release](https://docs.victoriametrics.com/changelog/changelog_2024/#v11020-rc2).

app/vmui/packages/vmui/src/components/Chart/GraphTips/GraphTips.tsx

@@ -5,7 +5,7 @@ import { TipIcon } from "../../Main/Icons";
 import Tooltip from "../../Main/Tooltip/Tooltip";
 import Modal from "../../Main/Modal/Modal";
 import useBoolean from "../../../hooks/useBoolean";
-import tips from "./contants/tips";
+import tips from "./constants/tips";
 
 const GraphTips: FC = () => {
   const {

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/LogsQL/pipes.tsx

@@ -104,7 +104,7 @@ export const pipeList = [
   },
   {
     "value": "uniq",
-    "description": "<a href=\"#uniq-pipe\"><code>uniq</code></a> returns unique log entires."
+    "description": "<a href=\"#uniq-pipe\"><code>uniq</code></a> returns unique log entries."
   },
   {
     "value": "unpack_json",

app/vmui/packages/vmui/src/constants/emojis.ts

@@ -1422,8 +1422,8 @@ export default {
   "put_litter_in_its_place": "🚮",
   "potable_water": "🚰",
   "wheelchair": "♿",
-  "mens": "🚹",
-  "womens": "🚺",
+  "men": "🚹",
+  "women": "🚺",
   "restroom": "🚻",
   "baby_symbol": "🚼",
   "wc": "🚾",

app/vmui/packages/vmui/src/pages/CustomPanel/DownloadReport/helperText.tsx

@@ -11,7 +11,7 @@ const filename = (
   </>
 );
 
-const tittle = (
+const title = (
   <>
     <p>Title - specify the title that will be displayed on the <Link
       to={router.queryAnalyzer}
@@ -51,7 +51,7 @@ const generate = (
 
 export default [
   filename,
-  tittle,
+  title,
   comment,
   trace,
   generate,

app/vmui/packages/vmui/src/pages/WithTemplate/WithTemplateTutorial/WithTemplateTutorial.tsx

@@ -185,7 +185,7 @@ count(
     </div>
     <div className="vm-with-template-tutorial-section">
       <p className="vm-with-template-tutorial-section__text">
-        4. Put node_cpu_seconds_total{"{commonFilters}"} into its own varialbe with the name cpuSeconds:
+        4. Put node_cpu_seconds_total{"{commonFilters}"} into its own variable with the name cpuSeconds:
       </p>
       <CodeExample
         code={`WITH (

app/vmui/packages/vmui/src/utils/uplot/index.ts

@@ -2,7 +2,7 @@ export * from "./axes";
 export * from "./heatmap";
 export * from "./helpers";
 export * from "./hooks";
-export * from "./instnance";
+export * from "./instance";
 export * from "./scales";
 export * from "./series";
 export * from "./bands";

apptest/README.md

@@ -8,7 +8,7 @@ as apposed to a unit test that verifies the behavior of a building block of an
 application.
 
 To achieve that an integration test starts an application in a separate process
-and then issues HTTP requets to it and verifies the responses, examines the
+and then issues HTTP requests to it and verifies the responses, examines the
 metrics the app exposes and/or files it creates, etc.
 
 Note that an object of testing may be not just a single app, but several apps

apptest/app.go

@@ -237,7 +237,7 @@ func newREExtractor(re *regexp.Regexp, timeout <-chan time.Time) *reExtractor {
 }
 
 // extractRE is a line processor that extracts some information from a line
-// based on a regular expression. The function returns trun (to request the
+// based on a regular expression. The function returns true (to request the
 // caller to not to be called again) either when the match is found or due to
 // the timeout. The found match is written to the x.result channel and it is
 // important that this channel is monitored by a separate goroutine, otherwise

apptest/client.go

@@ -88,9 +88,9 @@ func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
 	return string(b)
 }
 
-// ServesMetrics is used to retrive the app's metrics.
+// ServesMetrics is used to retrieve the app's metrics.
 //
-// This type is expected to be embdded by the apps that serve metrics.
+// This type is expected to be embedded by the apps that serve metrics.
 type ServesMetrics struct {
 	metricsURL string
 	cli        *Client

apptest/model.go

@@ -3,6 +3,7 @@ package apptest
 import (
 	"encoding/json"
 	"fmt"
+	"math"
 	"net/url"
 	"slices"
 	"sort"
@@ -34,25 +35,33 @@ type StorageFlusher interface {
 	ForceFlush(t *testing.T)
 }
 
+// StorageMerger defines a method that forces the merging of data inserted
+// into the storage.
+type StorageMerger interface {
+	ForceMerge(t *testing.T)
+}
+
 // PrometheusWriteQuerier encompasses the methods for writing, flushing and
 // querying the data.
 type PrometheusWriteQuerier interface {
 	PrometheusWriter
 	PrometheusQuerier
 	StorageFlusher
+	StorageMerger
 }
 
 // QueryOpts contains various params used for querying or ingesting data
 type QueryOpts struct {
-	Tenant       string
-	Timeout      string
-	Start        string
-	End          string
-	Time         string
-	Step         string
-	ExtraFilters []string
-	ExtraLabels  []string
-	Trace        string
+	Tenant         string
+	Timeout        string
+	Start          string
+	End            string
+	Time           string
+	Step           string
+	ExtraFilters   []string
+	ExtraLabels    []string
+	Trace          string
+	ReduceMemUsage string
 }
 
 func (qos *QueryOpts) asURLValues() url.Values {
@@ -73,6 +82,7 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("extra_label", qos.ExtraLabels...)
 	addNonEmpty("extra_filters", qos.ExtraFilters...)
 	addNonEmpty("trace", qos.Trace)
+	addNonEmpty("reduce_mem_usage", qos.ReduceMemUsage)
 
 	return uv
 }
@@ -92,6 +102,7 @@ type PrometheusAPIV1QueryResponse struct {
 	Data      *QueryData
 	ErrorType string
 	Error     string
+	IsPartial bool
 }
 
 // NewPrometheusAPIV1QueryResponse is a test helper function that creates a new
@@ -108,6 +119,10 @@ func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1Que
 
 // Sort performs data.Result sort by metric labels
 func (pqr *PrometheusAPIV1QueryResponse) Sort() {
+	if pqr.Data == nil {
+		return
+	}
+
 	sort.Slice(pqr.Data.Result, func(i, j int) bool {
 		leftS := make([]string, 0, len(pqr.Data.Result[i].Metric))
 		rightS := make([]string, 0, len(pqr.Data.Result[j].Metric))
@@ -123,6 +138,25 @@ func (pqr *PrometheusAPIV1QueryResponse) Sort() {
 		return strings.Join(leftS, ",") < strings.Join(rightS, ",")
 	})
 
+	for _, result := range pqr.Data.Result {
+		sort.Slice(result.Samples, func(i, j int) bool {
+			a := result.Samples[i]
+			b := result.Samples[j]
+			if a.Timestamp != b.Timestamp {
+				return a.Timestamp < b.Timestamp
+			}
+
+			// Put NaNs at the end of the slice.
+			if math.IsNaN(a.Value) {
+				return false
+			}
+			if math.IsNaN(b.Value) {
+				return true
+			}
+
+			return a.Value < b.Value
+		})
+	}
 }
 
 // QueryData holds the query result along with its type.
@@ -258,3 +292,15 @@ func (t *Trace) Contains(s string) int {
 	}
 	return times
 }
+
+// MetricNamesStatsResponse is an inmemory representation of the
+// /api/v1/status/metric_names_stats API response
+type MetricNamesStatsResponse struct {
+	Records []MetricNamesStatsRecord
+}
+
+// MetricNamesStatsRecord is a record item for MetricNamesStatsResponse
+type MetricNamesStatsRecord struct {
+	MetricName         string
+	QueryRequestsCount uint64
+}