oops, copy paste error

Previously per-token hashes for per-block bloom filters were re-calculated on every scanned block.
This could be slow when the number of tokens is big or when the number of blocks to scan is big.
Pre-calculate hashes for bloom filters and then use them for searching in bloom filters.
This improves performance by 2.5x for in(...) filters with many values to search inside `in()`.

.gitignore

@@ -26,3 +26,4 @@ _site
 /docs/.jekyll-metadata
 coverage.txt
 cspell.json
+*~

.golangci.yml

@@ -1,5 +1,7 @@
 run:
   timeout: 2m
+  skip-dirs:
+    - "lib/regexutil/syntax"
 
 linters:
   enable:

.wwhrd.yml

@@ -4,3 +4,4 @@ allowlist:
   - BSD-3-Clause
   - BSD-2-Clause
   - ISC
+  - MPL-2.0

Makefile

@@ -333,7 +333,7 @@ release-vmutils: \
 
 release-vmutils-linux-386:
 	GOOS=linux GOARCH=386 $(MAKE) release-vmutils-goos-goarch
-	
+
 release-vmutils-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
@@ -444,19 +444,19 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	go test ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
 
 test-race:
-	go test -race ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
 
 test-pure:
-	CGO_ENABLED=0 go test ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 benchmark:
 	go test -bench=. ./lib/...
@@ -495,7 +495,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

app/victoria-logs/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 EXPOSE 9428
 
 ENTRYPOINT ["/victoria-logs-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./victoria-logs-prod

app/victoria-logs/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 9428
 ENTRYPOINT ["/victoria-logs-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY victoria-logs-linux-${TARGETARCH}-prod ./victoria-logs-prod

app/victoria-metrics/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 EXPOSE 8428
 
 ENTRYPOINT ["/victoria-metrics-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./victoria-metrics-prod

app/victoria-metrics/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8428
 ENTRYPOINT ["/victoria-metrics-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY victoria-metrics-linux-${TARGETARCH}-prod ./victoria-metrics-prod

app/vlinsert/insertutils/common_params.go

@@ -2,6 +2,7 @@ package insertutils
 
 import (
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -38,22 +39,46 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		return nil, err
 	}
 
-	// Extract time field name from _time_field query arg
-	var timeField = "_time"
+	// Extract time field name from _time_field query arg or header
+	timeField := "_time"
 	if tf := r.FormValue("_time_field"); tf != "" {
 		timeField = tf
+	} else if tf = r.Header.Get("VL-Time-Field"); tf != "" {
+		timeField = tf
 	}
 
-	// Extract message field name from _msg_field query arg
-	var msgField = ""
+	// Extract message field name from _msg_field query arg or header
+	msgField := ""
 	if msgf := r.FormValue("_msg_field"); msgf != "" {
 		msgField = msgf
+	} else if msgf = r.Header.Get("VL-Msg-Field"); msgf != "" {
+		msgField = msgf
 	}
 
 	streamFields := httputils.GetArray(r, "_stream_fields")
+	if len(streamFields) == 0 {
+		if sf := r.Header.Get("VL-Stream-Fields"); len(sf) > 0 {
+			streamFields = strings.Split(sf, ",")
+		}
+	}
 	ignoreFields := httputils.GetArray(r, "ignore_fields")
+	if len(ignoreFields) == 0 {
+		if f := r.Header.Get("VL-Ignore-Fields"); len(f) > 0 {
+			ignoreFields = strings.Split(f, ",")
+		}
+	}
 
 	debug := httputils.GetBool(r, "debug")
+	if !debug {
+		if dh := r.Header.Get("VL-Debug"); len(dh) > 0 {
+			hv := strings.ToLower(dh)
+			switch hv {
+			case "", "0", "f", "false", "no":
+			default:
+				debug = true
+			}
+		}
+	}
 	debugRequestURI := ""
 	debugRemoteAddr := ""
 	if debug {
@@ -71,6 +96,7 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		DebugRequestURI: debugRequestURI,
 		DebugRemoteAddr: debugRemoteAddr,
 	}
+
 	return cp, nil
 }
 
@@ -154,7 +180,7 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields []logstorage.Fiel
 	if lmp.cp.Debug {
 		s := lmp.lr.GetRowString(0)
 		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
 		rowsDroppedTotalDebug.Inc()
 		return
 	}
@@ -196,5 +222,7 @@ func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor {
 	return lmp
 }
 
-var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
-var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
+var (
+	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
+)

app/vlinsert/main.go

@@ -7,6 +7,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/opentelemetry"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/syslog"
 )
 
@@ -41,6 +42,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case strings.HasPrefix(path, "/loki/"):
 		path = strings.TrimPrefix(path, "/loki")
 		return loki.RequestHandler(path, w, r)
+	case strings.HasPrefix(path, "/opentelemetry/"):
+		path = strings.TrimPrefix(path, "/opentelemetry")
+		return opentelemetry.RequestHandler(path, w, r)
 	default:
 		return false
 	}

app/vlinsert/opentelemetry/opentelemetry.go

@@ -0,0 +1,143 @@
+package opentelemetry
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+// RequestHandler processes Opentelemetry insert requests
+func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
+	switch path {
+	// use the same path as opentelemetry collector
+	// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
+	case "/v1/logs":
+		if r.Header.Get("Content-Type") == "application/json" {
+			httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+			return true
+		}
+		handleProtobuf(r, w)
+		return true
+	default:
+		return false
+	}
+}
+
+func handleProtobuf(r *http.Request, w http.ResponseWriter) {
+	startTime := time.Now()
+	requestsProtobufTotal.Inc()
+	reader := r.Body
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		zr, err := common.GetGzipReader(reader)
+		if err != nil {
+			httpserver.Errorf(w, r, "cannot initialize gzip reader: %s", err)
+			return
+		}
+		defer common.PutGzipReader(zr)
+		reader = zr
+	}
+
+	wcr := writeconcurrencylimiter.GetReader(reader)
+	data, err := io.ReadAll(wcr)
+	writeconcurrencylimiter.PutReader(wcr)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot read request body: %s", err)
+		return
+	}
+
+	cp, err := insertutils.GetCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
+		return
+	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	lmp := cp.NewLogMessageProcessor()
+	n, err := pushProtobufRequest(data, lmp)
+	lmp.MustClose()
+	if err != nil {
+		httpserver.Errorf(w, r, "cannot parse OpenTelemetry protobuf request: %s", err)
+		return
+	}
+
+	rowsIngestedProtobufTotal.Add(n)
+
+	// update requestProtobufDuration only for successfully parsed requests
+	// There is no need in updating requestProtobufDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	requestProtobufDuration.UpdateDuration(startTime)
+}
+
+var (
+	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="opentelemetry",format="protobuf"}`)
+
+	requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
+	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
+
+	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
+)
+
+func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int, error) {
+	var req pb.ExportLogsServiceRequest
+	if err := req.UnmarshalProtobuf(data); err != nil {
+		errorsTotal.Inc()
+		return 0, fmt.Errorf("cannot unmarshal request from %d bytes: %w", len(data), err)
+	}
+
+	var rowsIngested int
+	var commonFields []logstorage.Field
+	for _, rl := range req.ResourceLogs {
+		attributes := rl.Resource.Attributes
+		commonFields = slicesutil.SetLength(commonFields, len(attributes))
+		for i, attr := range attributes {
+			commonFields[i].Name = attr.Key
+			commonFields[i].Value = attr.Value.FormatString()
+		}
+		commonFieldsLen := len(commonFields)
+		for _, sc := range rl.ScopeLogs {
+			var scopeIngested int
+			commonFields, scopeIngested = pushFieldsFromScopeLogs(&sc, commonFields[:commonFieldsLen], lmp)
+			rowsIngested += scopeIngested
+		}
+	}
+
+	return rowsIngested, nil
+}
+
+func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field, lmp insertutils.LogMessageProcessor) ([]logstorage.Field, int) {
+	fields := commonFields
+	for _, lr := range sc.LogRecords {
+		fields = fields[:len(commonFields)]
+		fields = append(fields, logstorage.Field{
+			Name:  "_msg",
+			Value: lr.Body.FormatString(),
+		})
+		for _, attr := range lr.Attributes {
+			fields = append(fields, logstorage.Field{
+				Name:  attr.Key,
+				Value: attr.Value.FormatString(),
+			})
+		}
+		fields = append(fields, logstorage.Field{
+			Name:  "severity",
+			Value: lr.FormatSeverity(),
+		})
+
+		lmp.AddRow(lr.ExtractTimestampNano(), fields)
+	}
+	return fields, len(sc.LogRecords)
+}

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -0,0 +1,128 @@
+package opentelemetry
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
+)
+
+func TestPushProtoOk(t *testing.T) {
+	f := func(src []pb.ResourceLogs, timestampsExpected []int64, resultExpected string) {
+		t.Helper()
+		lr := pb.ExportLogsServiceRequest{
+			ResourceLogs: src,
+		}
+
+		pData := lr.MarshalProtobuf(nil)
+		tlp := &insertutils.TestLogMessageProcessor{}
+		n, err := pushProtobufRequest(pData, tlp)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		if err := tlp.Verify(n, timestampsExpected, resultExpected); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// single line without resource attributes
+	f([]pb.ResourceLogs{
+		{
+			ScopeLogs: []pb.ScopeLogs{
+				{
+					LogRecords: []pb.LogRecord{
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
+					},
+				},
+			},
+		},
+	},
+		[]int64{1234},
+		`{"_msg":"log-line-message","severity":"Trace"}`,
+	)
+	// multi-line with resource attributes
+	f([]pb.ResourceLogs{
+		{
+			Resource: pb.Resource{
+				Attributes: []*pb.KeyValue{
+					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
+					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
+					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
+						Values: []*pb.KeyValue{
+							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
+							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
+						},
+					}}},
+				},
+			},
+			ScopeLogs: []pb.ScopeLogs{
+				{
+					LogRecords: []pb.LogRecord{
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
+					},
+				},
+			},
+		},
+	},
+		[]int64{1234, 1235, 1236},
+		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
+{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
+	)
+
+	// multi-scope with resource attributes and multi-line
+	f([]pb.ResourceLogs{
+		{
+			Resource: pb.Resource{
+				Attributes: []*pb.KeyValue{
+					{Key: "logger", Value: &pb.AnyValue{StringValue: ptrTo("context")}},
+					{Key: "instance_id", Value: &pb.AnyValue{IntValue: ptrTo[int64](10)}},
+					{Key: "node_taints", Value: &pb.AnyValue{KeyValueList: &pb.KeyValueList{
+						Values: []*pb.KeyValue{
+							{Key: "role", Value: &pb.AnyValue{StringValue: ptrTo("dev")}},
+							{Key: "cluster_load_percent", Value: &pb.AnyValue{DoubleValue: ptrTo(0.55)}},
+						},
+					}}},
+				},
+			},
+			ScopeLogs: []pb.ScopeLogs{
+				{
+					LogRecords: []pb.LogRecord{
+						{TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
+						{TimeUnixNano: 1235, SeverityNumber: 5, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
+					},
+				},
+			},
+		},
+		{
+			ScopeLogs: []pb.ScopeLogs{
+				{
+					LogRecords: []pb.LogRecord{
+						{TimeUnixNano: 2345, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-0")}},
+						{TimeUnixNano: 2346, SeverityNumber: 10, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-0-1")}},
+					},
+				},
+				{
+					LogRecords: []pb.LogRecord{
+						{TimeUnixNano: 2347, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-0")}},
+						{ObservedTimeUnixNano: 2348, SeverityNumber: 12, Body: pb.AnyValue{StringValue: ptrTo("log-line-resource-scope-1-1-1")}},
+					},
+				},
+			},
+		},
+	},
+		[]int64{1234, 1235, 2345, 2346, 2347, 2348},
+		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
+{"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
+{"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
+{"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}
+{"_msg":"log-line-resource-scope-1-1-1","severity":"Info4"}`,
+	)
+}
+
+func ptrTo[T any](s T) *T {
+	return &s
+}

app/vlinsert/opentelemetry/opentemetry_timing_test.go

@@ -0,0 +1,79 @@
+package opentelemetry
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
+)
+
+func BenchmarkParseProtobufRequest(b *testing.B) {
+	for _, scopes := range []int{1, 2} {
+		for _, rows := range []int{100, 1000} {
+			for _, attributes := range []int{5, 10} {
+				b.Run(fmt.Sprintf("scopes_%d/rows_%d/attributes_%d", scopes, rows, attributes), func(b *testing.B) {
+					benchmarkParseProtobufRequest(b, scopes, rows, attributes)
+				})
+			}
+		}
+	}
+}
+
+func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
+	blp := &insertutils.BenchmarkLogMessageProcessor{}
+	b.ReportAllocs()
+	b.SetBytes(int64(streams * rows))
+	b.RunParallel(func(pb *testing.PB) {
+		body := getProtobufBody(streams, rows, labels)
+		for pb.Next() {
+			_, err := pushProtobufRequest(body, blp)
+			if err != nil {
+				panic(fmt.Errorf("unexpected error: %w", err))
+			}
+		}
+	})
+}
+
+func getProtobufBody(scopesCount, rowsCount, attributesCount int) []byte {
+	msg := "12345678910"
+
+	attrValues := []*pb.AnyValue{
+		{StringValue: ptrTo("string-attribute")},
+		{IntValue: ptrTo[int64](12345)},
+		{DoubleValue: ptrTo(3.14)},
+	}
+	attrs := make([]*pb.KeyValue, attributesCount)
+	for j := 0; j < attributesCount; j++ {
+		attrs[j] = &pb.KeyValue{
+			Key:   fmt.Sprintf("key-%d", j),
+			Value: attrValues[j%3],
+		}
+	}
+	entries := make([]pb.LogRecord, rowsCount)
+	for j := 0; j < rowsCount; j++ {
+		entries[j] = pb.LogRecord{
+			TimeUnixNano: 12345678910, ObservedTimeUnixNano: 12345678910, Body: pb.AnyValue{StringValue: &msg},
+		}
+	}
+	scopes := make([]pb.ScopeLogs, scopesCount)
+
+	for j := 0; j < scopesCount; j++ {
+		scopes[j] = pb.ScopeLogs{
+			LogRecords: entries,
+		}
+	}
+
+	pr := pb.ExportLogsServiceRequest{
+		ResourceLogs: []pb.ResourceLogs{
+			{
+				Resource: pb.Resource{
+					Attributes: attrs,
+				},
+				ScopeLogs: scopes,
+			},
+		},
+	}
+
+	return pr.MarshalProtobuf(nil)
+}

app/vmagent/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 EXPOSE 8429
 
 ENTRYPOINT ["/vmagent-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./vmagent-prod

app/vmagent/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8429
 ENTRYPOINT ["/vmagent-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY vmagent-linux-${TARGETARCH}-prod ./vmagent-prod

app/vmagent/remotewrite/remotewrite.go

@@ -494,7 +494,8 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 				tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
 			}
 			matchIdxsPool.Put(matchIdxs)
-		} else if deduplicatorGlobal != nil {
+		}
+		if deduplicatorGlobal != nil {
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
@@ -922,7 +923,8 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
 		}
 		matchIdxsPool.Put(matchIdxs)
-	} else if rwctx.deduplicator != nil {
+	}
+	if rwctx.deduplicator != nil {
 		rwctx.deduplicator.Push(tss)
 		return true
 	}

app/vmagent/remotewrite/streamaggr.go

@@ -130,11 +130,10 @@ func initStreamAggrConfigGlobal() {
 		sasGlobal.Store(sas)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrGlobalDedupInterval.Duration()
-		if dedupInterval > 0 {
-			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
-		}
+	}
+	dedupInterval := streamAggrGlobalDedupInterval.Duration()
+	if dedupInterval > 0 {
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 	}
 }
 
@@ -152,12 +151,11 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
 		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-		if dedupInterval > 0 {
-			alias := fmt.Sprintf("dedup-%d", idx+1)
-			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
-		}
+	}
+	dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+	if dedupInterval > 0 {
+		alias := fmt.Sprintf("dedup-%d", idx+1)
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
 	}
 }
 

app/vmalert-tool/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8429
 ENTRYPOINT ["/vmalert-tool-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY vmalert-tool-linux-${TARGETARCH}-prod ./vmalert-tool-prod

app/vmalert/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 EXPOSE 8880
 
 ENTRYPOINT ["/vmalert-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./vmalert-prod

app/vmalert/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8880
 ENTRYPOINT ["/vmalert-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY vmalert-linux-${TARGETARCH}-prod ./vmalert-prod

app/vmalert/rule/group_timing_test.go

@@ -11,7 +11,7 @@ import (
 func BenchmarkGetStaleSeries(b *testing.B) {
 	ts := time.Now()
 	n := 100
-	payload := make([]prompbmarshal.TimeSeries, n)
+	payload := make([]prompbmarshal.TimeSeries, 0, n)
 	for i := 0; i < n; i++ {
 		s := fmt.Sprintf("%d", i)
 		labels := toPromLabels(b,

app/vmauth/deployment/Dockerfile

@@ -1,8 +1,8 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 EXPOSE 8427
 
 ENTRYPOINT ["/vmauth-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./vmauth-prod

app/vmauth/multiarch/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
@@ -8,5 +8,5 @@ FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 EXPOSE 8427
 ENTRYPOINT ["/vmauth-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY vmauth-linux-${TARGETARCH}-prod ./vmauth-prod

app/vmbackup/deployment/Dockerfile

@@ -1,6 +1,6 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 ENTRYPOINT ["/vmbackup-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./vmbackup-prod

app/vmbackup/multiarch/Dockerfile

@@ -1,11 +1,11 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/vmbackup-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY vmbackup-linux-${TARGETARCH}-prod ./vmbackup-prod

app/vmctl/deployment/Dockerfile

@@ -1,6 +1,6 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 ENTRYPOINT ["/vmctl-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./vmctl-prod

app/vmctl/multiarch/Dockerfile

@@ -1,11 +1,11 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/vmctl-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY vmctl-linux-${TARGETARCH}-prod ./vmctl-prod

app/vmrestore/deployment/Dockerfile

@@ -1,6 +1,6 @@
-ARG base_image
+ARG base_image=non-existing
 FROM $base_image
 
 ENTRYPOINT ["/vmrestore-prod"]
-ARG src_binary
+ARG src_binary=non-existing
 COPY $src_binary ./vmrestore-prod

app/vmrestore/multiarch/Dockerfile

@@ -1,11 +1,11 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image=non-existing
 FROM $certs_image AS certs
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENTRYPOINT ["/vmrestore-prod"]
-ARG TARGETARCH
+ARG TARGETARCH=non-existing
 COPY vmrestore-linux-${TARGETARCH}-prod ./vmrestore-prod

dashboards/victoriametrics-cluster.json

@@ -4496,7 +4496,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "The number of rows or bytes that vminesrt internal buffer contains at the moment.",
+          "description": "The number of rows or bytes that vminsert internal buffer contains at the moment.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/victoriametrics-cluster.json

@@ -4497,7 +4497,7 @@
             "type": "victoriametrics-datasource",
             "uid": "$ds"
           },
-          "description": "The number of rows or bytes that vminesrt internal buffer contains at the moment.",
+          "description": "The number of rows or bytes that vminsert internal buffer contains at the moment.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/vmagent.json

@@ -5088,7 +5088,7 @@
               "editorMode": "code",
               "expr": "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5184,7 +5184,7 @@
               "editorMode": "code",
               "expr": "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5286,7 +5286,7 @@
               "editorMode": "code",
               "expr": "sum(rate(vm_streamaggr_output_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; output={{output}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5383,7 +5383,7 @@
               "editorMode": "code",
               "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "instant": false,
-              "legendFormat": "aggregation: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "aggregation: {{url}} ({{job}}): {{path}}:{{position}}",
               "range": true,
               "refId": "A"
             },
@@ -5396,7 +5396,7 @@
               "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "hide": false,
               "instant": false,
-              "legendFormat": "deduplication: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "deduplication: {{url}} ({{job}}): {{path}}:{{position}}",
               "range": true,
               "refId": "B"
             }
@@ -5494,7 +5494,7 @@
               "editorMode": "code",
               "expr": "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5591,7 +5591,7 @@
               "editorMode": "code",
               "expr": "histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }

dashboards/vmagent.json

@@ -5087,7 +5087,7 @@
               "editorMode": "code",
               "expr": "sum(rate(vm_streamaggr_matched_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5183,7 +5183,7 @@
               "editorMode": "code",
               "expr": "sum(rate(vm_streamaggr_ignored_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0) without (instance, pod)",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5285,7 +5285,7 @@
               "editorMode": "code",
               "expr": "sum(rate(vm_streamaggr_output_samples_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod) > 0",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; output={{output}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5382,7 +5382,7 @@
               "editorMode": "code",
               "expr": "increase(vm_streamaggr_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "instant": false,
-              "legendFormat": "aggregation: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "aggregation: {{url}} ({{job}}): {{path}}:{{position}}",
               "range": true,
               "refId": "A"
             },
@@ -5395,7 +5395,7 @@
               "expr": "increase(vm_streamaggr_dedup_flush_timeouts_total{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]) > 0",
               "hide": false,
               "instant": false,
-              "legendFormat": "deduplication: {{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "deduplication: {{url}} ({{job}}): {{path}}:{{position}}",
               "range": true,
               "refId": "B"
             }
@@ -5493,7 +5493,7 @@
               "editorMode": "code",
               "expr": "histogram_quantile(0.99, sum(rate(vm_streamaggr_samples_lag_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval])) without (instance, pod))",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
@@ -5590,7 +5590,7 @@
               "editorMode": "code",
               "expr": "histogram_quantile(0.99, rate(vm_streamaggr_dedup_flush_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\", url=~\"$url\"}[$__rate_interval]))",
               "instant": false,
-              "legendFormat": "{{url}} ({{job}}): match={{match}}; outputs={{outputs}}",
+              "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }

deployment/docker/alerts-vmagent.yml

@@ -27,8 +27,9 @@ groups:
           severity: warning
         annotations:
           dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=79&var-instance={{ $labels.instance }}"
-          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by 
-          remote-write server data blocks. Check the logs to find the reason for rejects."
+          summary: "Vmagent is dropping data blocks that are rejected by remote storage"
+          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by 
+            remote-write server data blocks. Check the logs to find the reason for rejects."
 
       - alert: TooManyScrapeErrors
         expr: increase(vm_promscrape_scrapes_failed_total[5m]) > 0
@@ -37,7 +38,8 @@ groups:
           severity: warning
         annotations:
           dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=31&var-instance={{ $labels.instance }}"
-          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to scrape targets for last 15m"
+          summary: "Vmagent fails to scrape one or more targets"
+          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to scrape targets for last 15m"
 
       - alert: TooManyWriteErrors
         expr: |
@@ -49,7 +51,8 @@ groups:
           severity: warning
         annotations:
           dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=77&var-instance={{ $labels.instance }}"
-          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} responds with errors to write requests for last 15m."
+          summary: "Vmagent responds with too many errors on data ingestion protocols"
+          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} responds with errors to write requests for last 15m."
 
       - alert: TooManyRemoteWriteErrors
         expr: rate(vmagent_remotewrite_retries_count_total[5m]) > 0

deployment/docker/base/Dockerfile

@@ -1,6 +1,6 @@
 # See https://medium.com/on-docker/use-multi-stage-builds-to-inject-ca-certs-ad1e8f01de1b
-ARG certs_image
-ARG root_image
+ARG certs_image=non-existing
+ARG root_image==non-existing
 FROM $certs_image AS certs
 
 RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates

deployment/docker/builder/Dockerfile

@@ -1,4 +1,4 @@
-ARG go_builder_image
+ARG go_builder_image=non-existing
 FROM $go_builder_image
 STOPSIGNAL SIGINT
 RUN apk add git gcc musl-dev make wget --no-cache && \

deployment/docker/docker-compose-cluster.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.102.1
+    image: victoriametrics/vmagent:v1.103.0
     depends_on:
       - "vminsert"
     ports:
@@ -39,7 +39,7 @@ services:
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
     container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.102.1-cluster
+    image: victoriametrics/vmstorage:v1.103.0-cluster
     ports:
       - 8482
       - 8400
@@ -51,7 +51,7 @@ services:
     restart: always
   vmstorage-2:
     container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.102.1-cluster
+    image: victoriametrics/vmstorage:v1.103.0-cluster
     ports:
       - 8482
       - 8400
@@ -66,7 +66,7 @@ services:
   # pre-process them and distributes across configured vmstorage shards.
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.102.1-cluster
+    image: victoriametrics/vminsert:v1.103.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -81,7 +81,7 @@ services:
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
     container_name: vmselect-1
-    image: victoriametrics/vmselect:v1.102.1-cluster
+    image: victoriametrics/vmselect:v1.103.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -94,7 +94,7 @@ services:
     restart: always
   vmselect-2:
     container_name: vmselect-2
-    image: victoriametrics/vmselect:v1.102.1-cluster
+    image: victoriametrics/vmselect:v1.103.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -112,7 +112,7 @@ services:
   # It can be used as an authentication proxy.
   vmauth:
     container_name: vmauth
-    image: victoriametrics/vmauth:v1.102.1
+    image: victoriametrics/vmauth:v1.103.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -127,7 +127,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.102.1
+    image: victoriametrics/vmalert:v1.103.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/docker-compose-victorialogs.yml

@@ -16,7 +16,7 @@ services:
       - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
       - ./../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json
     environment:
-      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.2.2/victorialogs-datasource-v0.2.2.zip;victorialogs-datasource"
+      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.4.0/victorialogs-datasource-v0.4.0.zip;victorialogs-datasource"
       - "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victorialogs-datasource"
     networks:
       - vm_net
@@ -55,7 +55,7 @@ services:
   # scraping, storing metrics and serve read requests.
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.102.1
+    image: victoriametrics/victoria-metrics:v1.103.0
     ports:
       - 8428:8428
     volumes:

deployment/docker/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.102.1
+    image: victoriametrics/vmagent:v1.103.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -22,7 +22,7 @@ services:
   # storing metrics and serve read requests.
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.102.1
+    image: victoriametrics/victoria-metrics:v1.103.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -65,7 +65,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.102.1
+    image: victoriametrics/vmalert:v1.103.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/victorialogs/filebeat-syslog/compose.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   filebeat-vlogs:
     image: docker.elastic.co/beats/filebeat:8.8.1

deployment/docker/victorialogs/filebeat/compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   filebeat-victorialogs:
     image: docker.elastic.co/beats/filebeat:8.8.1

deployment/docker/victorialogs/fluentbit-docker/fluent-bit.conf

@@ -1,30 +0,0 @@
-[INPUT]
-    name              tail
-    path              /var/lib/docker/containers/**/*.log
-    path_key         path
-    multiline.parser  docker, cri
-    Parser docker
-    Docker_Mode  On
-
-[INPUT]
-    Name     syslog
-    Listen   0.0.0.0
-    Port     5140
-    Parser   syslog-rfc3164
-    Mode     tcp
-
-[SERVICE]
-    Flush        1
-    Parsers_File parsers.conf
-
-[Output]
-    Name http
-    Match *
-    host victorialogs
-    port 9428
-    compress gzip
-    uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date
-    format json_lines
-    json_date_format iso8601
-    header AccountID 0
-    header ProjectID 0

deployment/docker/victorialogs/fluentbit/compose.yml

@@ -1,8 +1,6 @@
-version: "3"
-
 services:
   fluentbit:
-    image: cr.fluentbit.io/fluent/fluent-bit:3.0.2
+    image: cr.fluentbit.io/fluent/fluent-bit:3.0.7
     volumes:
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
       - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf

deployment/docker/victorialogs/fluentbit/fluent-bit.conf

@@ -0,0 +1,54 @@
+[INPUT]
+    name             tail
+    path             /var/lib/docker/containers/**/*.log
+    path_key         path
+    multiline.parser docker, cri
+    Parser           docker
+    Docker_Mode      On
+
+[INPUT]
+    Name     syslog
+    Listen   0.0.0.0
+    Port     5140
+    Parser   syslog-rfc3164
+    Mode     tcp
+
+[SERVICE]
+    Flush        1
+    Parsers_File parsers.conf
+
+[OUTPUT]
+    Name http
+    Match *
+    host victorialogs
+    port 9428
+    compress gzip
+    uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date
+    format json_lines
+    json_date_format iso8601
+    header AccountID 0
+    header ProjectID 0
+
+[OUTPUT]
+    Name es
+    Match *
+    host victorialogs
+    port 9428
+    compress gzip
+    path /insert/elasticsearch
+    header AccountID 0
+    header ProjectID 0
+    header VL-Stream-Fields path
+    header VL-Msg-Field log
+    header VL-Time-Field @timestamp
+
+[OUTPUT]
+    name       loki
+    match      *
+    host       victorialogs
+    uri        /insert/loki/api/v1/push
+    port       9428
+    label_keys $path,$log,$time
+    header     VL-Msg-Field log
+    header     VL-Time-Field time
+    header     VL-Stream-Fields path

deployment/docker/victorialogs/logstash/Dockerfile

@@ -1,3 +1,5 @@
 FROM docker.elastic.co/logstash/logstash:8.8.1
 
-RUN bin/logstash-plugin install logstash-output-opensearch
+RUN bin/logstash-plugin install \
+    logstash-output-opensearch \
+    logstash-output-loki

deployment/docker/victorialogs/logstash/compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   logstash:
     build:

deployment/docker/victorialogs/logstash/pipeline.conf

@@ -17,4 +17,12 @@ output {
         "_time_field" => "@timestamp"
     }
   }
-}
+  http {
+    url => "http://victorialogs:9428/insert/jsonline?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp"
+    format => "json"
+    http_method => "post"
+  }
+  loki {
+     url => "http://victorialogs:9428/insert/loki/api/v1/push?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp"
+  }
+}

deployment/docker/victorialogs/opentelemetry-collector/.gitignore

@@ -0,0 +1 @@
+**/logs

deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/README.md

@@ -0,0 +1,27 @@
+# Docker compose OpenTelemetry Elasticsearch integration with VictoriaLogs for docker
+
+The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs
+
+To spin-up environment  run the following command:
+```
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+```
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* collector - vector is configured to collect logs from the `docker`, you can find configuration in the `config.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics.
+* VictoriaLogs - the log database, it accepts the data from `collector` by elastic protocol
+* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics`
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/compose.yml

@@ -0,0 +1,48 @@
+services:
+  collector:
+    image: docker.io/otel/opentelemetry-collector-contrib:0.102.1
+    restart: on-failure
+    volumes:
+      - $PWD/logs:/tmp/logs
+      - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml
+    depends_on:
+      victorialogs:
+        condition: service_healthy
+      victoriametrics:
+        condition: service_healthy
+
+  victorialogs:
+    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    volumes:
+      - victorialogs-vector-docker-vl:/vlogs
+    ports:
+      - '9428:9428'
+    command:
+      - -storageDataPath=/vlogs
+      - -loggerFormat=json
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+  victoriametrics:
+    image: victoriametrics/victoria-metrics:latest
+    ports:
+      - '8428:8428'
+    command:
+      - -storageDataPath=/vmsingle
+      - -promscrape.config=/promscrape.yml
+      - -loggerFormat=json
+    volumes:
+      - victorialogs-vector-docker-vm:/vmsingle
+      - ./scrape.yml:/promscrape.yml
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+volumes:
+  victorialogs-vector-docker-vl:
+  victorialogs-vector-docker-vm:

deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/config.yaml

@@ -0,0 +1,14 @@
+exporters:
+  elasticsearch:
+    endpoints:
+      - http://victorialogs:9428/insert/elasticsearch
+receivers:
+  filelog:
+    include: [/tmp/logs/*.log]
+    resource:
+      region: us-east-1
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [elasticsearch]

deployment/docker/victorialogs/opentelemetry-collector/loki/README.md

@@ -0,0 +1,27 @@
+# Docker compose OpenTelemetry Loki integration with VictoriaLogs for docker
+
+The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs
+
+To spin-up environment  run the following command:
+```
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+```
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* collector - vector is configured to collect logs from the `docker`, you can find configuration in the `config.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics.
+* VictoriaLogs - the log database, it accepts the data from `collector` by Loki protocol
+* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics`
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

deployment/docker/victorialogs/opentelemetry-collector/loki/compose.yml

@@ -0,0 +1,48 @@
+services:
+  collector:
+    image: docker.io/otel/opentelemetry-collector-contrib:0.102.1
+    restart: on-failure
+    volumes:
+      - $PWD/logs:/tmp/logs
+      - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml
+    depends_on:
+      victorialogs:
+        condition: service_healthy
+      victoriametrics:
+        condition: service_healthy
+
+  victorialogs:
+    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    volumes:
+      - victorialogs-vector-docker-vl:/loki
+    ports:
+      - '9428:9428'
+    command:
+      - -storageDataPath=/loki
+      - -loggerFormat=json
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+  victoriametrics:
+    image: victoriametrics/victoria-metrics:latest
+    ports:
+      - '8428:8428'
+    command:
+      - -storageDataPath=/vmsingle
+      - -promscrape.config=/promscrape.yml
+      - -loggerFormat=json
+    volumes:
+      - victorialogs-vector-docker-vm:/vmsingle
+      - ./scrape.yml:/promscrape.yml
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+volumes:
+  victorialogs-vector-docker-vl:
+  victorialogs-vector-docker-vm:

deployment/docker/victorialogs/opentelemetry-collector/loki/config.yaml

@@ -0,0 +1,13 @@
+exporters:
+  loki:
+    endpoint: http://victorialogs:9428/insert/loki/api/v1/push
+receivers:
+  filelog:
+    include: [/tmp/logs/*.log]
+    resource:
+      region: us-east-1
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [loki]

deployment/docker/victorialogs/opentelemetry-collector/loki/scrape.yml

@@ -0,0 +1,11 @@
+scrape_configs:
+  - job_name: "victoriametrics"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victoriametrics:8428
+  - job_name: "victorialogs"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victorialogs:9428

deployment/docker/victorialogs/opentelemetry-collector/otlp/README.md

@@ -0,0 +1,27 @@
+# Docker compose OpenTelemetry OTLP integration with VictoriaLogs for docker
+
+The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs
+
+To spin-up environment  run the following command:
+```
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+```
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* collector - vector is configured to collect logs from the `docker`, you can find configuration in the `config.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics.
+* VictoriaLogs - the log database, it accepts the data from `collector` by otlp protocol
+* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics`
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

deployment/docker/victorialogs/opentelemetry-collector/otlp/compose.yml

@@ -0,0 +1,48 @@
+services:
+  collector:
+    image: docker.io/otel/opentelemetry-collector-contrib:0.102.1
+    restart: on-failure
+    volumes:
+      - $PWD/logs:/tmp/logs
+      - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml
+    depends_on:
+      victorialogs:
+        condition: service_healthy
+      victoriametrics:
+        condition: service_healthy
+
+  victorialogs:
+    image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs
+    volumes:
+      - victorialogs-vector-docker-vl:/otlp
+    ports:
+      - '9428:9428'
+    command:
+      - -storageDataPath=/otlp
+      - -loggerFormat=json
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+  victoriametrics:
+    image: victoriametrics/victoria-metrics:latest
+    ports:
+      - '8428:8428'
+    command:
+      - -storageDataPath=/vmsingle
+      - -promscrape.config=/promscrape.yml
+      - -loggerFormat=json
+    volumes:
+      - victorialogs-vector-docker-vm:/vmsingle
+      - ./scrape.yml:/promscrape.yml
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+volumes:
+  victorialogs-vector-docker-vl:
+  victorialogs-vector-docker-vm:

deployment/docker/victorialogs/opentelemetry-collector/otlp/config.yaml

@@ -0,0 +1,15 @@
+exporters:
+  otlphttp:
+    logs_endpoint: http://victorialogs:9428/insert/opentelemetry/v1/logs
+  debug:
+    verbosity: detailed
+receivers:
+  filelog:
+    include: [/tmp/logs/*.log]
+    resource:
+      region: us-east-1
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [otlphttp, debug]

deployment/docker/victorialogs/opentelemetry-collector/otlp/scrape.yml

@@ -0,0 +1,11 @@
+scrape_configs:
+  - job_name: "victoriametrics"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victoriametrics:8428
+  - job_name: "victorialogs"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victorialogs:9428

deployment/docker/victorialogs/opentelemetry-collector/syslog/README.md

@@ -0,0 +1,27 @@
+# Docker compose OpenTelemetry Syslog integration with VictoriaLogs for docker
+
+The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs
+
+To spin-up environment  run the following command:
+```
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+```
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* collector - vector is configured to collect logs from the `docker`, you can find configuration in the `config.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics.
+* VictoriaLogs - the log database, it accepts the data from `collector` by syslog protocol
+* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics`
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
+
+Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

deployment/docker/victorialogs/opentelemetry-collector/syslog/compose.yml

@@ -0,0 +1,49 @@
+services:
+  collector:
+    image: docker.io/otel/opentelemetry-collector-contrib:0.107.0
+    restart: on-failure
+    volumes:
+      - $PWD/logs:/tmp/logs
+      - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml
+    depends_on:
+      victorialogs:
+        condition: service_healthy
+      victoriametrics:
+        condition: service_healthy
+
+  victorialogs:
+    image: docker.io/victoriametrics/victoria-logs:v0.28.0-victorialogs
+    volumes:
+      - victorialogs-vector-docker-vl:/syslog
+    ports:
+      - '9428:9428'
+    command:
+      - -storageDataPath=/syslog
+      - -syslog.listenAddr.tcp=:5410
+      - -syslog.useLocalTimestamp.tcp
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+  victoriametrics:
+    image: victoriametrics/victoria-metrics:latest
+    ports:
+      - '8428:8428'
+    command:
+      - -storageDataPath=/vmsingle
+      - -promscrape.config=/promscrape.yml
+      - -loggerFormat=json
+    volumes:
+      - victorialogs-vector-docker-vm:/vmsingle
+      - ./scrape.yml:/promscrape.yml
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+volumes:
+  victorialogs-vector-docker-vl:
+  victorialogs-vector-docker-vm:

deployment/docker/victorialogs/opentelemetry-collector/syslog/config.yaml

@@ -0,0 +1,24 @@
+exporters:
+  syslog:
+    network: tcp
+    endpoint: victorialogs
+    port: 5410
+    tls:
+      insecure: true
+  debug:
+    verbosity: detailed
+processors:
+  transform:
+    log_statements:
+      - context: log
+        statements:
+          - set(attributes["message"], body)
+receivers:
+  filelog:
+    include: [/tmp/logs/*.log]
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [syslog, debug]
+      processors: [transform]

deployment/docker/victorialogs/opentelemetry-collector/syslog/scrape.yml

@@ -0,0 +1,11 @@
+scrape_configs:
+  - job_name: "victoriametrics"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victoriametrics:8428
+  - job_name: "victorialogs"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victorialogs:9428

deployment/docker/victorialogs/promtail/compose.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   promtail:
     image: grafana/promtail:2.8.2

deployment/docker/victorialogs/telegraf-docker/README.md

@@ -0,0 +1,25 @@
+# Docker compose Telegraf integration with VictoriaLogs for docker
+
+The folder contains the example of integration of [telegraf](https://www.influxdata.com/time-series-platform/telegraf/) with VictoriaLogs
+
+To spin-up environment  run the following command:
+```
+docker compose up -d 
+```
+
+To shut down the docker-compose environment run the following command:
+```
+docker compose down
+docker compose rm -f
+```
+
+The docker compose file contains the following components:
+
+* telegraf - telegraf is configured to collect logs from the `docker`, you can find configuration in the `telegraf.conf`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics.
+* VictoriaLogs - the log database, it accepts the data from `telegraf` by elastic protocol
+* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics`
+
+Querying the data
+
+* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui`
+* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)

deployment/docker/victorialogs/telegraf-docker/compose.yml

@@ -0,0 +1,55 @@
+services:
+  telegraf:
+    image: bitnami/telegraf:1.31.0
+    restart: on-failure
+    volumes:
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+      - type: bind
+        source: /var/lib/docker
+        target: /var/lib/docker
+      - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro
+    command: --config /etc/telegraf/telegraf.conf
+    depends_on:
+      victorialogs:
+        condition: service_healthy
+      victoriametrics:
+        condition: service_healthy
+
+  victorialogs:
+    image: docker.io/victoriametrics/victoria-logs:v0.20.2-victorialogs
+    volumes:
+      - victorialogs-vector-docker-vl:/vlogs
+    ports:
+      - '9428:9428'
+    command:
+      - -storageDataPath=/vlogs
+      - -loggerFormat=json
+      - -syslog.listenAddr.tcp=0.0.0.0:8094
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+  victoriametrics:
+    image: victoriametrics/victoria-metrics:latest
+    ports:
+      - '8428:8428'
+    command:
+      - -storageDataPath=/vmsingle
+      - -promscrape.config=/promscrape.yml
+      - -loggerFormat=json
+    volumes:
+      - victorialogs-vector-docker-vm:/vmsingle
+      - ./scrape.yml:/promscrape.yml
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+
+volumes:
+  victorialogs-vector-docker-vl:
+  victorialogs-vector-docker-vm:

deployment/docker/victorialogs/telegraf-docker/scrape.yml

@@ -0,0 +1,11 @@
+scrape_configs:
+  - job_name: "victoriametrics"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victoriametrics:8428
+  - job_name: "victorialogs"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victorialogs:9428

deployment/docker/victorialogs/telegraf-docker/telegraf.conf

@@ -0,0 +1,70 @@
+[agent]
+  interval = "10s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 100000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  debug = false
+  quiet = false
+  logtarget = "file"
+  logfile = "/dev/null"
+  hostname = "pop-os"
+  omit_hostname = false
+
+[[inputs.cpu]]
+
+[[outputs.http]]
+  url = "http://victorialogs:9428/insert/jsonline?_msg_field=fields.msg&_stream_fields=tags.log_source,tags.metric_type"
+  data_format = "json"
+  namepass = ["docker_log"]
+  use_batch_format = false
+
+[[outputs.loki]]
+  domain = "http://victorialogs:9428"
+  endpoint = "/insert/loki/api/v1/push?_msg_field=docker_log.msg&_time_field=@timestamp&_stream_fields=log_source,metric_type"
+  namepass = ["docker_log"]
+  gzip_request = true
+  sanitize_label_names = true
+
+[[outputs.syslog]]
+  address = "tcp://victorialogs:8094"
+  namepass = ["docker_log"]
+
+[[outputs.elasticsearch]]
+  urls = ["http://victorialogs:9428/insert/elasticsearch"]
+  timeout = "1m"
+  flush_interval = "30s"
+  enable_sniffer = false
+  health_check_interval = "0s"
+  index_name = "device_log-%Y.%m.%d"
+  manage_template = false
+  template_name = "telegraf"
+  overwrite_template = false
+  namepass = ["docker_log"]
+  [outputs.elasticsearch.headers]
+    VL-Msg-Field = "docker_log.msg"
+    VL-Time-Field = "@timestamp"
+    VL-Stream-Fields = "tag.log_source,tag.metric_type"
+
+[[outputs.http]]
+  url = "http://victoriametrics:8428/api/v1/write"
+  data_format = "prometheusremotewrite"
+  namepass = ["cpu"]
+  [outputs.http.headers]
+    Content-Type = "application/x-protobuf"
+    Content-Encoding = "snappy"
+    X-Prometheus-Remote-Write-Version = "0.1.0"
+
+[[inputs.docker_log]]
+  [inputs.docker_log.tags]
+     metric_type = "logs"
+     log_source = "telegraf"
+
+[[processors.rename]]
+  namepass = ["docker_log"]
+  [[processors.rename.replace]]
+    field = "message"
+    dest = "msg"

deployment/docker/victorialogs/vector-docker/vector.toml

@@ -1,75 +0,0 @@
-[api]
-  enabled = true
-  address = "0.0.0.0:8686"
-
-
-# ---------------------------------------------
-# Docker logs -> VictoriaLogs
-# ---------------------------------------------
-
-[sources.docker]
-  type = "docker_logs"
-
-[transforms.msg_parser]
-  type = "remap"
-  inputs = ["docker"]
-  source = '''
-  .log = parse_json!(.message)
-  del(.message)
-  '''
-
-[sinks.vlogs]
-  type = "http"
-  inputs = [ "msg_parser" ]
-  uri = "http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp"
-  encoding.codec = "json"
-  framing.method = "newline_delimited"
-  compression = "gzip"
-  healthcheck.enabled = false
-
-  [sinks.vlogs.request.headers]
-    AccountID = "0"
-    ProjectID = "0"
-
-
-# ---------------------------------------------
-# Generted demo logs -> VictoriaLogs
-# ---------------------------------------------
-
-[sources.demo]
-  type = "demo_logs"
-  format = "apache_common"
-  interval = 10
-
-[sinks.vlogs_demo]
-  type = "elasticsearch"
-  inputs = [ "demo" ]
-  endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  compression = "gzip"
-  healthcheck.enabled = false
-
-  [sinks.vlogs_demo.query]
-    _msg_field = "message"
-    _time_field = "timestamp"
-    _stream_fields = "source_type"
-
-  [sinks.vlogs_demo.request.headers]
-    AccountID = "0"
-    ProjectID = "0"
-
-
-
-# ---------------------------------------------
-# Vector Metrics -> VictoriaMetrics
-# ---------------------------------------------
-
-[sources.vector_metrics]
-  type = "internal_metrics"
-
-[sinks.victoriametrics]
-  type = "prometheus_remote_write"
-  endpoint = "http://victoriametrics:8428/api/v1/write"
-  inputs = ["vector_metrics"]
-  healthcheck.enabled = false

deployment/docker/victorialogs/vector/README.md

@@ -15,7 +15,7 @@ docker compose rm -f
 
 The docker compose file contains the following components:
 
-* vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.toml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics.
+* vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics.
 * VictoriaLogs - the log database, it accepts the data from `vector` by elastic protocol
 * VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics`
 
@@ -25,37 +25,39 @@ Querying the data
 * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
 
 
-the example of vector configuration(`vector.toml`)
+the example of vector configuration(`vector.yaml`)
 
 ```
-[sources.docker]
-  type = "docker_logs"
-
-[transforms.msg_parser]
-  type = "remap"
-  inputs = ["docker"]
-  source = '''
-  .log = parse_json!(.message)
-  del(.message)
-  '''
-
-[sinks.vlogs]
-  type = "elasticsearch"
-  inputs = [ "msg_parser" ]
-  endpoints = [ "http://victorialogs:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  compression = "gzip"
-  healthcheck.enabled = false
-
-  [sinks.vlogs.query]
-    _msg_field = "log.msg"
-    _time_field = "timestamp"
-    _stream_fields = "source_type,host,container_name"
-
-  [sinks.vlogs.request.headers]
-    AccountID = "0"
-    ProjectID = "0"
+sources:
+  docker:
+    type: docker_logs
+transforms:
+  msg_parser:
+    type: remap
+    inputs:
+      - docker
+    source: |
+      .log = parse_json!(.message)
+      del(.message)
+sinks:
+  vlogs_es:
+    type: elasticsearch
+    inputs:
+      - msg_parser
+    endpoints:
+      - http://victorialogs:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    compression: gzip
+    healthcheck.enabled: false
+    query:
+      _msg_field: log.msg
+      _time_field: timestamp
+      _stream_fields: source_type,host,container_name
+    request:
+      headers:
+        AccountID: "0"
+        ProjectID: "0"
 ```
 
 Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

deployment/docker/victorialogs/vector/compose.yml

@@ -1,8 +1,6 @@
-version: '3'
-
 services:
   vector:
-    image: docker.io/timberio/vector:0.30.0-distroless-static
+    image: docker.io/timberio/vector:0.38.0-distroless-static
     restart: on-failure
     volumes:
       - type: bind
@@ -11,7 +9,7 @@ services:
       - type: bind
         source: /var/lib/docker
         target: /var/lib/docker
-      - ./vector.toml:/etc/vector/vector.toml:ro
+      - ./vector.yaml:/etc/vector/vector.yaml:ro
     user: root
     ports:
       - '8686:8686'
@@ -31,7 +29,7 @@ services:
       - -storageDataPath=/vlogs
       - -loggerFormat=json
     healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://localhost:9428/health"]
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"]
       interval: 1s
       timeout: 1s
       retries: 10
@@ -48,7 +46,7 @@ services:
       - victorialogs-vector-docker-vm:/vmsingle
       - ./scrape.yml:/promscrape.yml
     healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://localhost:8428/health"]
+      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"]
       interval: 1s
       timeout: 1s
       retries: 10

deployment/docker/victorialogs/vector/scrape.yml

@@ -0,0 +1,11 @@
+scrape_configs:
+  - job_name: "victoriametrics"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victoriametrics:8428
+  - job_name: "victorialogs"
+    scrape_interval: 30s
+    static_configs:
+      - targets:
+          - victorialogs:9428

deployment/docker/victorialogs/vector/vector.yaml

@@ -0,0 +1,73 @@
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+sources:
+  docker:
+    type: docker_logs
+  demo:
+    type: demo_logs
+    format: json
+  metrics:
+    type: internal_metrics
+transforms:
+  msg_parser:
+    type: remap
+    inputs:
+      - docker
+    source: |
+      .log = parse_json!(.message)
+      del(.message)
+sinks:
+  vlogs_http:
+    type: http
+    inputs:
+      - msg_parser
+    uri: http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp
+    encoding:
+      codec: json
+    framing:
+      method: newline_delimited
+    compression: gzip
+    healthcheck:
+      enabled: false
+    request:
+      headers:
+        AccountID: '0'
+        ProjectID: '0'
+  vlogs_loki:
+    type: loki
+    inputs:
+      - demo
+    endpoint: http://victorialogs:9428/insert/loki/
+    compression: gzip
+    path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source
+    encoding:
+      codec: json
+    labels:
+      source: vector
+  victoriametrics:
+    type: prometheus_remote_write
+    endpoint: http://victoriametrics:8428/api/v1/write
+    inputs: 
+      - metrics
+    healthcheck:
+      enabled: false
+  vlogs_es:
+    type: elasticsearch
+    inputs:
+      - demo
+    endpoints:
+      - http://victorialogs:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    compression: gzip
+    healthcheck:
+      enabled: false
+    query:
+      _msg_field: message
+      _time_field: timestamp
+      _stream_fields: source_type
+    request:
+      headers:
+        AccountID: '0'
+        ProjectID: '0'

deployment/docker/vm-datasource/docker-compose-cluster.yml

@@ -16,6 +16,6 @@ services:
       - ./../../dashboards/vm/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
       - ./../../dashboards/vm/vmauth.json:/var/lib/grafana/dashboards/vmauth.json
     environment:
-      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.8.2/victoriametrics-datasource-v0.8.2.zip;victoriametrics-datasource"
+      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.9.0/victoriametrics-datasource-v0.9.0.zip;victoriametrics-datasource"
       - "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource"
     restart: always

deployment/docker/vm-datasource/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - ./../../dashboards/vm/vmagent.json:/var/lib/grafana/dashboards/vmagent.json
       - ./../../dashboards/vm/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
     environment:
-      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.8.2/victoriametrics-datasource-v0.8.2.zip;victoriametrics-datasource"
+      - "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.9.0/victoriametrics-datasource-v0.9.0.zip;victoriametrics-datasource"
       - "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource"
     networks:
       - vm_net

deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.102.0
+    image: victoriametrics/vmagent:v1.103.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.102.0
+    image: victoriametrics/victoria-metrics:v1.103.0
     ports:
       - 8428:8428
     volumes:
@@ -51,7 +51,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.102.0
+    image: victoriametrics/vmalert:v1.103.0
     depends_on:
       - "victoriametrics"
     ports:

deployment/logs-benchmark/docker-compose.yml

@@ -46,7 +46,7 @@ services:
       - '--config=/config.yml'
 
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.102.0
+    image: victoriametrics/victoria-metrics:v1.103.0
     ports:
       - '8428:8428'
     command:

deployment/marketplace/digitialocean/one-click-droplet/files/etc/update-motd.d/99-one-click

@@ -19,8 +19,8 @@ On the server:
   * VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
 
 ********************************************************************************
-  # This image includes v1.102.0 release of VictoriaMetrics.
-  # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0
+  # This image includes v1.103.0 release of VictoriaMetrics.
+  # See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.103.0
 
   # Welcome to VictoriaMetrics droplet!
 

docs/LTS-releases.md

@@ -13,11 +13,14 @@ Every LTS line receives bugfixes and [security fixes](https://github.com/Victori
 the initial release. New LTS lines are published every 6 months, so the latest two LTS lines are supported at any given moment. This gives up to 6 months
 for the migration to new LTS lines for [VictoriaMetrics Enterprise](https://docs.victoriametrics.com/enterprise/) users.
 
+LTS releases are published for [Enterprise versions of VictoriaMetrics](https://docs.victoriametrics.com/enterprise/) only.
+When a new LTS line is created, the new LTS release might be publicly available for everyone until the new major OS release will be published.
+
 All the bugfixes and security fixes, which are included in LTS releases, are also available in [the latest release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
 so non-enterprise users are advised to regularly [upgrade](https://docs.victoriametrics.com/#how-to-upgrade-victoriametrics) VictoriaMetrics products
 to [the latest available releases](https://docs.victoriametrics.com/changelog/).
 
 ## Currently supported LTS release lines
 
-- 1.102.x - the latest one is [v1.102.1 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.1)
-- 1.97.x - the latest one is [v1.97.6 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.6)
+- 1.102.x - the latest one is [v1.102.2 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.2)
+- 1.97.x - the latest one is [v1.97.7 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.7)

docs/Quick-Start.md

@@ -11,16 +11,18 @@ aliases:
 ---
 ## How to install
 
-VictoriaMetrics is distributed in two forms:
+VictoriaMetrics is distributed in the following forms:
+
 * [Single-server-VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) - all-in-one
   binary, which is very easy to use and maintain.
   Single-server-VictoriaMetrics perfectly scales vertically and easily handles millions of metrics/s;
 * [VictoriaMetrics Cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) - set of components
   for building horizontally scalable clusters.
+* [VictoriaMetrics Cloud](https://cloud.victoriametrics.com/signUp?utm_source=website&utm_campaign=docs_vm_quickstart_guide) - allows 
+  users to run VictoriaMetrics, hosted on AWS, without the need to perform typical DevOps tasks such as proper configuration, monitoring, logs collection, access protection, software updates, backups, etc. 
 
-Single-server-VictoriaMetrics VictoriaMetrics is available as:
+VictoriaMetrics is available as:
 
-* [VictoriaMetrics Cloud](https://cloud.victoriametrics.com/signUp?utm_source=website&utm_campaign=docs_vm_quickstart) – the most cost-efficient hosted monitoring platform, operated by VictoriaMetrics core team.
 * [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/)
 * [Helm Charts](https://github.com/VictoriaMetrics/helm-charts#list-of-charts)
 * [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
@@ -37,6 +39,12 @@ and [Grafana setup](https://docs.victoriametrics.com/single-server-victoriametri
 
 VictoriaMetrics is developed at a fast pace, so it is recommended periodically checking the [CHANGELOG](https://docs.victoriametrics.com/changelog/) and performing [regular upgrades](https://docs.victoriametrics.com/#how-to-upgrade-victoriametrics).
 
+### Starting VictoriaMetrics Single Node or Cluster on VictoriaMetrics Cloud {anchor="starting-vm-on-cloud"}
+
+The following steps will guide you through starting VictoriaMetrics on VictoriaMetrics Cloud:
+1. Go to [VictoriaMetrics Cloud](https://cloud.victoriametrics.com/signUp?utm_source=website&utm_campaign=docs_vm_quickstart_guide) and sign up (it's free).
+1. After signing up, you will be immediately granted $200 of trial credits you can spend on running Single node or Cluster.
+1. Navigate to the VictoriaMetrics Cloud [quick start](https://docs.victoriametrics.com/victoriametrics-cloud/quickstart/#creating-deployments) guide for detailed instructions.
 
 ### Starting VictoriaMetrics Single Node via Docker {anchor="starting-vm-single-via-docker"}
 

docs/Release-Guide.md

@@ -157,11 +157,4 @@ Once updated, run the following commands:
 1. Update the version of VictoriaMetrics components at [https://github.com/VictoriaMetrics/ansible-playbooks](https://github.com/VictoriaMetrics/ansible-playbooks).
 1. Commit changes.
 1. Create a new tag with `git tag -sm <TAG> <TAG>`.
-1. Push the changes with the new tag. This automatically publishes the new versions to galaxy.ansible.com.
-
-## RPM packages
-
-### Bump the version of components
-
-1. Update the version of VictoriaMetrics components at [https://github.com/VictoriaMetrics/victoriametrics-lts-rpm](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm).
-1. Commit and push changes to the repository. This will automatically build and publish new versions of RPM packages.
+1. Push the changes with the new tag. This automatically publishes the new versions to galaxy.ansible.com.

docs/VictoriaLogs/CHANGELOG.md

@@ -15,6 +15,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 
 ## tip
 
+* FEATURE: optimize [multi-exact queries](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter) with many phrases to search. For example, `ip:in(path:="/foo/bar" | keep ip)` when there are many unique values for `ip` field among log entries with `/foo/bar` path.
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add support for displaying the top 5 log streams in the hits graph. The remaining log streams are grouped into an "other" label. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6545).
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the ability to customize the graph display with options for bar, line, stepped line, and points.
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add fields for setting AccountID and ProjectID. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6631).
@@ -24,9 +25,12 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): move the Markdown toggle to the general settings panel in the upper left corner.
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add search functionality to the column display settings in the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668).
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add the ability to select all columns in the column display settings of the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6680).
+* FEATURE: Allow to define ingestion parameters via headers. Supported headers - `VL-Msg-Field`,`VL-Stream-Fields`,`VL-Ignore-Fields`,`VL-Time-Field`, `VL-Debug`. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6443) for details.
+* FEATURE: [vlinsert](https://docs.victoriametrics.com/victorialogs/): added OpenTelemetry logs ingestion support. See this [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6218) for details.
 
-* BUGFIX: properly handle Logstash requests for Elasticsearch configuration when using `outputs.elasticsearch` in Logstash pipelines. Previously, the requests could be rejected with `400 Bad Request` response.
+* BUGFIX: properly handle Logstash requests for Elasticsearch configuration when using `outputs.elasticsearch` in Logstash pipelines. Previously, the requests could be rejected with `400 Bad Request` response. Updates [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4750).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix `not found index.js` error when loading vmui in VictoriaLogs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6764). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6770).
+* BUGFIX: properly execute queries with `OR` [filters](https://docs.victoriametrics.com/victorialogs/logsql/#logical-filter) for distinct [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). For example, `field1:foo OR field2:bar`. Previously logs matching these filters may be skipped during querying. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6554) for details. Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6556).
 
 ## [v0.28.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.28.0-victorialogs)
 
@@ -241,10 +245,10 @@ Released at 2024-05-24
 
 * FEATURE: return the number of matching log entries per returned value in [HTTP API](https://docs.victoriametrics.com/victorialogs/querying/#http-api) results. This simplifies detecting [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) / [stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) values with the biggest number of logs for the given [LogsQL query](https://docs.victoriametrics.com/victorialogs/logsql/).
 * FEATURE: improve performance for [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter) in the following cases:
-  - If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`.
-  - If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`.
-  - If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`.
-  - If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`.
+  * If the regexp contains just a phrase without special regular expression chars. For example, `~"foo"`.
+  * If the regexp starts with `.*` or ends with `.*`. For example, `~".*foo.*"`.
+  * If the regexp contains multiple strings delimited by `|`. For example, `~"foo|bar|baz"`.
+  * If the regexp contains multiple [words](https://docs.victoriametrics.com/victorialogs/logsql/#word). For example, `~"foo bar baz"`.
 * FEATURE: allow disabling automatic unquoting of the matched placeholders in [`extract` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#extract-pipe). See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#format-for-extract-pipe-pattern).
 
 * BUGFIX: properly parse `!` in front of [exact filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter), [exact-prefix filter](https://docs.victoriametrics.com/victorialogs/logsql/#exact-prefix-filter) and [regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter). For example, `!~"some regexp"` is properly parsed as `not ="some regexp"`. Previously it was incorrectly parsed as `'~="some regexp"'` [phrase filter](https://docs.victoriametrics.com/victorialogs/logsql/#phrase-filter).
@@ -304,7 +308,6 @@ Released at 2024-05-15
 * FEATURE: add ability to return the first `N` results from [`sort` pipe](#https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe). This is useful when `N` biggest or `N` smallest values must be returned from large amounts of logs.
 * FEATURE: add [`quantile`](https://docs.victoriametrics.com/victorialogs/logsql/#quantile-stats) and [`median`](https://docs.victoriametrics.com/victorialogs/logsql/#median-stats) [stats functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe).
 
-
 ## [v0.6.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.1-victorialogs)
 
 Released at 2024-05-14
@@ -314,7 +317,6 @@ Released at 2024-05-14
 * BUGFIX: properly return matching logs in [streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) with small number of entries. Previously they could be skipped. The issue has been introduced in [the release v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs).
 * BUGFIX: fix `runtime error: index out of range` panic when using [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe) like `_time:1h | sort by (_time)`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6258).
 
-
 ## [v0.6.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.6.0-victorialogs)
 
 Released at 2024-05-12
@@ -342,7 +344,6 @@ Released at 2024-04-11
 
 * BUGFIX: properly register new [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) under high data ingestion rate. The issue has been introduced in [v0.5.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.0-victorialogs).
 
-
 ## [v0.5.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.5.1-victorialogs)
 
 Released at 2024-04-04

docs/VictoriaLogs/data-ingestion/Fluentbit.md

@@ -14,6 +14,51 @@ aliases:
 
 # Fluentbit setup
 
+VictoriaLogs supports given below Fluentbit outputs:
+- [Elasticsearch](#elasticsearch)
+- [Loki](#loki)
+- [HTTP JSON](#http)
+
+## Elasticsearch
+
+Specify [elasticsearch output](https://docs.fluentbit.io/manual/pipeline/outputs/elasticsearch) section in the `fluentbit.conf`
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```conf
+[Output]
+    Name es
+    Match *
+    host victorialogs
+    port 9428
+    compress gzip
+    path /insert/elasticsearch
+    header AccountID 0
+    header ProjectID 0
+    header VL-Stream-Fields path
+    header VL-Msg-Field log
+    header VL-Time-Field @timestamp
+```
+
+## Loki
+
+Specify [loki output](https://docs.fluentbit.io/manual/pipeline/outputs/loki) section in the `fluentbit.conf`
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```conf
+[OUTPUT]
+    name       loki
+    match      *
+    host       victorialogs
+    uri        /insert/loki/api/v1/push
+    port       9428
+    label_keys $path,$log,$time
+    header     VL-Msg-Field log
+    header     VL-Time-Field time
+    header     VL-Stream-Fields path
+```
+
+## HTTP
+
 Specify [http output](https://docs.fluentbit.io/manual/pipeline/outputs/http) section in the `fluentbit.conf`
 for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
 

docs/VictoriaLogs/data-ingestion/Logstash.md

@@ -11,6 +11,13 @@ aliases:
   - /victorialogs/data-ingestion/logstash.html
   - /victorialogs/data-ingestion/Logstash.html
 ---
+VictoriaLogs supports given below Logstash outputs:
+- [Elasticsearch](#elasticsearch)
+- [Loki](#loki)
+- [HTTP JSON](#http)
+
+## Elasticsearch
+
 Specify [`output.elasticsearch`](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) section in the `logstash.conf` file
 for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
 
@@ -106,6 +113,32 @@ output {
 }
 ```
 
+## Loki
+
+Specify [`output.loki`](https://grafana.com/docs/loki/latest/send-data/logstash/) section in the `logstash.conf` file
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```conf
+output {
+  loki {
+     url => "http://victorialogs:9428/insert/loki/api/v1/push?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp"
+  }
+}
+```
+
+## HTTP
+
+Specify [`output.http`](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-http.html) section in the `logstash.conf` file
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```conf
+output {
+  url => "http://victorialogs:9428/insert/jsonline?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp"
+  format => "json"
+  http_method => "post"
+}
+```
+
 See also:
 
 - [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting).

docs/VictoriaLogs/data-ingestion/README.md

@@ -6,6 +6,8 @@
 - Logstash - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/logstash/).
 - Vector - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/).
 - Promtail (aka Grafana Loki) - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/).
+- Telegraf - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/telegraf/).
+- OpenTelemetry Collector - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/opentelemetry/).
 
 The ingested logs can be queried according to [these docs](https://docs.victoriametrics.com/victorialogs/querying/).
 
@@ -14,7 +16,6 @@ See also:
 - [Log collectors and data ingestion formats](#log-collectors-and-data-ingestion-formats).
 - [Data ingestion troubleshooting](#troubleshooting).
 
-
 ## HTTP APIs
 
 VictoriaLogs supports the following data ingestion HTTP APIs:
@@ -22,6 +23,7 @@ VictoriaLogs supports the following data ingestion HTTP APIs:
 - Elasticsearch bulk API. See [these docs](#elasticsearch-bulk-api).
 - JSON stream API aka [ndjson](https://jsonlines.org/). See [these docs](#json-stream-api).
 - Loki JSON API. See [these docs](#loki-json-api).
+- OpenTelemetry API. See [these docs](#opentelemetry-api).
 
 VictoriaLogs accepts optional [HTTP parameters](#http-parameters) at data ingestion HTTP APIs.
 
@@ -168,7 +170,12 @@ See also:
 
 ### HTTP parameters
 
-VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#http-apis):
+ VictoriaLogs accepts the following configuration parameters via [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) or URL [Query string](https://en.wikipedia.org/wiki/Query_string) at [data ingestion HTTP APIs](#http-apis).
+First defined parameter is used. [Query string](https://en.wikipedia.org/wiki/Query_string) parameters have priority over HTTP Headers.
+
+#### HTTP Query string parameters
+
+ List of supported [Query string](https://en.wikipedia.org/wiki/Query_string) parameters:
 
 - `_msg_field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
   with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper.
@@ -193,10 +200,37 @@ VictoriaLogs accepts the following parameters at [data ingestion HTTP APIs](#htt
 
 See also [HTTP headers](#http-headers).
 
-### HTTP headers
+#### HTTP headers
 
+ List of supported [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields) parameters:
+
+- `AccountID` - may contain the needed accountID of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
+
+- `ProjectID`- may contain the projectID needed of tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
 VictoriaLogs accepts optional `AccountID` and `ProjectID` headers at [data ingestion HTTP APIs](#http-apis).
-These headers may contain the needed tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victorialogs/#multitenancy) for details.
+
+- `VL-Msg-Field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
+  with the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) generated by the log shipper.
+  This is usually the `message` field for Filebeat and Logstash.
+  If the `VL-Msg-Field` header isn't set, then VictoriaLogs reads the log message from the `_msg` field.
+
+- `VL-Time-Field` - it must contain the name of the [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
+  with the [log timestamp](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) generated by the log shipper.
+  This is usually the `@timestamp` field for Filebeat and Logstash.
+  If the `VL-Time-Field` header isn't set, then VictoriaLogs reads the timestamp from the `_time` field.
+  If this field doesn't exist, then the current timestamp is used.
+
+- `VL-Stream-Fields` - it should contain comma-separated list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names,
+  which uniquely identify every [log stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) collected the log shipper.
+  If the `VL-Stream-Fields` header isn't set, then all the ingested logs are written to default log stream - `{}`.
+
+- `VL-Ignore-Fields` - this parameter may contain the list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names,
+  which must be ignored during data ingestion.
+
+- `VL-Debug` - if this parameter is set to `1`, then the ingested logs aren't stored in VictoriaLogs. Instead,
+  the ingested data is logged by VictoriaLogs, so it can be investigated later.
+
+See also [HTTP Query string parameters](#http-query-string-parameters).
 
 ## Troubleshooting
 
@@ -241,12 +275,14 @@ VictoriaLogs exposes various [metrics](https://docs.victoriametrics.com/victoria
 
 Here is the list of log collectors and their ingestion formats supported by VictoriaLogs:
 
-| How to setup the collector | Format: Elasticsearch | Format: JSON Stream | Format: Loki | Format: syslog |
-|----------------------------|-----------------------|---------------------|--------------|----------------|
-| [Rsyslog](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/) | [Yes](https://www.rsyslog.com/doc/configuration/modules/omelasticsearch.html) | No | No | [Yes](https://www.rsyslog.com/doc/configuration/modules/omfwd.html) |
-| [Syslog-ng](https://docs.victoriametrics.com/victorialogs/data-ingestion/filebeat/) | Yes, [v1](https://support.oneidentity.com/technical-documents/syslog-ng-open-source-edition/3.16/administration-guide/28#TOPIC-956489), [v2](https://support.oneidentity.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/29#TOPIC-956494) | No | No | [Yes](https://support.oneidentity.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/44#TOPIC-956553) |
-| [Filebeat](https://docs.victoriametrics.com/victorialogs/data-ingestion/filebeat/) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No | No | No |
-| [Fluentbit](https://docs.victoriametrics.com/victorialogs/data-ingestion/fluentbit/) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/loki) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/syslog) |
-| [Logstash](https://docs.victoriametrics.com/victorialogs/data-ingestion/logstash/)   | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | No | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-syslog.html) |
-| [Vector](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/http/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/loki/) | No |
-| [Promtail](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/)   | No | No | [Yes](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#clients) | No |
+| How to setup the collector | Format: Elasticsearch | Format: JSON Stream | Format: Loki | Format: syslog | Format: OpenTelemetry |
+|----------------------------|-----------------------|---------------------|--------------|----------------|-----------------------|
+| [Rsyslog](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/) | [Yes](https://www.rsyslog.com/doc/configuration/modules/omelasticsearch.html) | No | No | [Yes](https://www.rsyslog.com/doc/configuration/modules/omfwd.html) | No |
+| [Syslog-ng](https://docs.victoriametrics.com/victorialogs/data-ingestion/filebeat/) | Yes, [v1](https://support.oneidentity.com/technical-documents/syslog-ng-open-source-edition/3.16/administration-guide/28#TOPIC-956489), [v2](https://support.oneidentity.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/29#TOPIC-956494) | No | No | [Yes](https://support.oneidentity.com/technical-documents/doc/syslog-ng-open-source-edition/3.16/administration-guide/44#TOPIC-956553) | No |
+| [Filebeat](https://docs.victoriametrics.com/victorialogs/data-ingestion/filebeat/) | [Yes](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) | No | No | No | No |
+| [Fluentbit](https://docs.victoriametrics.com/victorialogs/data-ingestion/fluentbit/) | No | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/http) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/loki) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/syslog) | [Yes](https://docs.fluentbit.io/manual/pipeline/outputs/opentelemetry) |
+| [Logstash](https://docs.victoriametrics.com/victorialogs/data-ingestion/logstash/)   | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) | No | No | [Yes](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-syslog.html) | [Yes](https://github.com/paulgrav/logstash-output-opentelemetry) |
+| [Vector](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/http/) | [Yes](https://vector.dev/docs/reference/configuration/sinks/loki/) | No | [Yes](https://vector.dev/docs/reference/configuration/sources/opentelemetry/) |
+| [Promtail](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/)   | No | No | [Yes](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#clients) | No | No |
+| [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) | [Yes](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter) | No | [Yes](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/lokiexporter) | [Yes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/syslogexporter) | [Yes](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter) |
+| [Telegraf](https://docs.victoriametrics.com/victorialogs/data-ingestion/telegraf/) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/http) | [Yes](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/loki) | [Yes](https://github.com/influxdata/telegraf/blob/master/plugins/outputs/syslog) | Yes |

docs/VictoriaLogs/data-ingestion/Telegraf.md

@@ -0,0 +1,123 @@
+---
+weight: 5
+title: Telegraf setup
+disableToc: true
+menu:
+  docs:
+    parent: "victorialogs-data-ingestion"
+    weight: 5
+aliases:
+  - /VictoriaLogs/data-ingestion/Telegraf.html
+---
+# Telegraf setup
+
+VictoriaLogs supports given below Telegraf outputs:
+- [Elasticsearch](#elasticsearch)
+- [Loki](#loki)
+- [HTTP JSON](#http)
+
+## Elasticsearch
+
+Specify [Elasticsearch output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch) in the `telegraf.toml`
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```toml
+[[outputs.elasticsearch]]
+  urls = ["http://localhost:9428/insert/elasticsearch"]
+  timeout = "1m"
+  flush_interval = "30s"
+  enable_sniffer = false
+  health_check_interval = "0s"
+  index_name = "device_log-%Y.%m.%d"
+  manage_template = false
+  template_name = "telegraf"
+  overwrite_template = false
+  namepass = ["tail"]
+  [outputs.elasticsearch.headers]
+    "VL-Msg-Field" = "tail.value"
+    "VL-Time-Field" = "@timestamp"
+    "VL-Stream-Fields" = "tag.log_source,tag.metric_type"
+
+[[inputs.tail]]
+  files = ["/tmp/telegraf.log"]
+  from_beginning = false
+  interval = "10s"
+  pipe = false
+  watch_method = "inotify"
+  data_format = "value"
+  data_type = "string"
+  character_encoding = "utf-8"
+  [inputs.tail.tags]
+     metric_type = "logs"
+     log_source = "telegraf"
+```
+
+
+## Loki
+
+Specify [Loki output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/loki) in the `telegraf.toml`
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```toml
+[[outputs.loki]]
+  domain = "http://localhost:9428"
+  endpoint = "/insert/loki/api/v1/push&_msg_field=tail.value&_time_field=@timefield&_stream_fields=log_source,metric_type"
+  namepass = ["tail"]
+  gzip_request = true
+  sanitize_label_names = true
+
+[[inputs.tail]]
+  files = ["/tmp/telegraf.log"]
+  from_beginning = false
+  interval = "10s"
+  pipe = false
+  watch_method = "inotify"
+  data_format = "value"
+  data_type = "string"
+  character_encoding = "utf-8"
+  [inputs.tail.tags]
+     metric_type = "logs"
+     log_source = "telegraf"
+```
+
+
+## HTTP
+
+Specify [HTTP output](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/http) in the `telegraf.toml with batch mode disabled`
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```toml
+[[inputs.tail]]
+  files = ["/tmp/telegraf.log"]
+  from_beginning = false
+  interval = "10s"
+  pipe = false
+  watch_method = "inotify"
+  data_format = "value"
+  data_type = "string"
+  character_encoding = "utf-8"
+  [inputs.tail.tags]
+     metric_type = "logs"
+     log_source = "telegraf"
+
+[[outputs.http]]
+  url = "http://localhost:9428/insert/jsonline?_msg_field=fields.message&_time_field=timestamp,_stream_fields=tags.log_source,tags.metric_type"
+  data_format = "json"
+  namepass = ["docker_log"]
+  use_batch_format = false
+```
+
+Substitute the `localhost:9428` address inside `endpoints` section with the real TCP address of VictoriaLogs.
+
+See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-headers) for details on headers specified
+in the `[[output.elasticsearch]]` section.
+
+It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
+and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
+
+See also:
+
+- [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting).
+- [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/).
+- [Elasticsearch output docs for Telegraf](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch).
+- [Docker-compose demo for Telegraf integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf-docker).

docs/VictoriaLogs/data-ingestion/Vector.md

@@ -11,24 +11,52 @@ aliases:
   - /victorialogs/data-ingestion/Vector.html
   - /victorialogs/data-ingestion/vector.html
 ---
-## Elasticsearch sink 
+VictoriaLogs supports given below Vector sinks:
+- [Elasticsearch](#elasticsearch)
+- [Loki](#loki)
+- [HTTP JSON](#http)
 
-Specify [Elasticsearch sink type](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) in the `vector.toml`
+## Elasticsearch
+
+Specify [Elasticsearch sink type](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) in the `vector.yaml`
 for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
 
-```toml
-[sinks.vlogs]
-  inputs = [ "your_input" ]
-  type = "elasticsearch"
-  endpoints = [ "http://localhost:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  healthcheck.enabled = false
+```yaml
+sinks:
+  vlogs:
+    inputs:
+      - your_input
+    type: elasticsearch
+    endpoints:
+      - http://localhost:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    healthcheck:
+      enabled: false
+    query:
+      _msg_field: message
+      _time_field: timestamp
+      _stream_fields: host,container_name
+```
 
-  [sinks.vlogs.query]
-    _msg_field = "message"
-    _time_field = "timestamp"
-    _stream_fields = "host,container_name"
+## Loki
+
+Specify [Loki sink type](https://vector.dev/docs/reference/configuration/sinks/loki/) in the `vector.yaml`
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
+
+```yaml
+sinks:
+  vlogs:
+    type: "loki"
+    endpoint = "http://localhost:9428/insert/loki/"
+    inputs:
+      - your_input
+    compression: gzip
+    path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source
+    encoding:
+      codec: json
+    labels:
+      source: vector
 ```
 
 Substitute the `localhost:9428` address inside `endpoints` section with the real TCP address of VictoriaLogs.
@@ -36,129 +64,148 @@ Substitute the `localhost:9428` address inside `endpoints` section with the real
 Replace `your_input` with the name of the `inputs` section, which collects logs. See [these docs](https://vector.dev/docs/reference/configuration/sources/) for details.
 
 See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on parameters specified
-in the `[sinks.vlogs.query]` section.
+in the `sinks.vlogs.query` section.
 
 It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
 and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
 This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
-in the `[sinks.vlogs.query]` section and inspecting VictoriaLogs logs then:
+in the `sinks.vlogs.query` section and inspecting VictoriaLogs logs then:
 
-```toml
-[sinks.vlogs]
-  inputs = [ "your_input" ]
-  type = "elasticsearch"
-  endpoints = [ "http://localhost:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  healthcheck.enabled = false
-
-  [sinks.vlogs.query]
-    _msg_field = "message"
-    _time_field = "timestamp"
-    _stream_fields = "host,container_name"
-    debug = "1"
+```yaml
+sinks:
+  vlogs:
+    inputs:
+      - your_input
+    type: elasticsearch
+    endpoints:
+      - http://localhost:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    healthcheck:
+      enabled: false
+    query:
+      _msg_field: message
+      _time_field: timestamp
+      _stream_fields: host,container_name
+      debug: "1"
 ```
 
 If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) must be skipped
 during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
 For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
 
-```toml
-[sinks.vlogs]
-  inputs = [ "your_input" ]
-  type = "elasticsearch"
-  endpoints = [ "http://localhost:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  healthcheck.enabled = false
-
-  [sinks.vlogs.query]
-    _msg_field = "message"
-    _time_field = "timestamp"
-    _stream_fields = "host,container_name"
-    ignore_fields = "log.offset,event.original"
+```yaml
+sinks:
+  vlogs:
+    inputs:
+      - your_input
+    type: elasticsearch
+    endpoints:
+      - http://localhost:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    healthcheck:
+      enabled: false
+    query:
+      _msg_field: message
+      _time_field: timestamp
+      _stream_fields: host,container_name
+      _ignore_fields: log.offset,event.original
 ```
 
 When Vector ingests logs into VictoriaLogs at a high rate, then it may be needed to tune `batch.max_events` option.
 For example, the following config is optimized for higher than usual ingestion rate:
 
-```toml
-[sinks.vlogs]
-  inputs = [ "your_input" ]
-  type = "elasticsearch"
-  endpoints = [ "http://localhost:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  healthcheck.enabled = false
-
-  [sinks.vlogs.query]
-    _msg_field = "message"
-    _time_field = "timestamp"
-    _stream_fields = "host,container_name"
-
-  [sinks.vlogs.batch]
-    max_events = 1000
+```yaml
+sinks:
+  vlogs:
+    inputs:
+      - your_input
+    type: elasticsearch
+    endpoints:
+      - http://localhost:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    healthcheck:
+      enabled: false
+    query: 
+      _msg_field: message
+      _time_field: timestamp
+      _stream_fields: host,container_name
+    batch]
+      max_events: 1000
 ```
 
 If the Vector sends logs to VictoriaLogs in another datacenter, then it may be useful enabling data compression via `compression = "gzip"` option.
 This usually allows saving network bandwidth and costs by up to 5 times:
 
-```toml
-[sinks.vlogs]
-  inputs = [ "your_input" ]
-  type = "elasticsearch"
-  endpoints = [ "http://localhost:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  healthcheck.enabled = false
-  compression = "gzip"
-
-  [sinks.vlogs.query]
-    _msg_field = "message"
-    _time_field = "timestamp"
-    _stream_fields = "host,container_name"
+```yaml
+sinks:
+  vlogs:
+    inputs:
+      - your_input
+    type: elasticsearch
+    endpoints:
+      - http://localhost:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    healthcheck:
+      enabled: false
+    compression: gzip
+    query:
+      _msg_field: message
+      _time_field: timestamp
+      _stream_fields: host,container_name
 ```
 
 By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/keyconcepts/#multitenancy).
-If you need storing logs in other tenant, then specify the needed tenant via `[sinks.vlogs.request.headers]` section.
-For example, the following `vector.toml` config instructs Vector to store the data to `(AccountID=12, ProjectID=34)` tenant:
+If you need storing logs in other tenant, then specify the needed tenant via `sinks.vlogs.request.headers` section.
+For example, the following `vector.yaml` config instructs Vector to store the data to `(AccountID=12, ProjectID=34)` tenant:
 
-```toml
-[sinks.vlogs]
-  inputs = [ "your_input" ]
-  type = "elasticsearch"
-  endpoints = [ "http://localhost:9428/insert/elasticsearch/" ]
-  mode = "bulk"
-  api_version = "v8"
-  healthcheck.enabled = false
-
-  [sinks.vlogs.query]
-    _msg_field = "message"
-    _time_field = "timestamp"
-    _stream_fields = "host,container_name"
-
-  [sinks.vlogs.request.headers]
-    AccountID = "12"
-    ProjectID = "34"
+```yaml
+sinks:
+  vlogs:
+    inputs:
+      - your_input
+    type: elasticsearch
+    endpoints:
+      - http://localhost:9428/insert/elasticsearch/
+    mode: bulk
+    api_version: v8
+    healthcheck:
+      enabled: false
+    query:
+      _msg_field: message
+      _time_field: timestamp
+      _stream_fields: host,container_name
+    request:
+      headers:
+        AccountID: "12"
+        ProjectID: "34"
 ```
 
-## HTTP sink
+## HTTP
 
 Vector can be configured with [HTTP](https://vector.dev/docs/reference/configuration/sinks/http/) sink type 
 for sending data to [JSON stream API](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api):
 
-```toml
-[sinks.vlogs]
-  inputs = [ "your_input" ]
-  type = "http"
-  uri = "http://localhost:9428/insert/jsonline?_stream_fields=host,container_name&_msg_field=message&_time_field=timestamp"
-  encoding.codec = "json"
-  framing.method = "newline_delimited"
-  healthcheck.enabled = false
-
-  [sinks.vlogs.request.headers]
-    AccountID = "12"
-    ProjectID = "34"
+```yaml
+sinks:
+  vlogs:
+    inputs:
+      - your_input
+    type: http
+    uri: http://localhost:9428/insert/jsonline?_stream_fields=host,container_name&_msg_field=message&_time_field=timestamp
+    encoding:
+      codec: json
+    framing:
+      method: newline_delimited
+    healthcheck:
+      enabled: false
+    request:
+      headers:
+        AccountID: "12"
+        ProjectID: "34"
 ```
 
 See also:

docs/VictoriaLogs/data-ingestion/opentelemetry.md

@@ -0,0 +1,120 @@
+---
+weight: 4
+title: OpenTelemetry setup
+disableToc: true
+menu:
+  docs:
+    parent: "victorialogs-data-ingestion"
+    weight: 4
+aliases:
+  - /VictoriaLogs/data-ingestion/OpenTelemetry.html
+---
+
+
+VictoriaLogs supports both client open-telemetry [SDK](https://opentelemetry.io/docs/languages/) and [collector](https://opentelemetry.io/docs/collector/).
+
+## Client SDK
+
+ Specify `EndpointURL`  for http-exporter builder.
+
+Consider the following example for `golang` `SDK`:
+
+```go
+ // Create the OTLP log exporter that sends logs to configured destination
+ logExporter, err := otlploghttp.New(ctx,
+  otlploghttp.WithEndpointURL("http://victorialogs:9428/insert/opentelemetry/v1/logs"),
+ )
+```
+
+ Optionally, [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) could be defined via headers:
+
+```go
+ // Create the OTLP log exporter that sends logs to configured destination
+ logExporter, err := otlploghttp.New(ctx,
+  otlploghttp.WithEndpointURL("http://victorialogs:9428/insert/opentelemetry/v1/logs"),
+   otlploghttp.WithHeaders(map[string]string{"VL-Stream-Fields": "telemetry.sdk.language,severity"}),
+ )
+
+```
+
+ Given config defines 2 stream fields - `severity` and `telemetry.sdk.language`.
+
+See also [HTTP headers](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-headers)
+
+## Collector configuration
+
+VictoriaLogs supports given below OpenTelemetry collector exporters:
+
+* [Elasticsearch](#elasticsearch)
+* [Loki](#loki)
+* [OpenTelemetry](#opentelemetry)
+
+### Elasticsearch
+
+```yaml
+exporters:
+  elasticsearch:
+    endpoints:
+      - http://victorialogs:9428/insert/elasticsearch
+receivers:
+  filelog:
+    include: [/tmp/logs/*.log]
+    resource:
+      region: us-east-1
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [elasticsearch]
+```
+
+### Loki
+
+```yaml
+exporters:
+  loki:
+    endpoint: http://victorialogs:9428/insert/loki/api/v1/push
+receivers:
+  filelog:
+    include: [/tmp/logs/*.log]
+    resource:
+      region: us-east-1
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      exporters: [loki]
+```
+
+### OpenTelemetry
+
+Specify logs endpoint for [OTLP/HTTP exporter](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/otlphttpexporter/README.md) in configuration file
+for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/):
+
+```yaml
+exporters:
+  otlphttp:
+    logs_endpoint: http://localhost:9428/insert/opentelemetry/v1/logs
+```
+
+ Optionally, [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) could be defined via headers:
+
+```yaml
+exporters:
+  otlphttp:
+    logs_endpoint: http://localhost:9428/insert/opentelemetry/v1/logs
+    headers:
+     VL-Stream-Fields: telemetry.sdk.language,severity
+```
+
+See also [HTTP headers](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-headers)
+
+Substitute `localhost:9428` address inside `exporters.oltphttp.logs_endpoint` with the real address of VictoriaLogs.
+
+The ingested log entries can be queried according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/querying/).
+
+See also:
+
+* [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting).
+* [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/).
+* [Docker-compose demo for OpenTelemetry collector integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/opentelemetry-collector).

docs/VictoriaLogs/victorialogs-datasource.md

@@ -20,7 +20,6 @@ The VictoriaLogs datasource plugin allows you to query and visualize
 * [Installation](#installation)
 * [How to make new release](#how-to-make-new-release)
 * [Notes](#notes)
-* [Frequently Asked Questions](#faq)
 * [License](#license)
 
 ## Installation
@@ -96,7 +95,7 @@ docker-compose -f docker-compose.yaml up
 
 After Grafana starts successfully, datasource should be available in the datasources tab
 
-![Configuration](./provision_datasources.webp)
+<img src="provision_datasources.webp" width="800" alt="Configuration">
 
 ### Install in Kubernetes
 
@@ -136,7 +135,7 @@ extraInitContainers:
      - |
        set -ex
        mkdir -p /var/lib/grafana/plugins/
-       ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
+       ver=$(curl -s -L https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
        curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
        tar -xf /var/lib/grafana/plugins/vl-plugin.tar.gz -C /var/lib/grafana/plugins/
        rm /var/lib/grafana/plugins/vl-plugin.tar.gz
@@ -309,12 +308,6 @@ In the `plugin.json` file of our plugin, the `metrics` field is set to `true`. T
 
 For more information on the fields in `plugin.json`, please refer to the [Grafana documentation](https://grafana.com/developers/plugin-tools/reference-plugin-json#properties).
 
-## FAQ
-
-### Which version of Grafana is required in order to use VictoriaLogs datasource?
-
-[10.0.3](https://grafana.com/grafana/download/10.0.3) or newer.
-
 ## License
 
 This project is licensed under

docs/changelog/CHANGELOG.md

@@ -18,6 +18,18 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 ## tip
 
+**Update note 1: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): perform deduplication for all received data when specifying `-streamAggr.dedupInterval` or `-remoteWrite.streamAggr.dedupInterval` command-line flag. Previously, if the `-remoteWrite.streamAggr.config` or `-streamAggr.config` is set, only series that matched aggregation config were deduplicated. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6711#issuecomment-2288361213) for details.**
+
+* FEATURE [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): perform deduplication for all received data when specifying `-streamAggr.dedupInterval` or `-remoteWrite.streamAggr.dedupInterval` command-line flags are set. Previously, if the `-remoteWrite.streamAggr.config` or `-streamAggr.config` is set, only series that matched aggregation config were deduplicated. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6711#issuecomment-2288361213) for details.
+* FEATURE: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) components: add support of hot-reload for license key supplied by `-licenseFile` command-line flag.
+
+*
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink` in `Host:Port` format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6912).
+* BUGFIX: [vmagent dashboard](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/vmagent.json): fix legend captions for stream aggregation related panels. Before they were displaying wrong label names.
+* BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway/): add missing `datadog`, `newrelic`, `opentelemetry` and `pushgateway` routes to the `JWT` authorization routes. Allows prefixed (`promtheus/graphite`) routes for query requests.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric `vm_object_references{type="indexdb"}`. Previously, it was overcounted.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ingest stale NaN samples. Previously it could be dropped if series didn't exist at storage node. See this issue [https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5069] for details.
+
 ## [v1.103.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.103.0)
 
 Released at 2024-08-28
@@ -53,7 +65,32 @@ Released at 2024-08-28
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly register index metrics for previous index part. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6868) for details.
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Removes the fallback to global index search when the search using per-day index fails due to too many time series found (the global index will fail anyway with the same error and so the fallback is not needed and only slows down the search). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6836) for details.
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric names registering in the per-day index for new dates for existing time series when making calls to `/tags/tagSeries` and `/tags/tagMultiSeries` handlers of [Grpahite API](https://docs.victoriametrics.com/#graphite-api-usage). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6872/) for details.
-BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ignore deleted metrics when applying [retention filters](https://docs.victoriametrics.com/#retention-filters) and [downsampling](https://docs.victoriametrics.com/#downsampling). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6891) issue for the details.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ignore deleted metrics when applying [retention filters](https://docs.victoriametrics.com/#retention-filters) and [downsampling](https://docs.victoriametrics.com/#downsampling). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6891) issue for the details.
+
+## [v1.102.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.2)
+
+Released at 2024-08-28
+
+**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent): fixes `proxy_url` authorization for scrape targets. Previously proxy authorization configuration was ignored for `https` targets. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6771) issue for details.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6784).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): respect HTTP headers defined in [notifier configuration file](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file) for each request to notifiers. Previously, this param was ignored by mistake.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): correctly apply `-streamAggr.dropInputLabels` when global stream deduplication is enabled without `-streamAggr.config`. Previously, `-remoteWrite.streamAggr.dropInputLabels` was used instead.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): fix command-line flag `-remoteWrite.streamAggr.ignoreFirstIntervals` to accept multiple values and be applied per each corresponding `-remoteWrite.url`. Previously, this flag only could have been used globally for all URLs.
+* BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): respect `-search.denyPartialResponse` cmd-line flag and `deny_partial_response` query GET param when serving requests via Graphite API. Before, partial responses were always denied. Thanks to @penguinlav for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6748).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): account for `-usePromCompatibleNaming` cmd-line flag during when pushing data to remote storages. Thanks to @12345XXX  for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6776).
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): reduce CPU usage by limiting the number of concurrently running inserts. The issue was introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/498fe1cfa523be5bfecaa372293c3cded85e75ab) starting from v1.101.0. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6733) issue for details.
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): fix calculation [histogram_quantile](https://docs.victoriametrics.com/metricsql/#histogram_quantile) over Prometheus buckets with inconsistent values. It was producing incorrect results in case lower buckets. The issue was introduced in [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release, see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6714) for the details.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/), [vmctl](https://docs.victoriametrics.com/vmctl/) and snapshot API: verify correctness of URLs provided via cmd-line flags before executing HTTP requests. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6740) issue for details.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): reduce memory usage when parsing responses with big number of metrics in response. The memory usage was increased in [v1.102.0-rc1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0-rc1) after attempt to reduce CPU usage for heavy loaded vmalerts.
+* BUGFIX: all VictoriaMetrics components: forcefully set owner/group for release tars to 1000:1000. This helps to avoid unpacking [issues](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6788) on systems with limitations around UID:GID configuration. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6846).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly register index metrics for previous index part. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6868) for details.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): Removes the fallback to global index search when the search using per-day index fails due to too many time series found (the global index will fail anyway with the same error and so the fallback is not needed and only slows down the search). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6836) for details.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric names registering in the per-day index for new dates for existing time series when making calls to `/tags/tagSeries` and `/tags/tagMultiSeries` handlers of [Grpahite API](https://docs.victoriametrics.com/#graphite-api-usage). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6872/) for details.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ignore deleted metrics when applying [retention filters](https://docs.victoriametrics.com/#retention-filters) and [downsampling](https://docs.victoriametrics.com/#downsampling). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6891) issue for the details.
 
 ## [v1.102.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.1)
 
@@ -389,6 +426,25 @@ Released at 2024-02-14
 * BUGFIX: [dashboards](https://grafana.com/orgs/victoriametrics): update `Storage full ETA` panels for Single-node and Cluster dashboards to prevent them from showing negative or blank results caused by increase of deduplicated samples. Deduplicated samples were part of the expression to provide a better estimate for disk usage, but due to sporadic nature of [deduplication](https://docs.victoriametrics.com/#deduplication) in VictoriaMetrics it rather produced skewed results. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5747).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/#vmalert): reduce memory usage for ENT version of vmalert for configurations with high number of groups with enabled multitenancy.
 
+## [v1.97.7](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.7)
+
+Released at 2024-08-28
+
+**v1.97.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.97.x line will be supported for at least 12 months since [v1.97.0](https://docs.victoriametrics.com/CHANGELOG.html#v1970) release**
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent): fixes `proxy_url` authorization for scrape targets. Previously proxy authorization configuration was ignored for `https` targets. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6771) issue for details.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) fix service discovery of Azure Virtual Machines for response contains `nextLink`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6784).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): respect HTTP headers defined in [notifier configuration file](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file) for each request to notifiers. Previously, this param was ignored by mistake.
+* BUGFIX: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): respect `-search.denyPartialResponse` cmd-line flag and `deny_partial_response` query GET param when serving requests via Graphite API. Before, partial responses were always denied. Thanks to @penguinlav for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6748).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): account for `-usePromCompatibleNaming` cmd-line flag during when pushing data to remote storages. Thanks to @12345XXX  for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6776).
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): fix calculation [histogram_quantile](https://docs.victoriametrics.com/metricsql/#histogram_quantile) over Prometheus buckets with inconsistent values. It was producing incorrect results in case lower buckets. The issue was introduced in [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release, see [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6714) for the details.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/), [vmctl](https://docs.victoriametrics.com/vmctl/) and snapshot API: verify correctness of URLs provided via cmd-line flags before executing HTTP requests. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6740) issue for details.
+* BUGFIX: all VictoriaMetrics components: forcefully set owner/group for release tars to 1000:1000. This helps to avoid unpacking [issues](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6788) on systems with limitations around UID:GID configuration. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6846).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly register index metrics for previous index part. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6868) for details.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric names registering in the per-day index for new dates for existing time series when making calls to `/tags/tagSeries` and `/tags/tagMultiSeries` handlers of [Grpahite API](https://docs.victoriametrics.com/#graphite-api-usage). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6872/) for details.
+
 ## [v1.97.6](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.6)
 
 Released at 2024-07-17

docs/changelog/CHANGELOG_2020.md

@@ -8,6 +8,7 @@ menu:
     weight: 6
 aliases:
 - /CHANGELOG_2020.html
+- /changelog_2020
 ---
 ## [v1.51.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.51.0)
 

docs/changelog/CHANGELOG_2021.md

@@ -8,6 +8,7 @@ menu:
     weight: 5
 aliases:
 - /CHANGELOG_2021.html
+- /changelog_2021
 ---
 ## [v1.71.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.71.0)
 

docs/changelog/CHANGELOG_2022.md

@@ -8,6 +8,7 @@ menu:
     weight: 4
 aliases:
 - /CHANGELOG_2022.html
+- /changelog_2022
 ---
 ## [v1.85.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.3)
 

docs/changelog/CHANGELOG_2023.md

@@ -8,6 +8,7 @@ menu:
     weight: 3
 aliases:
 - /CHANGELOG_2023.html
+- /changelog_2023
 ---
 ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0)
 

docs/guides/getting-started-with-opentelemetry/README.md

@@ -314,7 +314,7 @@ func newMetricsController(ctx context.Context) (*controller.Controller, error) {
 
 This controller will collect and push collected metrics to VictoriaMetrics address with interval of `10s`.
 
-See the full source code of the example [here](/guides/app.go.example).
+See the full source code of the example [here](app.go.example).
 
 ### Test metrics ingestion