fix link

Co-authored-by: Roman Khavronenko <roman@victoriametrics.com>

app/vlinsert/datadog/datadog.go

@@ -101,7 +101,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 
 var (
 	v2LogsRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
-	v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
+	v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
 )
 
 // datadog message field has two formats:

app/vlinsert/elasticsearch/elasticsearch.go

@@ -129,7 +129,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 
 var (
 	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
-	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
+	bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
 func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {

app/vlinsert/internalinsert/internalinsert.go

@@ -84,5 +84,5 @@ var (
 	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
 	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
 
-	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
+	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/internal/insert"}`)
 )

app/vlinsert/journald/journald.go

@@ -28,24 +28,6 @@ import (
 // See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
 const maxFieldNameLen = 64
 
-func isValidJournaldFieldName(s string) bool {
-	if len(s) == 0 {
-		return false
-	}
-	c := s[0]
-	if !(c >= 'A' && c <= 'Z' || c == '_') {
-		return false
-	}
-
-	for i := 1; i < len(s); i++ {
-		c := s[i]
-		if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
-			return false
-		}
-	}
-	return true
-}
-
 var (
 	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
 		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields")
@@ -163,7 +145,7 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
 var (
 	requestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
 	errorsTotal     = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
-	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
+	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
 )
 
 func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
@@ -214,6 +196,18 @@ func (fb *fieldsBuf) addField(name, value string) {
 	})
 }
 
+func (fb *fieldsBuf) appendNextLineToValue(lr *insertutil.LineReader) error {
+	if !lr.NextLine() {
+		if err := lr.Err(); err != nil {
+			return err
+		}
+		return fmt.Errorf("unexpected end of stream")
+	}
+	fb.value = append(fb.value, lr.Line...)
+	fb.value = append(fb.value, '\n')
+	return nil
+}
+
 func getFieldsBuf() *fieldsBuf {
 	fb := fieldsBufPool.Get()
 	if fb == nil {
@@ -259,42 +253,35 @@ func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp inse
 			return nil
 		}
 
-		// line could be either "key=value\n" or "key\n<little_endian_size_64>value\n"
+		// line could be either "key=value" or "key"
 		// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
 		if n := bytes.IndexByte(line, '='); n >= 0 {
-			// "key=value\n"
+			// line = "key=value"
 			fb.name = append(fb.name[:0], line[:n]...)
 			name = bytesutil.ToUnsafeString(fb.name)
 
 			fb.value = append(fb.value[:0], line[n+1:]...)
 			value = bytesutil.ToUnsafeString(fb.value)
 		} else {
-			// "key\n<little_endian_size_64>value\n"
+			// line = "key"
+			// Parse the binary-encoded value from the next line according to "key\n<little_endian_size_64>value\n" format
 			fb.name = append(fb.name[:0], line...)
 			name = bytesutil.ToUnsafeString(fb.name)
 
 			fb.value = fb.value[:0]
 			for len(fb.value) < 8 {
-				if !lr.NextLine() {
-					if err := lr.Err(); err != nil {
-						return fmt.Errorf("cannot read value size: %w", err)
-					}
-					return fmt.Errorf("unexpected end of stream while reading value size")
+				if err := fb.appendNextLineToValue(lr); err != nil {
+					return fmt.Errorf("cannot read value size: %w", err)
 				}
-				fb.value = append(fb.value, lr.Line...)
-				fb.value = append(fb.value, '\n')
 			}
 			size := binary.LittleEndian.Uint64(fb.value[:8])
 
-			for size > uint64(len(fb.value[8:])) {
-				if !lr.NextLine() {
-					if err := lr.Err(); err != nil {
-						return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
-					}
-					return fmt.Errorf("unexpected end of stream while reading %q value with size %d bytes; read only %d bytes", fb.name, size, len(fb.value[8:]))
+			// Read the value until its lenth exceeds the given size - the last char in the read value will always be '\n'
+			// because it is appended by appendNextLineToValue().
+			for uint64(len(fb.value[8:])) <= size {
+				if err := fb.appendNextLineToValue(lr); err != nil {
+					return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
 				}
-				fb.value = append(fb.value, lr.Line...)
-				fb.value = append(fb.value, '\n')
 			}
 			value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
 			if uint64(len(value)) != size {
@@ -314,7 +301,7 @@ func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp inse
 			logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
 			continue
 		}
-		if !isValidJournaldFieldName(name) {
+		if !isValidFieldName(name) {
 			logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
 			continue
 		}
@@ -350,13 +337,19 @@ func journaldPriorityToLevel(priority string) string {
 	// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
 	// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
 	switch priority {
-	case "0", "1", "2":
+	case "0":
+		return "emerg"
+	case "1":
+		return "alert"
+	case "2":
 		return "critical"
 	case "3":
 		return "error"
 	case "4":
 		return "warning"
-	case "5", "6":
+	case "5":
+		return "notice"
+	case "6":
 		return "info"
 	case "7":
 		return "debug"
@@ -364,3 +357,21 @@ func journaldPriorityToLevel(priority string) string {
 		return priority
 	}
 }
+
+func isValidFieldName(s string) bool {
+	if len(s) == 0 {
+		return false
+	}
+	c := s[0]
+	if !(c >= 'A' && c <= 'Z' || c == '_') {
+		return false
+	}
+
+	for i := 1; i < len(s); i++ {
+		c := s[i]
+		if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
+			return false
+		}
+	}
+	return true
+}

app/vlinsert/journald/journald_test.go

@@ -8,11 +8,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
 )
 
-func TestIsValidJournaldFieldName(t *testing.T) {
+func TestIsValidFieldName(t *testing.T) {
 	f := func(name string, resultExpected bool) {
 		t.Helper()
 
-		result := isValidJournaldFieldName(name)
+		result := isValidFieldName(name)
 		if result != resultExpected {
 			t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
 		}
@@ -102,6 +102,24 @@ func TestPushJournald_Success(t *testing.T) {
 		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
 	)
 
+	// Parse binary data with trailing newline
+	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x14\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n\n_PID=2763\n\n",
+		[]int64{1729698775704404000},
+		`{"_CMDLINE":"python3","_msg":"foo\nbar\n\n\nasda\nasda\n","_PID":"2763"}`,
+	)
+	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00\n_PID=2763\n\n",
+		[]int64{1729698775704404000},
+		`{"_CMDLINE":"python3","_PID":"2763"}`,
+	)
+	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n\n_PID=2763\n\n",
+		[]int64{1729698775704404000},
+		`{"_CMDLINE":"python3","_msg":"123456789\n","_PID":"2763"}`,
+	)
+	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890\n_PID=2763\n\n",
+		[]int64{1729698775704404000},
+		`{"_CMDLINE":"python3","_msg":"1234567890","_PID":"2763"}`,
+	)
+
 	// Empty field name must be ignored
 	f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
 	f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
@@ -138,7 +156,10 @@ func TestPushJournald_Failure(t *testing.T) {
 	}
 
 	// too short binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda")
+	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasd")
+	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00_PID=2763\n\n")
+	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890_PID=2763\n\n")
+	f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n_PID=2763\n\n")
 
 	// too long binary encoded message
 	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")

app/vlinsert/journald/journald_timing_test.go

@@ -4,12 +4,55 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
 )
 
+func BenchmarkIsValidFieldName(b *testing.B) {
+	b.ReportAllocs()
+	b.SetBytes(int64(len(benchmarkFields)))
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			for _, field := range benchmarkFields {
+				if !isValidFieldName(field) {
+					panic(fmt.Errorf("cannot validate field %q", field))
+				}
+			}
+		}
+	})
+}
+
+var benchmarkFields = strings.Split(
+	"E,_BOOT_ID,_UID,_GID,_MACHINE_ID,_HOSTNAME,_RUNTIME_SCOPE,_TRANSPORT,_CAP_EFFECTIVE,_SYSTEMD_CGROUP,_SYSTEMD_UNIT,"+
+		"_SYSTEMD_SLICE,CODE_FILE,CODE_LINE,CODE_FUNC,SYSLOG_IDENTIFIER,_COMM,_EXE,_CMDLINE,MESSAGE,_PID,_SOURCE_REALTIME_TIMESTAMP,_REALTIME_TIMESTAMP",
+	",")
+
+func BenchmarkPushJournaldPerformance(b *testing.B) {
+	cp := &insertutil.CommonParams{
+		TimeFields: []string{"__REALTIME_TIMESTAMP"},
+		MsgFields:  []string{"MESSAGE"},
+	}
+	const dataChunkSize = 1024 * 1024
+
+	data := generateJournaldData(dataChunkSize)
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(data)))
+	b.RunParallel(func(pb *testing.PB) {
+		r := &bytes.Reader{}
+		blp := &insertutil.BenchmarkLogMessageProcessor{}
+		for pb.Next() {
+			r.Reset(data)
+			if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
+				panic(fmt.Errorf("unexpected error: %w", err))
+			}
+		}
+	})
+}
+
 func generateJournaldData(size int) []byte {
 	var buf []byte
 	timestamp := time.Now().UnixMicro()
@@ -37,26 +80,3 @@ func generateJournaldData(size int) []byte {
 	}
 	return buf
 }
-
-func BenchmarkPushJournaldPerformance(b *testing.B) {
-	cp := &insertutil.CommonParams{
-		TimeFields: []string{"__REALTIME_TIMESTAMP"},
-		MsgFields:  []string{"MESSAGE"},
-	}
-	const dataChunkSize = 1024 * 1024
-
-	data := generateJournaldData(dataChunkSize)
-
-	b.ReportAllocs()
-	b.SetBytes(int64(len(data)))
-	b.RunParallel(func(pb *testing.PB) {
-		r := &bytes.Reader{}
-		blp := &insertutil.BenchmarkLogMessageProcessor{}
-		for pb.Next() {
-			r.Reset(data)
-			if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
-				panic(fmt.Errorf("unexpected error: %w", err))
-			}
-		}
-	})
-}

app/vlinsert/jsonline/jsonline.go

@@ -119,5 +119,5 @@ var (
 	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
 	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
 
-	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
+	requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
 )

app/vlinsert/loki/loki_json.go

@@ -58,7 +58,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 
 var (
 	requestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
+	requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
 )
 
 func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {

app/vlinsert/loki/loki_protobuf.go

@@ -62,7 +62,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 
 var (
 	requestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
-	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
 func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {

app/vlinsert/opentelemetry/opentelemetry.go

@@ -70,7 +70,7 @@ var (
 	requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
 	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
 
-	requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
+	requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
 )
 
 func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {

app/vlinsert/syslog/syslog_test.go

@@ -101,7 +101,7 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	currentYear := 2023
 	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
 	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility_keyword":"local4","level":"info","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
 {"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
 	f(data, currentYear, timestampsExpected, resultExpected)
 }

app/vlselect/vmui/assets/MetricsQL-Ckl-MH95.md

@@ -742,7 +742,23 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 This function is supported by PromQL.
 
-See also [irate](#irate) and [rollup_rate](#rollup_rate).
+See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
+
+#### rate_prometheus
+
+`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
+increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
+The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
+
+It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
+See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
+
+See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
+
 
 #### rate_over_sum
 

app/vlselect/vmui/index.html

@@ -35,10 +35,10 @@
   <meta property="og:title" content="UI for VictoriaLogs">
   <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
   <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-DhqzKCNf.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
+  <script type="module" crossorigin src="./assets/index-721xTF8u.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-V4vnRsM-.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-C36SC0pJ.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmalert/replay.go

@@ -20,9 +20,8 @@ var (
 		"The time filter in RFC3339 format to finish the replay by. E.g. '2020-01-01T20:07:00Z'. "+
 			"By default, is set to the current time.")
 	replayRulesDelay = flag.Duration("replay.rulesDelay", time.Second,
-		"Delay between rules evaluation within the group. Could be important if there are chained rules inside the group "+
-			"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
-			"Keep it equal or bigger than -remoteWrite.flushInterval.")
+		"Delay before evaluating the next rule within the group. Is important for chained rules. "+
+			"Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting.")
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
 		"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,

app/vmalert/replay_test.go

@@ -39,7 +39,7 @@ func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to ti
 }
 
 func TestReplay(t *testing.T) {
-	f := func(from, to string, maxDP int, cfg []config.Group, qb *fakeReplayQuerier) {
+	f := func(from, to string, maxDP int, ruleDelay time.Duration, cfg []config.Group, qb *fakeReplayQuerier) {
 		t.Helper()
 
 		fromOrig, toOrig, maxDatapointsOrig := *replayFrom, *replayTo, *replayMaxDatapoints
@@ -51,7 +51,7 @@ func TestReplay(t *testing.T) {
 		}()
 
 		*replayRuleRetryAttempts = 1
-		*replayRulesDelay = time.Millisecond
+		*replayRulesDelay = ruleDelay
 		rwb := &remotewrite.DebugClient{}
 		*replayFrom = from
 		*replayTo = to
@@ -65,7 +65,7 @@ func TestReplay(t *testing.T) {
 	}
 
 	// one rule + one response
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, []config.Group{
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, time.Millisecond, []config.Group{
 		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
 	}, &fakeReplayQuerier{
 		registry: map[string]map[string]struct{}{
@@ -74,7 +74,7 @@ func TestReplay(t *testing.T) {
 	})
 
 	// one rule + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
 		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
 	}, &fakeReplayQuerier{
 		registry: map[string]map[string]struct{}{
@@ -87,7 +87,7 @@ func TestReplay(t *testing.T) {
 	})
 
 	// datapoints per step
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, []config.Group{
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, time.Millisecond, []config.Group{
 		{Interval: promutil.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
 	}, &fakeReplayQuerier{
 		registry: map[string]map[string]struct{}{
@@ -101,7 +101,7 @@ func TestReplay(t *testing.T) {
 	})
 
 	// multiple recording rules + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
 		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
 		{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
 	}, &fakeReplayQuerier{
@@ -120,7 +120,7 @@ func TestReplay(t *testing.T) {
 	})
 
 	// multiple alerting rules + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
 		{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
 		{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
 	}, &fakeReplayQuerier{
@@ -137,4 +137,21 @@ func TestReplay(t *testing.T) {
 			},
 		},
 	})
+
+	// multiple alerting rules in one group+ multiple responses + concurrency
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 0, []config.Group{
+		{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}, {Alert: "bar", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
+		registry: map[string]map[string]struct{}{
+			"sum(up) > 1": {
+				"12:00:00+12:01:00": {},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+			"max(up) < 1": {
+				"12:00:00+12:01:00": {},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	})
 }

app/vmalert/rule/group.go

@@ -445,11 +445,17 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 
 			g.infof("re-started")
 		case <-t.C:
-			missed := (time.Since(evalTS) / g.Interval) - 1
+			// calculate the real wall clock offset by stripping the monotonic clock first,
+			// then evalTS can be corrected when wall clock is adjusted.
+			// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8790#issuecomment-2986541829
+			offset := time.Now().Round(0).Sub(evalTS.Round(0))
+			missed := (offset / g.Interval) - 1
 			if missed < 0 {
 				// missed can become < 0 due to irregular delays during evaluation
-				// which can result in time.Since(evalTS) < g.Interval
+				// which can result in time.Since(evalTS) < g.Interval;
+				// or the system wall clock was changed backward
 				missed = 0
+				evalTS = time.Now()
 			}
 			if missed > 0 {
 				g.metrics.iterationMissed.Inc()
@@ -514,36 +520,84 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 	iterations := int(end.Sub(start)/step) + 1
 	fmt.Printf("\nGroup %q"+
 		"\ninterval: \t%v"+
-		"\nrequests to make: \t%d"+
+		"\nconcurrency: \t %d"+
+		"\nrequests to make per rule: \t%d"+
 		"\nmax range per request: \t%v\n",
-		g.Name, g.Interval, iterations, step)
+		g.Name, g.Interval, g.Concurrency, iterations, step)
 	if g.Limit > 0 {
-		fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
+		fmt.Printf("\nWarning: `limit: %d` param has no effect during replay.\n",
 			g.Limit)
 	}
-	for _, rule := range g.Rules {
-		fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
-		var bar *pb.ProgressBar
-		if !disableProgressBar {
-			bar = pb.StartNew(iterations)
-		}
-		ri.reset()
-		for ri.next() {
-			n, err := replayRule(rule, ri.s, ri.e, rw, replayRuleRetryAttempts)
-			if err != nil {
-				logger.Fatalf("rule %q: %s", rule, err)
+	concurrency := g.Concurrency
+	if g.Concurrency > 1 && replayDelay > 0 {
+		fmt.Printf("\nWarning: group concurrency %d will be ignored since `-replay.rulesDelay` is %.3f seconds."+
+			" Set -replay.rulesDelay=0 to enable concurrency for replay.\n", g.Concurrency, replayDelay.Seconds())
+		concurrency = 1
+	}
+
+	if concurrency == 1 {
+		for _, rule := range g.Rules {
+			var bar *pb.ProgressBar
+			if !disableProgressBar {
+				bar = pb.StartNew(iterations)
 			}
-			total += n
+			// pass ri as a copy, so it can be modified within the replayRuleRange
+			total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts)
 			if bar != nil {
-				bar.Increment()
+				bar.Finish()
 			}
+			// sleep to let remote storage to flush data on-disk
+			// so chained rules could be calculated correctly
+			time.Sleep(replayDelay)
+		}
+		return total
+	}
+
+	sem := make(chan struct{}, g.Concurrency)
+	res := make(chan int, len(g.Rules)*iterations)
+	wg := sync.WaitGroup{}
+	var bar *pb.ProgressBar
+	if !disableProgressBar {
+		bar = pb.StartNew(iterations * len(g.Rules))
+	}
+	for _, r := range g.Rules {
+		sem <- struct{}{}
+		wg.Add(1)
+		go func(r Rule, ri rangeIterator) {
+			// pass ri as a copy, so it can be modified within the replayRuleRange
+			res <- replayRuleRange(r, ri, bar, rw, replayRuleRetryAttempts)
+			<-sem
+			wg.Done()
+		}(r, ri)
+	}
+
+	wg.Wait()
+	close(res)
+	close(sem)
+
+	if bar != nil {
+		bar.Finish()
+	}
+
+	total = 0
+	for n := range res {
+		total += n
+	}
+	return total
+}
+
+func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts int) int {
+	fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
+	total := 0
+	for ri.next() {
+		n, err := replayRule(r, ri.s, ri.e, rw, replayRuleRetryAttempts)
+		if err != nil {
+			logger.Fatalf("rule %q: %s", r, err)
 		}
 		if bar != nil {
-			bar.Finish()
+			bar.Increment()
 		}
-		// sleep to let remote storage to flush data on-disk
-		// so chained rules could be calculated correctly
-		time.Sleep(replayDelay)
+		total += n
 	}
 	return total
 }
@@ -570,11 +624,10 @@ type rangeIterator struct {
 	s, e time.Time
 }
 
-func (ri *rangeIterator) reset() {
-	ri.iter = 0
-	ri.s, ri.e = time.Time{}, time.Time{}
-}
-
+// next iterates with given step between start and end
+// by modifying iter, s and e.
+// Returns true until it reaches end.
+// next modifies ri and isn't thread-safe.
 func (ri *rangeIterator) next() bool {
 	ri.s = ri.start.Add(ri.step * time.Duration(ri.iter))
 	if !ri.end.After(ri.s) {

app/vmselect/vmui/assets/MetricsQL-Ckl-MH95.md

@@ -742,7 +742,23 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 This function is supported by PromQL.
 
-See also [irate](#irate) and [rollup_rate](#rollup_rate).
+See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
+
+#### rate_prometheus
+
+`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
+increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
+The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
+
+It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
+See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
+
+See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
+
 
 #### rate_over_sum
 

app/vmselect/vmui/index.html

@@ -36,10 +36,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-D-ssBbZq.js"></script>
+  <script type="module" crossorigin src="./assets/index-BiQY-19a.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-ojCMu5lE.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -63,6 +63,8 @@ var (
 
 	cacheSizeStorageTSID = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
+	cacheSizeStorageMetricName = flagutil.NewBytes("storage.cacheSizeStorageMetricName", 0, "Overrides max size for storage/metricName cache. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
 	cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
 	cacheSizeIndexDBDataBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. "+
@@ -111,6 +113,7 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
 	storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
+	storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
 	mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())

app/vmui/packages/vmui/package-lock.json

@@ -10,6 +10,8 @@
       "dependencies": {
         "@types/lodash.debounce": "^4.0.9",
         "@types/lodash.get": "^4.4.9",
+        "@types/lodash.orderBy": "^4.6.9",
+        "@types/lodash.throttle": "^4.1.9",
         "@types/qs": "^6.9.18",
         "@types/react": "^19.1.2",
         "@types/react-input-mask": "^3.0.6",
@@ -18,6 +20,8 @@
         "dayjs": "^1.11.13",
         "lodash.debounce": "^4.0.8",
         "lodash.get": "^4.4.2",
+        "lodash.orderBy": "^4.6.0",
+        "lodash.throttle": "^4.1.1",
         "marked": "^15.0.8",
         "marked-emoji": "^2.0.0",
         "preact": "^10.26.5",
@@ -2191,6 +2195,24 @@
         "@types/lodash": "*"
       }
     },
+    "node_modules/@types/lodash.orderBy": {
+      "version": "4.6.9",
+      "resolved": "https://registry.npmjs.org/@types/lodash.orderby/-/lodash.orderby-4.6.9.tgz",
+      "integrity": "sha512-T9o2wkIJOmxXwVTPTmwJ59W6eTi2FseiLR369fxszG649Po/xe9vqFNhf/MtnvT5jrbDiyWKxPFPZbpSVK0SVQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/lodash": "*"
+      }
+    },
+    "node_modules/@types/lodash.throttle": {
+      "version": "4.1.9",
+      "resolved": "https://registry.npmjs.org/@types/lodash.throttle/-/lodash.throttle-4.1.9.tgz",
+      "integrity": "sha512-PCPVfpfueguWZQB7pJQK890F2scYKoDUL3iM522AptHWn7d5NQmeS/LTEHIcLr5PaTzl3dK2Z0xSUHHTHwaL5g==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/lodash": "*"
+      }
+    },
     "node_modules/@types/node": {
       "version": "22.14.1",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz",
@@ -5742,6 +5764,18 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/lodash.orderBy": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/lodash.orderby/-/lodash.orderby-4.6.0.tgz",
+      "integrity": "sha512-T0rZxKmghOOf5YPnn8EY5iLYeWCpZq8G41FfqoVHH5QDTAFaghJRmAdLiadEDq+ztgM2q5PjA+Z1fOwGrLgmtg==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.throttle": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz",
+      "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==",
+      "license": "MIT"
+    },
     "node_modules/loose-envify": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",

app/vmui/packages/vmui/package.json

@@ -7,6 +7,8 @@
   "dependencies": {
     "@types/lodash.debounce": "^4.0.9",
     "@types/lodash.get": "^4.4.9",
+    "@types/lodash.orderBy": "^4.6.9",
+    "@types/lodash.throttle": "^4.1.9",
     "@types/qs": "^6.9.18",
     "@types/react": "^19.1.2",
     "@types/react-input-mask": "^3.0.6",
@@ -15,6 +17,8 @@
     "dayjs": "^1.11.13",
     "lodash.debounce": "^4.0.8",
     "lodash.get": "^4.4.2",
+    "lodash.orderBy": "^4.6.0",
+    "lodash.throttle": "^4.1.1",
     "marked": "^15.0.8",
     "marked-emoji": "^2.0.0",
     "preact": "^10.26.5",

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -742,7 +742,23 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 
 This function is supported by PromQL.
 
-See also [irate](#irate) and [rollup_rate](#rollup_rate).
+See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
+
+#### rate_prometheus
+
+`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
+increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
+The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
+
+It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
+See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
+
+Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
+
+This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
+
+See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
+
 
 #### rate_over_sum
 

app/vmui/packages/vmui/src/components/Main/Button/Button.tsx

@@ -1,4 +1,4 @@
-import React, { FC } from "preact/compat";
+import { FC } from "preact/compat";
 import classNames from "classnames";
 import { MouseEvent as ReactMouseEvent, ReactNode } from "react";
 import "./style.scss";
@@ -14,6 +14,7 @@ interface ButtonProps {
   disabled?: boolean
   children?: ReactNode
   className?: string
+  "data-id"?: string
   onClick?: (e: ReactMouseEvent<HTMLButtonElement>) => void
   onMouseDown?: (e: ReactMouseEvent<HTMLButtonElement>) => void
 }
@@ -31,6 +32,7 @@ const Button: FC<ButtonProps> = ({
   disabled,
   onClick,
   onMouseDown,
+  "data-id": dataId
 }) => {
 
   const classesButton = classNames({
@@ -50,6 +52,7 @@ const Button: FC<ButtonProps> = ({
       aria-label={ariaLabel}
       onClick={onClick}
       onMouseDown={onMouseDown}
+      data-id={dataId}
     >
       {startIcon}{children}{endIcon}
     </button>

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -656,3 +656,33 @@ export const ScrollToTopIcon = () => (
     />
   </svg>
 );
+
+export const SortIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path d="M4 3 L4 15 L1.5 15 L5.5 21 L9.5 15 L7 15 L7 3 Z"/>
+    <path d="M13 21 L13 9 L10.5 9 L14.5 3 L18.5 9 L16 9 L16 21 Z"/>
+  </svg>
+);
+
+export const SortArrowDownIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path d="M10.5 3 L10.5 15 L8 15 L12 21 L16 15 L13.5 15 L13.5 3 Z"/>
+  </svg>
+);
+
+export const SortArrowUpIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path d="M10.5 21 L10.5 9 L8 9 L12 3 L16 9 L13.5 9 L13.5 21 Z"/>
+  </svg>
+);
+
+

app/vmui/packages/vmui/src/components/Table/TableSettings/TableSettings.tsx

@@ -18,8 +18,8 @@ const title = "Table settings";
 interface TableSettingsProps {
   columns: string[];
   selectedColumns?: string[];
-  tableCompact: boolean;
-  toggleTableCompact: () => void;
+  tableCompact?: boolean;
+  toggleTableCompact?: () => void;
   onChangeColumns: (arr: string[]) => void
 }
 
@@ -195,18 +195,20 @@ const TableSettings: FC<TableSettingsProps> = ({
               </div>
             </div>
           </div>
-          <div className="vm-table-settings-modal-section">
-            <div className="vm-table-settings-modal-section__title">
+          {toggleTableCompact && tableCompact !== undefined && (
+            <div className="vm-table-settings-modal-section">
+              <div className="vm-table-settings-modal-section__title">
               Table view
+              </div>
+              <div className="vm-table-settings-modal-columns-list__item">
+                <Switch
+                  label={"Compact view"}
+                  value={tableCompact}
+                  onChange={toggleTableCompact}
+                />
+              </div>
             </div>
-            <div className="vm-table-settings-modal-columns-list__item">
-              <Switch
-                label={"Compact view"}
-                value={tableCompact}
-                onChange={toggleTableCompact}
-              />
-            </div>
-          </div>
+          )}
         </Modal>)}
     </div>
   );

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/style.scss

@@ -5,7 +5,7 @@
 
   &-header {
     background-color: $color-background-block;
-    z-index: 1;
+    z-index: 3;
     margin: -$padding-medium 0-$padding-medium 0;
     position: sticky;
     top: 0;

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/JsonView/JsonView.tsx

@@ -1,21 +1,67 @@
-import React, { FC } from "preact/compat";
+import React, { FC, useMemo, useCallback, createPortal } from "preact/compat";
 import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
-import { createPortal } from "preact/compat";
 import JsonViewComponent from "../../../../../components/Views/JsonView/JsonView";
 import { ViewProps } from "../../types";
 import EmptyLogs from "../components/EmptyLogs/EmptyLogs";
-import { useCallback } from "react";
+import JsonViewSettings from "./JsonViewSettings/JsonViewSettings";
+import { useSearchParams } from "react-router-dom";
+import orderBy from "lodash.orderBy";
+import "./style.scss";
+import { Logs } from "../../../../../api/types";
+import { SortDirection } from "./types";
 
 const MemoizedJsonView = React.memo(JsonViewComponent);
 
+const jsonQuerySortParam = "json_sort";
+const fieldSortQueryParamName = "json_field_sort";
+
 const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
   const getLogs = useCallback(() => data, [data]);
 
+  const [searchParams] = useSearchParams();
+  const sortParam = searchParams.get(jsonQuerySortParam);
+  const fieldSortParam = searchParams.get(fieldSortQueryParamName) as SortDirection;
+
+  const [sortField, sortDirection] = useMemo(() => {
+    const [sortField, sortDirection] = sortParam?.split(":").map(decodeURIComponent) || [];
+    return [sortField, sortDirection as "asc" | "desc" | undefined];
+  }, [sortParam]);
+
+  const fields = useMemo(() => {
+    const keys = new Set(data.flatMap(Object.keys));
+    return Array.from(keys);
+  }, [data]);
+
+  const orderedFieldsData = useMemo(() => {
+    if (!fieldSortParam) return data;
+    const orderedFields = fields.toSorted((a, b) => fieldSortParam === "asc" ? a.localeCompare(b): b.localeCompare(a));
+    return data.map((item) => {
+      return orderedFields.reduce((acc, field) => {
+        if (item[field]) acc[field] = item[field];
+        return acc;
+      }, {} as Logs);
+    });
+  }, [fields, fieldSortParam, data]);
+
+  const sortedData = useMemo(() => {
+    if (!sortField || !sortDirection) return orderedFieldsData;
+    return orderBy(orderedFieldsData, [sortField], [sortDirection]);
+  }, [orderedFieldsData, sortField, sortDirection]);
+
   const renderSettings = () => {
     if (!settingsRef.current) return null;
 
     return createPortal(
-      data.length > 0 && <DownloadLogsButton getLogs={getLogs} />,
+      data.length > 0 && (
+        <div className="vm-json-view__settings-container">
+          <DownloadLogsButton getLogs={getLogs} />
+          <JsonViewSettings
+            fields={fields}
+            sortQueryParamName={jsonQuerySortParam}
+            fieldSortQueryParamName={fieldSortQueryParamName}
+          />
+        </div>
+      ),
       settingsRef.current
     );
   };
@@ -25,9 +71,11 @@ const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
   return (
     <>
       {renderSettings()}
-      <MemoizedJsonView data={data} />
+      <MemoizedJsonView
+        data={sortedData}
+      />
     </>
   );
 };
 
-export default JsonView; 
+export default JsonView;

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/JsonView/JsonViewSettings/JsonViewSettings.tsx

@@ -0,0 +1,185 @@
+import { FC, useMemo, useRef } from "preact/compat";
+import Button from "../../../../../../components/Main/Button/Button";
+import { SettingsIcon, SortArrowDownIcon, SortArrowUpIcon, SortIcon } from "../../../../../../components/Main/Icons";
+import Tooltip from "../../../../../../components/Main/Tooltip/Tooltip";
+import Select from "../../../../../../components/Main/Select/Select";
+import useBoolean from "../../../../../../hooks/useBoolean";
+import { useState, useEffect, useCallback } from "react";
+import Modal from "../../../../../../components/Main/Modal/Modal";
+import { useSearchParams } from "react-router-dom";
+import "./style.scss";
+import { SortDirection } from "../types";
+
+const title = "JSON settings";
+const directionList = ["asc", "desc"];
+
+interface JsonSettingsProps {
+  fields: string[];
+  sortQueryParamName: string;
+  fieldSortQueryParamName: string;
+}
+
+const JsonViewSettings: FC<JsonSettingsProps> = ({
+  fields,
+  sortQueryParamName,
+  fieldSortQueryParamName
+}) => {
+  const [searchParams, setSearchParams] = useSearchParams();
+  const buttonRef = useRef<HTMLDivElement>(null);
+  const [fieldSortDirection, setFieldSortDirection] = useState<SortDirection>(null);
+
+  const {
+    value: openSettings,
+    toggle: toggleOpenSettings,
+    setFalse: handleClose,
+  } = useBoolean(false);
+  
+  const [sortField, setSortField] = useState<string | null>(null);
+  const [sortDirection, setSortDirection] = useState<SortDirection>(null);
+
+  useEffect(() => {
+    const sortParam = searchParams.get(sortQueryParamName);
+    const isSortDirection = (value: string) : value is Exclude<SortDirection, null> => directionList.includes(value);
+    if (sortParam) {
+      const [field, direction] = sortParam.split(":").map(decodeURIComponent);
+      if (field && (isSortDirection(direction))) {
+        setSortField(field);
+        setSortDirection(direction);
+      }
+    }
+
+    const fieldSortParam = searchParams.get(fieldSortQueryParamName);
+    if (fieldSortParam === "asc" || fieldSortParam === "desc") {
+      setFieldSortDirection(fieldSortParam);
+    }
+  }, [searchParams, sortQueryParamName, fieldSortQueryParamName, setSortField, setSortDirection, setFieldSortDirection]);
+
+  const updateSortParams = useCallback((field: string | null, direction: SortDirection) => {
+    const updatedParams = new URLSearchParams(searchParams.toString());
+
+    if (!field || !direction) {
+      updatedParams.delete(sortQueryParamName);
+    } else {
+      updatedParams.set(sortQueryParamName, `${field}:${direction || ""}`);
+    }
+
+    setSearchParams(updatedParams);
+  }, [searchParams, sortQueryParamName]);
+
+  const handleSort = (field: string) => {
+    const newDirection: SortDirection = sortDirection || "asc";
+    setSortField(field);
+    setSortDirection(newDirection);
+    updateSortParams(field, newDirection);
+  };
+
+  const resetSort = () => {
+    setSortField(null);
+    setSortDirection(null);
+    updateSortParams(null, null);
+  };
+
+  const changeFieldSortDirection = useCallback(() => {
+    let newFieldSortDirection: SortDirection = null;
+    if (fieldSortDirection === null) {
+      newFieldSortDirection = "asc";
+    }else if (fieldSortDirection === "asc") {
+      newFieldSortDirection = "desc";
+    }
+    setFieldSortDirection(newFieldSortDirection);
+    const updatedParams = new URLSearchParams(searchParams.toString());
+
+    if (!newFieldSortDirection) {
+      updatedParams.delete(fieldSortQueryParamName);
+    } else {
+      updatedParams.set(fieldSortQueryParamName, encodeURIComponent(newFieldSortDirection));
+    }
+
+    setSearchParams(updatedParams);
+  },[fieldSortDirection, searchParams, fieldSortQueryParamName]);
+
+  const handleChangeSortDirection = (direction: string) => {
+    const field = sortField || fields[0];
+    setSortField(field);
+    setSortDirection(direction as SortDirection);
+    updateSortParams(field, direction as SortDirection);
+  };
+
+  const fieldSortMeta = useMemo(() => ({
+    default: {
+      title: "Set field sort order. Click to sort in ascending order",
+      icon: <SortIcon />
+    },
+    asc: {
+      title: "Fields sorted ascending. Click to sort in descending order",
+      icon: <SortArrowDownIcon />
+    },
+    desc: {
+      title: "Fields sorted descending. Click to reset sort",
+      icon: <SortArrowUpIcon />
+    },
+  }), []);
+
+  const fieldSortButton = useMemo(() => {
+    const { title, icon } = fieldSortMeta[fieldSortDirection ?? "default"];
+    return <Tooltip title={title}>
+      <Button
+        variant="text"
+        startIcon={icon}
+        onClick={changeFieldSortDirection}
+        ariaLabel={title}
+      />
+    </Tooltip>;
+  }, [fieldSortDirection, toggleOpenSettings, changeFieldSortDirection, fieldSortMeta]);
+
+
+  return (
+    <div className="vm-json-settings">
+      {fieldSortButton}
+      <Tooltip title={title}>
+        <div ref={buttonRef}>
+          <Button
+            variant="text"
+            startIcon={<SettingsIcon/>}
+            onClick={toggleOpenSettings}
+            ariaLabel={title}
+          />
+        </div>
+      </Tooltip>
+      {openSettings && (
+        <Modal
+          title={title}
+          className="vm-json-settings-modal"
+          onClose={handleClose}
+        >
+          <div className="vm-json-settings-modal-section">
+            <div className="vm-json-settings-modal-section__sort-settings-container">
+              <Select
+                value={sortField || ""}
+                onChange={handleSort}
+                list={fields}
+                label="Select field"
+              />
+              <Select
+                value={sortDirection || ""}
+                onChange={handleChangeSortDirection}
+                list={directionList}
+                label="Sort direction"
+              />
+              {(sortField || sortDirection) && (
+                <Button
+                  variant="outlined"
+                  color="error"
+                  onClick={resetSort}
+                >
+                  Reset sort
+                </Button>
+              )}
+            </div>
+          </div>
+        </Modal>)}
+    </div>
+  );
+};
+
+export default JsonViewSettings;

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/JsonView/JsonViewSettings/style.scss

@@ -0,0 +1,34 @@
+@use "src/styles/variables" as *;
+
+.vm-json-settings {
+  display: flex;
+  flex-direction: row;
+
+  &-modal {
+    .vm-modal-content-body {
+      min-width: clamp(300px, 600px, 90vw);
+      padding: 0;
+    }
+
+    &-section {
+      padding-block: $padding-global;
+      border-top: $border-divider;
+
+      &:first-child {
+        padding-top: 0;
+        border-top: none;
+      }
+
+      &__sort-settings-container {
+        display: grid;
+        padding: $padding-medium;
+        grid-template-columns: 1fr 1fr 80px;
+        gap: $padding-medium;
+
+        @media (max-width: 500px) {
+          grid-template-columns: 1fr;
+        }
+      }
+    }
+  }
+}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/JsonView/style.scss

@@ -0,0 +1,9 @@
+@use "src/styles/variables" as *;
+
+.vm-json-view {
+  &__settings-container {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+  }
+}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/JsonView/types.ts

@@ -0,0 +1 @@
+export type SortDirection = "asc" | "desc" | null;

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/LiveTailingView/LiveTailingView.tsx

@@ -7,7 +7,7 @@ import { useLiveTailingLogs } from "./useLiveTailingLogs";
 import { LOGS_DISPLAY_FIELDS, LOGS_URL_PARAMS } from "../../../../../constants/logs";
 import { useMemo } from "react";
 import { useSearchParams } from "react-router-dom";
-import throttle from "lodash/throttle";
+import throttle from "lodash.throttle";
 import GroupLogsItem from "../../../GroupLogs/GroupLogsItem";
 import LiveTailingSettings from "./LiveTailingSettings";
 import Alert from "../../../../../components/Main/Alert/Alert";

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/TableView/TableView.tsx

@@ -3,7 +3,6 @@ import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
 import { createPortal } from "preact/compat";
 import "./style.scss";
 import { ViewProps } from "../../types";
-import useBoolean from "../../../../../hooks/useBoolean";
 import useStateSearchParams from "../../../../../hooks/useStateSearchParams";
 import TableLogs from "../../TableLogs";
 import SelectLimit from "../../../../../components/Main/Pagination/SelectLimit/SelectLimit";
@@ -18,7 +17,6 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
   const [displayColumns, setDisplayColumns] = useState<string[]>([]);
   const [rowsPerPage, setRowsPerPage] = useStateSearchParams(100, "rows_per_page");
-  const { value: tableCompact, toggle: toggleTableCompact } = useBoolean(false);
 
   const columns = useMemo(() => {
     const keys = new Set<string>();
@@ -52,8 +50,6 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
             columns={columns}
             selectedColumns={displayColumns}
             onChangeColumns={setDisplayColumns}
-            tableCompact={tableCompact}
-            toggleTableCompact={toggleTableCompact}
           />
         </div>
       </div>,
@@ -69,7 +65,7 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
       <MemoizedTableView
         logs={data}
         displayColumns={displayColumns}
-        tableCompact={tableCompact}
+        tableCompact={false}
         columns={columns}
         rowsPerPage={Number(rowsPerPage)}
       />

deployment/docker/Makefile

@@ -95,7 +95,7 @@ publish-via-docker:
 		--label "org.opencontainers.image.version=$(PKG_TAG)" \
 		--label "org.opencontainers.image.created=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")" \
 		$(foreach registry,$(DOCKER_REGISTRIES),\
-		--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
+		--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)$(EXTRA_TAG_SUFFIX) \
 		) \
 		-o type=image \
 		--provenance=false \
@@ -115,7 +115,7 @@ publish-via-docker:
 		--label "org.opencontainers.image.version=$(PKG_TAG)" \
 		--label "org.opencontainers.image.created=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")" \
 		$(foreach registry,$(DOCKER_REGISTRIES),\
-		--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \
+		--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)$(EXTRA_TAG_SUFFIX)-scratch \
 		) \
 		-o type=image \
 		--provenance=false \

deployment/docker/compose-vl-cluster.yml

@@ -36,30 +36,30 @@ services:
     user: root
 
   vlinsert:
-    image: victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: victoriametrics/victoria-logs:v1.24.0-victorialogs
     command:
       - "--storageNode=vlstorage-1:9428"
       - "--storageNode=vlstorage-2:9428"
 
   vlselect-1:
-    image: victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: victoriametrics/victoria-logs:v1.24.0-victorialogs
     command:
       - "--storageNode=vlstorage-1:9428"
       - "--storageNode=vlstorage-2:9428"
   vlselect-2:
-    image: victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: victoriametrics/victoria-logs:v1.24.0-victorialogs
     command:
       - "--storageNode=vlstorage-1:9428"
       - "--storageNode=vlstorage-2:9428"
 
   vlstorage-1:
-    image: victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: victoriametrics/victoria-logs:v1.24.0-victorialogs
     command:
       - "--storageDataPath=/vlogs"
     volumes:
       - vldata-1:/vlogs
   vlstorage-2:
-    image: victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: victoriametrics/victoria-logs:v1.24.0-victorialogs
     command:
       - "--storageDataPath=/vlogs"
     volumes:

deployment/docker/compose-vl-single.yml

@@ -38,7 +38,7 @@ services:
   #  VictoriaLogs instance, a single process responsible for
   #  storing logs and serving read queries.
   victorialogs:
-    image: victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: victoriametrics/victoria-logs:v1.24.0-victorialogs
     ports:
       - "9428:9428"
     command:

deployment/docker/victorialogs/compose-base.yml

@@ -1,7 +1,7 @@
 services:
   # meta service will be ignored by compose
   .victorialogs:
-    image: docker.io/victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v1.24.0-victorialogs
     command:
       - -storageDataPath=/vlogs
       - -loggerFormat=json

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.24.0
+    image: victoriametrics/vmanomaly:v1.24.1
     depends_on:
       - "victoriametrics"
     ports:

deployment/logs-benchmark/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3"
 services:
   # Run `make package-victoria-logs` to build victoria-logs image
   vlogs:
-    image: docker.io/victoriametrics/victoria-logs:v1.23.3-victorialogs
+    image: docker.io/victoriametrics/victoria-logs:v1.24.0-victorialogs
     volumes:
       - vlogs:/vlogs
     ports:

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,16 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+## v1.24.1
+Released: 2025-06-20
+
+- BUGFIX: Resolved the issue first seen in [v1.23.0](#v1230) where some fit and infer jobs were silently skipped at task submission time (due to a bug in the new background scheduler behind [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler)) followed by similar warnings in the logs later on, such as:
+  ```shellhelp
+  2025-06-19 14:32:50,568 - apscheduler.executors.default - WARNING - Run time of job "{job_name}" (trigger: interval[1 day, 0:00:00], next run at: 2025-06-20 14:32:50 UTC)" was missed by 0:00:01.024753
+  ```
+
+- BUGFIX: Resolved the issue where `vmanomaly` service on [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) where `start_from` argument was set and [state restoration](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) was enabled, didn't resume infer jobs after respective fitted models were restored from the previous run. This could lead to a situation where the service, *if restore happened in-between fit calls*, would not produce any anomaly scores and stay idle until the next `fit_every` happens, which is *expected in stateless mode*, but not in *stateful* mode with `restore_state` enabled.
+
 ## v1.24.0
 Released: 2025-06-18
 
@@ -45,6 +55,13 @@ Released: 2025-06-08
 ## v1.23.0
 Released: 2025-06-05
 
+> There is a known bug that can cause some fit and infer jobs to be silently skipped at task submission time (due to a bug in the new background scheduler behind [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler)) followed by similar warnings in the logs later on, such as:
+>  ```shellhelp
+>  2025-06-19 14:32:50,568 - apscheduler.executors.default - WARNING - Run time of job "{job_name}" (trigger: interval[1 day, 0:00:00], next run at: 2025-06-20 14:32:50 UTC)" was missed by 0:00:01.024753
+>  ```
+> Releases affected: [v1.23.0](#v1230) - [v1.23.3](#v1233).
+> **The issue has been resolved in patch [v1.24.1](#v1241), upgrade is recommended.**
+
 - FEATURE: Added `decay` [argument](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay) to [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models). This parameters allows for newer data to be weighted more heavily in online models. By default this is set to 1 which means all data points are weighted the same to maintain backward compatibility with existing configs. The closer this value is to 0 the more important new data is.
 
 - IMPROVEMENT: **Restored back parallelization** in the read/fit/infer pipeline, previously disabled in [v1.22.0](#v1220-experimental) due to deadlock issues. The new implementation prevents deadlocks, allowing to control the parallelization level via `n_workers` in [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). It's suggested to upgrade from [v1.22.0](#v1220) - [v1.22.1](#v1221) to this version to regain the performance benefits of parallel processing.

docs/anomaly-detection/FAQ.md

@@ -230,7 +230,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.24.0
+    image: victoriametrics/vmanomaly:v1.24.1
     # ...
     ports:
       - "8490:8490"
@@ -443,7 +443,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.24.0 && docker image tag victoriametrics/vmanomaly:v1.24.0 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.24.1 && docker image tag victoriametrics/vmanomaly:v1.24.1 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/QuickStart.md

@@ -120,13 +120,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.24.0
+docker pull victoriametrics/vmanomaly:v1.24.1
 ```
 
 2. (Optional step) tag the `vmanomaly` Docker image:
 
 ```sh
-docker image tag victoriametrics/vmanomaly:v1.24.0 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.24.1 vmanomaly
 ```
 
 3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
@@ -160,7 +160,7 @@ docker run -it --user 1000:1000 \
 services:
   # ...
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.24.0
+    image: victoriametrics/vmanomaly:v1.24.1
     volumes:
         $YOUR_LICENSE_FILE_PATH:/license
         $YOUR_CONFIG_FILE_PATH:/config.yml

docs/anomaly-detection/components/models.md

@@ -1278,7 +1278,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.24.0
+docker pull victoriametrics/vmanomaly:v1.24.1
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -1292,7 +1292,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.24.0 /config.yaml \
+victoriametrics/vmanomaly:v1.24.1 /config.yaml \
 --licenseFile=/license
 ```
 

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -387,7 +387,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.24.0
+    image: victoriametrics/vmanomaly:v1.24.1
     depends_on:
       - "victoriametrics"
     ports:

docs/victorialogs/CHANGELOG.md

@@ -18,6 +18,11 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 
 ## tip
 
+## [v1.24.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.24.0-victorialogs)
+
+Released at 2025-06-20
+
+* FEATURE: add `-http.disableKeepAlive` to disable HTTP keep-alives for incoming connections. The flag could improve load balancing among replicas behind HTTP load balancers. See [#9125](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9125) and [#2395](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2395) for details.
 * FEATURE: [`delete` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#delete-pipe): allow deleting all the fields with common prefix via `... | delete prefix*` syntax.
 * FEATURE: [`fields` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#fields-pipe): allow keeping all the fields with common prefix via `... | fields prefix*` syntax.
 * FEATURE: [`copy` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#copy-pipe): allow copying all the fields with common prefix to fields with another common prefix via `... | copy old_prefix* as new_prefix*` syntax.
@@ -25,10 +30,10 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 * FEATURE: [`unpack_json` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_json-pipe): allow unpacking JSON fields with common prefix via `... fields (prefix*)` syntax.
 * FEATURE: [`unpack_logfmt` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_logfmt-pipe): allow unpacking JSON fields with common prefix via `... fields (prefix*)` syntax.
 * FEATURE: [`avg` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the average value over all the fields with common prefix via `avg(prefix*)` syntax.
-* FEATURE: [`max` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the maximum value over all the fields with common prefix via `max(prefix*)` syntax.
-* FEATURE: [`min` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the minimum value over all the fields with common prefix via `min(prefix*)` syntax.
-* FEATURE: [`median` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the median value over all the fields with common prefix via `median(prefix*)` syntax.
-* FEATURE: [`quantile` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the maximum value over all the fields with common prefix via `quantile(prefix*)` syntax.
+* FEATURE: [`max` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#max-stats): allow calculating the maximum value over all the fields with common prefix via `max(prefix*)` syntax.
+* FEATURE: [`min` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#min-stats): allow calculating the minimum value over all the fields with common prefix via `min(prefix*)` syntax.
+* FEATURE: [`median` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#median-stats): allow calculating the median value over all the fields with common prefix via `median(prefix*)` syntax.
+* FEATURE: [`quantile` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#quantile-stats): allow calculating the maximum value over all the fields with common prefix via `quantile(prefix*)` syntax.
 * FEATURE: [`sum` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#sum-stats): allow calculating the sum for all the fields with common prefix via `sum(prefix*)` syntax.
 * FEATURE: [`sum_len` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#sum_len-stats): allow calculating the sum of byte lengths for all the fields with common prefix via `sum_len(prefix*)` syntax.
 * FEATURE: [`count` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#count-stats): allow calculating the number of logs with at least a single non-empty field across fields with common prefix via `count(prefix*)` syntax.
@@ -53,6 +58,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): live tailing tab automatically reconnects when the connection is lost. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9129).
 * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix issue with hits chart ignoring selected AccountID and ProjectID. See [#9157](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9157).
 * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix missing field values in auto-complete. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8749)
+* BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): remove the compact mode of the table tab and add field sorting capabilities to the JSON tab. See [#7047](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7047).
 * BUGFIX: [Journald data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/): properly read log timestamp from `__REALTIME_TIMESTAMP` field according to [the docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field). See [#9144](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9144). The bug has been introduced in [v1.22.0-victorialogs](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.22.0-victorialogs).
 * BUGFIX: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): support `-` as a timestamp value, as described in [RFC5424](https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3).
 * BUGFIX: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): properly handle quotes inside quoted strings such as `"\""`. Previously this could lead to panics. See [#9219](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9219).

docs/victorialogs/QuickStart.md

@@ -36,8 +36,8 @@ Just download archive for the needed Operating system and architecture, unpack i
 For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
 
 ```sh
-curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.23.3-victorialogs/victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
-tar xzf victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
+curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.24.0-victorialogs/victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
+tar xzf victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
 ./victoria-logs-prod -storageDataPath=victoria-logs-data
 ```
 
@@ -61,7 +61,7 @@ Here is the command to run VictoriaLogs in a Docker container:
 
 ```sh
 docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
-  docker.io/victoriametrics/victoria-logs:v1.23.3-victorialogs -storageDataPath=victoria-logs-data
+  docker.io/victoriametrics/victoria-logs:v1.24.0-victorialogs -storageDataPath=victoria-logs-data
 ```
 
 See also:

docs/victorialogs/README.md

@@ -400,6 +400,8 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line
     	Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
     	Disable CORS for all origins (*)
+  -http.disableKeepAlive
+    	Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
     	Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victorialogs/cluster.md

@@ -155,8 +155,8 @@ The following guide covers the following topics for Linux host:
 Download and unpack the latest VictoriaLogs release:
 
 ```sh
-curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.23.3-victorialogs/victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
-tar xzf victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
+curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.24.0-victorialogs/victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
+tar xzf victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
 ```
 
 Start the first [`vlstorage` node](#architecture), which accepts incoming requests at the port `9491` and stores the ingested logs at `victoria-logs-data-1` directory:

docs/victorialogs/data-ingestion/Journald.md

@@ -36,7 +36,14 @@ VictoriaLogs uses `(_MACHINE_ID, _HOSTNAME, _SYSTEMD_UNIT)` as [stream fields](h
 for logs ingested via jorunald protocol. The list of log stream fields can be changed via `-journald.streamFields` command-line flag if needed,
 by providing comma-separated list of journald fields form [this list](https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html).
 
-See [the list of supported Journald fields](https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html).
+Please make sure that the log stream fields passed to `-journlad.streamFields` do not contain fields with high number or unbound number of unique values,
+since this may lead to [high cardinality issues](https://docs.victoriametrics.com/victorialogs/keyconcepts/#high-cardinality).
+
+The following Journald fields are also good candidates for stream fields:
+
+- `_TRANSPORT`
+- `_SYSTEMD_USER_UNIT`
+
 
 ## Dropping fields
 

docs/victorialogs/querying/vlogscli.md

@@ -25,8 +25,8 @@ or from docker images at [Docker Hub](https://hub.docker.com/r/victoriametrics/v
 ### Running `vlogscli` from release binary
 
 ```sh
-curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.23.3-victorialogs/vlogscli-linux-amd64-v1.23.3-victorialogs.tar.gz
-tar xzf vlogscli-linux-amd64-v1.23.3-victorialogs.tar.gz
+curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.24.0-victorialogs/vlogscli-linux-amd64-v1.24.0-victorialogs.tar.gz
+tar xzf vlogscli-linux-amd64-v1.24.0-victorialogs.tar.gz
 ./vlogscli-prod
 ```
 
@@ -48,7 +48,6 @@ which queries `(AccountID=123, ProjectID=456)` [tenant](https://docs.victoriamet
 ./vlogscli -header='AccountID: 123' -header='ProjectID: 456'
 ```
 
-
 ## Multitenancy
 
 `AccountID` and `ProjectID` [values](https://docs.victoriametrics.com/victorialogs/#multitenancy)

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -1164,6 +1164,8 @@ Below is the output for `/path/to/vminsert -help`:
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string
@@ -1470,6 +1472,8 @@ Below is the output for `/path/to/vmselect -help`:
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string
@@ -1811,6 +1815,8 @@ Below is the output for `/path/to/vmstorage -help`:
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriametrics/README.md

@@ -2459,6 +2459,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriametrics/changelog/CHANGELOG.md

@@ -18,13 +18,21 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+## [v1.120.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.120.0)
+
+Released at 2025-06-20
+
 * SECURITY: upgrade Go builder from Go1.24.3 to Go1.24.4. See [the list of issues addressed in Go1.24.4](https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved).
 * SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
 
+* FEATURE: all the VictoriaMetrics components: add `-http.disableKeepAlive` to disable HTTP keep-alives for incoming connections. The flag could improve load balancing among replicas behind HTTP load balancers. See [#9125](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9125) and [#2395](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2395) for details.
 * FEATURE: [dashboards/cluster](https://grafana.com/grafana/dashboards/11176): add panel `Partitions scheduled for re-processing` to `Troubleshooting` row. It shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters). The new panel should help to correlate resource usage with background re-processing of partitions. 
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support [rate_prometheus](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate_prometheus) function, an equivalent to `increase_prometheus(series_selector[d]) / d`. See [#8901](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8901) and [#8891](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8891) for details.
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): respect staleness markers when calculating `rate` and `increase` functions. The new behavior will interrupt rate/increase calculation if last sample on the selected time window is a [staleness marker](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers), making the series to disappear immediately instead of slowly fading away. See more details in [#8891-comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8891#issuecomment-2875542721).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): do not break vmalert process under replay mode when rule uses `query` template, but only logging a warning.
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): correct the rule evaluation timestamp if the system clock is changed during runtime. See [#8790](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8790).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): respect [group](https://docs.victoriametrics.com/victoriametrics/vmalert/#groups) `concurrency` setting in [replay mode](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling) when `-replay.rulesDelay=0`. See this [#7387](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7387) for details. Thanks to @BenNF for the [PR #9214](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9214).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): allow overriding default limits for in-memory cache `storage/metricName` via flag `-storage.cacheSizeStorageMetricName`.
 
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect sorting of tag filters, which led to suboptimal tag filter evaluation order and potentially degraded query performance in rare cases. See [#9127](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9127) for details.
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix server-side copying of objects for Azure Blob Storage when using managed identity for authentication. Previously, it wasn't possible to use [smart backups](https://docs.victoriametrics.com/victoriametrics/vmbackup/#smart-backups) strategy for `vmbackup` as server-side copy would fail. See [#9131](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9131).

docs/victoriametrics/vmagent.md

@@ -1552,6 +1552,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriametrics/vmalert.md

@@ -817,9 +817,11 @@ max range per request:  8h20m0s
 2021-06-07T09:59:12.098Z        info    app/vmalert/replay.go:68        replay finished! Imported 511734 samples
 ```
 
-In `replay` mode all groups are executed sequentially one-by-one. Rules within the group are
-executed sequentially as well (`concurrency` setting is ignored). vmalert sends rule's expression
-to [/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) endpoint
+> In replay mode, groups are executed one after another in sequence. Within each group, rules are also executed sequentially, 
+regardless of the `concurrency` setting. This ensures that any potential chaining between rules is preserved (see `-replay.rulesDelay`).
+If you want rules to run concurrently based on the `concurrency` setting, set `-replay.rulesDelay=0`.
+
+vmalert sends rule's expression to [/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) endpoint
 of the configured `-datasource.url`. Returned data is then processed according to the rule type and
 backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
 vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
@@ -853,7 +855,8 @@ There are following non-required `replay` flags:
 * `-replay.rulesDelay` - delay between sequential rules execution. Important in cases if there are chaining
   (rules which depend on each other) rules. It is expected, that remote storage will be able to persist
   previously accepted data during the delay, so data will be available for the subsequent queries.
-  Keep it equal or bigger than `-remoteWrite.flushInterval`.
+  Keep it equal or bigger than `-remoteWrite.flushInterval`. When set to `0`, allows executing rules within
+  the group concurrently.
 * `-replay.disableProgressBar` - whether to disable progress bar which shows progress work.
   Progress bar may generate a lot of log records, which is not formatted as standard VictoriaMetrics logger.
   It could break logs parsing by external system and generate additional load on it.
@@ -1190,6 +1193,8 @@ The shortlist of configuration flags is the following:
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string
@@ -1492,7 +1497,7 @@ The shortlist of configuration flags is the following:
   -replay.ruleRetryAttempts int
      Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
   -replay.rulesDelay duration
-     Delay between rules evaluation within the group. Could be important if there are chained rules inside the group and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule.Keep it equal or bigger than -remoteWrite.flushInterval. (default 1s)
+     Delay before evaluating the next rule within the group. Is important for chained rules. Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting. (default 1s)
   -replay.timeFrom string
      The time filter in RFC3339 format to start the replay from. E.g. '2020-01-01T20:07:00Z'
   -replay.timeTo string

docs/victoriametrics/vmauth.md

@@ -1271,6 +1271,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriametrics/vmbackup.md

@@ -366,6 +366,8 @@ Run `vmbackup -help` in order to see all the available options:
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriametrics/vmbackupmanager.md

@@ -493,6 +493,8 @@ command-line flags:
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriametrics/vmgateway.md

@@ -377,6 +377,8 @@ Below is the list of configuration flags (it can be viewed by running `./vmgatew
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriametrics/vmrestore.md

@@ -90,6 +90,8 @@ Run `vmrestore -help` in order to see all the available options:
      Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
   -http.disableCORS
      Disable CORS for all origins (*)
+  -http.disableKeepAlive
+     Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
   -http.disableResponseCompression
      Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
   -http.header.csp string

docs/victoriatraces/README.md

@@ -0,0 +1,153 @@
+> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
+
+VictoriaTraces is an open-source, user-friendly database designed for storing and querying distributed [tracing data](https://en.wikipedia.org/wiki/Tracing_(software)), 
+built by the [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) team.
+
+VictoriaTraces provides the following features:
+- It is resource-efficient and fast. It uses up to [**3.7x less RAM and up to 2.6x less CPU**](https://victoriametrics.com/blog/dev-note-distributed-tracing-with-victorialogs/) than other solutions such as Grafana Tempo.
+- VictoriaTraces' capacity and performance scales linearly with the available resources (CPU, RAM, disk IO, disk space).
+- It accepts trace spans in the popular [OpenTelemetry protocol](https://opentelemetry.io/docs/specs/otel/protocol/)(OTLP).
+- It provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json) 
+  to integrate with [Grafana](https://grafana.com/docs/grafana/latest/datasources/jaeger/) or [Jaeger Frontend](https://www.jaegertracing.io/docs/2.6/frontend-ui/).
+
+## Quick Start
+
+The easiest way to get started with VictoriaTraces is by using the pre-built Docker Compose file.
+It launches VictoriaTraces, Grafana, and HotROD (a sample application that generates tracing data).
+Everything is preconfigured and connected out of the box, so you can start exploring distributed tracing within minutes.
+
+Clone the repository:
+```bash 
+git clone -b victoriatraces --single-branch https://github.com/VictoriaMetrics/VictoriaMetrics.git;
+cd  VictoriaMetrics;
+```
+
+Run VictoriaTraces with Docker Compose:
+```bash
+make docker-vt-single-up;
+```
+
+Now you can open HotROD at [http://localhost:8080](http://localhost:8080) and click around to generate some traces.
+Then, you can open Grafana at [http://localhost:3000/explore](http://localhost:3000/explore) and explore the traces using the Jaeger data source.
+
+To stop the services, run:
+```bash
+make docker-vt-single-down;
+```
+
+You can read more about docker compose and what's available there in the [Docker compose environment for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/deployment/victoriatraces/deployment/docker/README.md#victoriatraces-server).
+
+### How to build from sources
+
+Building from sources is reasonable when developing additional features specific to your needs or when testing bugfixes.
+
+{{% collapse name="How to build from sources" %}}
+
+Clone VictoriaMetrics repository: 
+```bash 
+git clone -b victoriatraces --single-branch https://github.com/VictoriaMetrics/VictoriaMetrics.git;
+cd  VictoriaMetrics;
+```
+
+#### Build binary with go build
+
+1. [Install Go](https://golang.org/doc/install).
+1. Run `make victoria-traces` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces).
+   It builds `victoria-traces` binary and puts it into the `bin` folder.
+
+#### Build binary with Docker
+
+1. [Install docker](https://docs.docker.com/install/).
+1. Run `make victoria-traces-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces).
+   It builds `victoria-traces-prod` binary and puts it into the `bin` folder.
+
+#### Building docker images
+
+Run `make package-victoria-traces`. It builds `victoriametrics/victoria-traces:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-victoria-traces`.
+
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable.
+For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
+
+```sh
+ROOT_IMAGE=scratch make package-victoria-traces
+```
+
+{{% /collapse %}}
+
+### Configure and run VictoriaTraces
+
+VictoriaTraces can be run with:
+```shell
+/path/to/victoria-traces -storageDataPath=victoria-traces-data -retentionPeriod=7d
+```
+
+or with Docker:
+```shell
+docker run --rm -it -p 9428:9428 -v ./victoria-traces-data:/victoria-traces-data \
+  docker.io/victoriametrics/victoria-traces:latest -storageDataPath=victoria-traces-data
+```
+
+VictoriaTraces is configured via command-line flags. 
+All the command-line flags have sane defaults, so there is no need in tuning them in general case. 
+VictoriaTraces runs smoothly in most environments without additional configuration.
+
+Pass `-help` to VictoriaTraces in order to see the list of supported command-line flags with their description and default values:
+
+```bash
+/path/to/victoria-traces -help
+```
+
+The following command-line flags are used the most:
+
+* `-storageDataPath` - VictoriaTraces stores all the data in this directory. The default path is `victoria-traces-data` in the current working directory.
+* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 7 days.
+
+Once it's running, it will listen to port `9428` (`-httpListenAddr`) and provide the following APIs:
+1. for ingestion:
+```
+http://<victoria-traces>:9428/insert/opentelemetry/v1/traces
+```
+2. for querying:
+```
+http://<victoria-traces>:9428/select/jaeger/<endpoints>
+```
+
+See [data ingestion](https://docs.victoriametrics.com/victoriatraces/data-ingestion/) and [querying](https://docs.victoriametrics.com/VictoriaTraces/querying/) for more details.
+
+## How does it work
+
+VictoriaTraces was initially built on top of [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/), a log database.
+It receives trace spans in OTLP format, transforms them into structured logs, and provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json) for querying.
+
+For detailed data model and example, see: [Key Concepts](https://docs.victoriametrics.com/victoriatraces/keyConcepts).
+
+![How does VictoriaTraces work](how-does-it-work.webp)
+
+Building VictoriaTraces in this way enables it to scale easily and linearly with the available resources, like VictoriaLogs.
+
+## List of command-line flags
+
+```shell
+  -search.traceMaxDurationWindow
+    	The lookbehind/lookahead window of searching for the rest trace spans after finding one span.
+        It allows extending the search start time and end time by `-search.traceMaxDurationWindow` to make sure all spans are included.
+        It affects both Jaeger's `/api/traces` and `/api/traces/<trace_id>` APIs. (default: 10m)
+  -search.traceMaxServiceNameList
+        The maximum number of service name can return in a get service name request.
+        This limit affects Jaeger's `/api/services` API. (default: 1000)
+  -search.traceMaxSpanNameList
+        The maximum number of span name can return in a get span name request.
+        This limit affects Jaeger's `/api/services/*/operations` API. (default: 1000)
+  -search.traceSearchStep
+        Splits the [0, now] time range into many small time ranges by -search.traceSearchStep
+        when searching for spans by `trace_id`. Once it finds spans in a time range, it performs an additional search according to `-search.traceMaxDurationWindow` and then stops.
+        It affects Jaeger's `/api/traces/<trace_id>` API.  (default: 1d)
+  -search.traceServiceAndSpanNameLookbehind
+        The time range of searching for service names and span names. 
+        It affects Jaeger's `/api/services` and `/api/services/*/operations` APIs. (default: 7d)
+```
+
+See also: [VictoriaLogs - List of Command-line flags](https://docs.victoriametrics.com/victorialogs/#list-of-command-line-flags)

docs/victoriatraces/_index.md

@@ -0,0 +1,11 @@
+---
+title: VictoriaTraces
+menu:
+  docs:
+    weight: 17
+    identifier: victoriatraces
+    pageRef: "/"
+tags:
+  - traces
+---
+{{% content "README.md" %}}

docs/victoriatraces/data-ingestion/README.md

@@ -0,0 +1,90 @@
+> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
+
+[VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) can accept trace spans via [the OpenTelemetry protocol (OTLP)](https://opentelemetry.io/docs/specs/otlp/).
+
+## HTTP APIs
+
+### Opentelemetry API
+
+VictoriaTraces provides the following API for OpenTelemetry data ingestion:
+
+- `/insert/opentelemetry/v1/traces`
+
+See more details [in this docs](https://docs.victoriametrics.com/victoriatraces/data-ingestion/opentelemetry/).
+
+### HTTP parameters
+
+VictoriaTraces accepts optional HTTP parameters at data ingestion HTTP API via [HTTP query string parameters](https://en.wikipedia.org/wiki/Query_string), or via [HTTP headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields).
+
+HTTP query string parameters have priority over HTTP Headers.
+
+#### HTTP Query string parameters
+
+All the [HTTP-based data ingestion protocols](#http-apis) support the following [HTTP query string](https://en.wikipedia.org/wiki/Query_string) args:
+
+- `extra_fields` - an optional comma-separated list of [trace fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model),
+  which must be added to all the ingested traces. The format of every `extra_fields` entry is `field_name=field_value`.
+  If the trace entry contains fields from the `extra_fields`, then they are overwritten by the values specified in `extra_fields`.
+
+- `debug` - if this arg is set to `1`, then the ingested traces aren't stored in VictoriaTraces. Instead,
+  the ingested data is traceged by VictoriaTraces, so it can be investigated later.
+
+See also [HTTP headers](#http-headers).
+
+#### HTTP headers
+
+All the [HTTP-based data ingestion protocols](#http-apis) support the following [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields)
+additionally to [HTTP query args](#http-query-string-parameters):
+
+- `AccountID` - accountID of the tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victoriatraces/#multitenancy) for details.
+
+- `ProjectID`- projectID of the tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victoriatraces/#multitenancy) for details.
+
+- `VL-Extra-Fields` - an optional comma-separated list of [trace fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model),
+  which must be added to all the ingested traces. The format of every `extra_fields` entry is `field_name=field_value`.
+  If the trace entry contains fields from the `extra_fields`, then they are overwritten by the values specified in `extra_fields`.
+
+- `VL-Debug` - if this parameter is set to `1`, then the ingested traces aren't stored in VictoriaTraces. Instead,
+  the ingested data is traceged by VictoriaTraces, so it can be investigated later.
+
+See also [HTTP Query string parameters](#http-query-string-parameters).
+
+## Troubleshooting
+
+The following command can be used for verifying whether the data is successfully ingested into VictoriaTraces:
+
+```sh
+curl http://<victoria-traces>:9428/select/logsql/query -d 'query=*' | head
+```
+
+This command selects all the data ingested into VictoriaTraces via [HTTP query API](https://docs.victoriametrics.com/victoriatraces/querying/#http-api)
+using [any value filter](https://docs.victoriametrics.com/victorialogs/logsql/#any-value-filter),
+while `head` cancels query execution after reading the first 10 trace spans. See [these docs](https://docs.victoriametrics.com/victoriatraces/querying/#command-line)
+for more details on how `head` integrates with VictoriaTraces.
+
+The response by default contains all the [trace span fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model).
+See [how to query specific fields](https://docs.victoriametrics.com/victoriatraces/logsql/#querying-specific-fields).
+
+VictoriaTraces provides the following command-line flags, which can help debugging data ingestion issues:
+
+- `-logNewStreams` - if this flag is passed to VictoriaTraces, then it traces all the newly
+  registered [streams](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields).
+  This may help debugging [high cardinality issues](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#high-cardinality).
+- `-logIngestedRows` - if this flag is passed to VictoriaTraces, then it traces all the ingested
+  [trace span entries](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model).
+  See also `debug` [parameter](#http-parameters).
+
+VictoriaTraces exposes various [metrics](https://docs.victoriametrics.com/victoriatraces/#monitoring), which may help debugging data ingestion issues:
+
+- `vl_rows_ingested_total` - the number of ingested [trace span entries](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model)
+  since the last VictoriaTraces restart. If this number increases over time, then trace spans are successfully ingested into VictoriaTraces.
+  The ingested trace spans can be inspected in the following ways:
+    - By passing `debug=1` parameter to every request to [data ingestion APIs](#http-apis). The ingested spans aren't stored in VictoriaTraces
+      in this case. Instead, they are logged, so they can be investigated later.
+      The `vl_rows_dropped_total` [metric](https://docs.victoriametrics.com/victoriatraces/#monitoring) is incremented for each logged row.
+    - By passing `-logIngestedRows` command-line flag to VictoriaTraces. In this case it traces all the ingested data, so it can be investigated later.
+- `vl_streams_created_total` - the number of created [trace streams](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields)
+  since the last VictoriaTraces restart. If this metric grows rapidly during extended periods of time, then this may lead
+  to [high cardinality issues](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#high-cardinality).
+  The newly created trace streams can be inspected in traces by passing `-logNewStreams` command-line flag to VictoriaTraces.
+

docs/victoriatraces/data-ingestion/_index.md

@@ -0,0 +1,15 @@
+---
+title: Data ingestion
+weight: 3
+menu:
+  docs:
+    identifier: victoriatraces-data-ingestion
+    parent: "victoriatraces"
+    weight: 3
+tags:
+  - traces
+aliases:
+  - /victoriatraces/data-ingestion/
+  - /victoriatraces/data-ingestion/index.html
+---
+{{% content "README.md" %}}

docs/victoriatraces/data-ingestion/opentelemetry.md

@@ -0,0 +1,82 @@
+---
+weight: 4
+title: OpenTelemetry setup
+disableToc: true
+menu:
+  docs:
+    parent: "victoriatraces-data-ingestion"
+    weight: 4
+tags:
+  - traces
+aliases:
+  - /victoriatraces/data-ingestion/OpenTelemetry.html
+---
+
+> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
+
+VictoriaTraces supports both client open-telemetry [SDK](https://opentelemetry.io/docs/languages/) and [collector](https://opentelemetry.io/docs/collector/).
+
+## Client SDK
+
+The OpenTelemetry provides detailed document and examples for various programming languages:
+- [C++](https://opentelemetry.io/docs/languages/cpp/)
+- [C#/.NET](https://opentelemetry.io/docs/languages/dotnet/)
+- [Erlang/Elixir](https://opentelemetry.io/docs/languages/erlang/)
+- [Go](https://opentelemetry.io/docs/languages/go/)
+- [Java](https://opentelemetry.io/docs/languages/java/)
+- [JavaScript](https://opentelemetry.io/docs/languages/js/)
+- [PHP](https://opentelemetry.io/docs/languages/php/)
+- [Python](https://opentelemetry.io/docs/languages/python/)
+- [Ruby](https://opentelemetry.io/docs/languages/ruby/)
+- [Rust](https://opentelemetry.io/docs/languages/rust/)
+- [Swift](https://opentelemetry.io/docs/languages/swift/)
+
+To send data to VictoriaTraces, specify the `EndpointURL` for http-exporter builder to `http://<victoria-traces>:9428/insert/opentelemetry/v1/traces`.
+
+Consider the following example for Go SDK:
+
+```go
+traceExporter, err := otlptracehttp.New(ctx,
+	otlptracehttp.WithEndpointURL("http://<victoria-traces>:9428/insert/opentelemetry/v1/traces"),
+)
+```
+
+VictoriaTraces automatically use `service.name` in **resource attributes** and `name` in **span** as [stream fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields).
+While the remaining data (including [resource](https://opentelemetry.io/docs/specs/otel/overview/#resources), [instrumentation scope](https://opentelemetry.io/docs/specs/otel/common/instrumentation-scope/), and fields in [span](https://opentelemetry.io/docs/specs/otel/trace/api/#span), like `trace_id`, `span_id`, span `attributes` and more) are stored as [regular fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model):
+
+VictoriaTraces supports other HTTP headers - see the list [here](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers).
+
+The ingested trace spans can be queried according to [these docs](https://docs.victoriametrics.com/victoriatraces/querying/).
+
+## Collector configuration
+
+VictoriaTraces supports receiving traces from the following OpenTelemetry collector:
+
+* [OpenTelemetry](#opentelemetry)
+
+### OpenTelemetry
+
+To send the collected traces to VictoriaTraces, specify traces endpoint for [OTLP/HTTP exporter](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/otlphttpexporter/README.md) in configuration file:
+
+```yaml
+exporters:
+  otlphttp:
+    traces_endpoint: http://<victoria-traces>:9428/insert/opentelemetry/v1/traces
+```
+
+VictoriaTraces supports various HTTP headers, which can be used during data ingestion - see the list [here](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers).
+These headers can be passed to OpenTelemetry exporter config via `headers` options. For example, the following configs add (or overwrites) `foo: bar` field to each trace span during data ingestion:
+
+```yaml
+exporters:
+  otlphttp:
+    traces_endpoint: http://<victoria-traces>:9428/insert/opentelemetry/v1/traces
+    headers:
+      VL-Extra-Fields: foo=bar
+```
+
+See also:
+
+* [Data ingestion troubleshooting](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#troubleshooting).
+* [How to query VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/querying/).
+* [Docker-compose demo for OpenTelemetry collector integration with VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victoriatraces/opentelemetry-collector).

docs/victoriatraces/keyConcepts.md

@@ -0,0 +1,234 @@
+---
+weight: 2
+title: Key concepts
+menu:
+  docs:
+    identifier: vt-key-concepts
+    parent: victoriatraces
+    weight: 2
+    title: Key concepts
+tags:
+  - traces
+aliases:
+- /victoriatraces/keyConcepts.html
+---
+
+> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
+
+## Data model
+
+VictoriaTraces is built on VictoriaLogs. It's recommended to go through [the data model of VictoriaLogs](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) first.
+
+**Every [trace span](https://github.com/open-telemetry/opentelemetry-proto/blob/v1.7.0/opentelemetry/proto/trace/v1/trace.proto) must contain `service.name` in its resource attributes and a span `name`**. They will be used as the [stream fields](#stream-fields). All other data of the span will be mapped as the ordinary fields.
+
+For example, here's how a trace span looks like in an OTLP export request:
+
+{{% collapse name="OTLP trace span" %}}
+
+```json
+{
+  "ResourceSpans": [{
+    "Resource": {
+      "Attributes": {
+        "service.name": "payment",
+        "telemetry.sdk.language": "nodejs",
+        "telemetry.sdk.name": "opentelemetry",
+        "telemetry.sdk.version": "1.30.1",
+        "container.id": "18ee03279d38ed0e0eedad037c260df78dfc3323aa662ca14a2d38fcc8bf3762",
+        "service.namespace": "opentelemetry-demo",
+        "service.version": "2.0.2",
+        "host.name": "18ee03279d38",
+        "host.arch": "arm64",
+        "os.type": "linux",
+        "os.version": "6.10.14-linuxkit",
+        "process.pid": 17,
+        "process.executable.name": "node",
+        "process.executable.path": "/usr/local/bin/node",
+        "process.command_args": [
+          "/usr/local/bin/node",
+          "--require",
+          "./opentelemetry.js",
+          "/usr/src/app/index.js"
+        ],
+        "process.runtime.version": "22.16.0",
+        "process.runtime.name": "nodejs",
+        "process.runtime.description": "Node.js",
+        "process.command": "/usr/src/app/index.js",
+        "process.owner": "node"
+      }
+    },
+    "ScopeSpans": [{
+      "Scope": {
+        "Name": "@opentelemetry/instrumentation-net",
+        "Version": "0.43.1",
+        "Attributes": null,
+        "DroppedAttributesCount": 0
+      },
+      "Spans": [{
+        "TraceID": "769d28c4b8633dc9de2cc421d1a1616f",
+        "SpanID": "2a1f3c4bda1d0e43",
+        "TraceState": "",
+        "ParentSpanID": "3ea61f2d2a5a003d",
+        "Flags": 0,
+        "Name": "tcp.connect",
+        "Kind": 1,
+        "StartTimeUnixNano": 1750044408780000000,
+        "EndTimeUnixNano": 1750044408796828584,
+        "Attributes": {
+          "net.transport": "ip_tcp",
+          "net.peer.name": "169.254.169.254",
+          "net.peer.port": 80,
+          "net.peer.ip": "169.254.169.254",
+          "net.host.ip": "172.18.0.14",
+          "net.host.port": 50722
+        },
+        "DroppedAttributesCount": 0,
+        "Events": null,
+        "DroppedEventsCount": 0,
+        "Links": null,
+        "DroppedLinksCount": 0,
+        "Status": {
+          "Message": "",
+          "Code": 0
+        }
+      }],
+      "SchemaURL": ""
+    }],
+    "SchemaURL": ""
+  }]
+}
+```
+
+{{% /collapse %}}
+
+And here's how this trace span looks like in VictoriaTraces:
+
+{{% collapse name="span in VictoriaTraces" %}}
+
+```json
+  {
+  "_time": "2025-06-16T03:26:48.796828584Z",
+  "_stream_id": "00000000000000006fc12c07dedb6f693e73fc7b11b943e6",
+  "_stream": "{name=\"tcp.connect\",resource_attr:service.name=\"payment\"}",
+  "_msg": "-",
+  "dropped_attributes_count": "0",
+  "dropped_events_count": "0",
+  "dropped_links_count": "0",
+  "flags": "0",
+  "kind": "1",
+  "name": "tcp.connect",
+  "resource_attr:container.id": "18ee03279d38ed0e0eedad037c260df78dfc3323aa662ca14a2d38fcc8bf3762",
+  "resource_attr:host.arch": "arm64",
+  "resource_attr:host.name": "18ee03279d38",
+  "resource_attr:os.type": "linux",
+  "resource_attr:os.version": "6.10.14-linuxkit",
+  "resource_attr:process.command": "/usr/src/app/index.js",
+  "resource_attr:process.command_args": "[\"/usr/local/bin/node\",\"--require\",\"./opentelemetry.js\",\"/usr/src/app/index.js\"]",
+  "resource_attr:process.executable.name": "node",
+  "resource_attr:process.executable.path": "/usr/local/bin/node",
+  "resource_attr:process.owner": "node",
+  "resource_attr:process.pid": "17",
+  "resource_attr:process.runtime.description": "Node.js",
+  "resource_attr:process.runtime.name": "nodejs",
+  "resource_attr:process.runtime.version": "22.16.0",
+  "resource_attr:service.name": "payment",
+  "resource_attr:service.namespace": "opentelemetry-demo",
+  "resource_attr:service.version": "2.0.2",
+  "resource_attr:telemetry.sdk.language": "nodejs",
+  "resource_attr:telemetry.sdk.name": "opentelemetry",
+  "resource_attr:telemetry.sdk.version": "1.30.1",
+  "scope_name": "@opentelemetry/instrumentation-net",
+  "scope_version": "0.43.1",
+  "span_attr:net.transport": "ip_tcp",
+  "duration": "16828584",
+  "end_time_unix_nano": "1750044408796828584",
+  "parent_span_id": "3ea61f2d2a5a003d",
+  "span_attr:net.host.ip": "172.18.0.14",
+  "span_attr:net.host.port": "50722",
+  "span_attr:net.peer.ip": "169.254.169.254",
+  "span_attr:net.peer.name": "169.254.169.254",
+  "span_attr:net.peer.port": "80",
+  "span_id": "2a1f3c4bda1d0e43",
+  "start_time_unix_nano": "1750044408780000000",
+  "status_code": "0",
+  "trace_id": "769d28c4b8633dc9de2cc421d1a1616f"
+}
+```
+
+{{% /collapse %}}
+
+### Special mappings
+
+There are some special mappings when transforming a trace span into VictoriaTraces data model:
+1. Empty attribute values in trace spans are replaced with `-`.
+2. Resource, scope and span attributes are stored with corresponding prefixes `resource_attr`, `scope_attr` and `span_attr:` accordingly. 
+3. For some attributes within a list (event list, link list in span), a corresponding prefix and index (such as `event:0:` and `event:0:event_attr:`) is added.
+4. The `duration` field does not exist in the OTLP request, but for query efficiency, it's calculated during ingestion and stored as a separated field.
+
+VictoriaTraces automatically indexes all the fields for ingested trace spans.
+This enables [full-text search](https://docs.victoriametrics.com/victorialogs/logsql/) across all the fields.
+
+VictoriaTraces stores data in different fields. There are some special fields in addition to [arbitrary fields](#other-fields):
+
+* [`_time` field](#time-field)
+* [`_stream` and `_stream_id` fields](#stream-fields)
+
+### Time field
+
+The ingested [trace spans](#data-model) may contain `_time` field with the timestamp of the ingested trace span.
+
+By default, VictoriaTraces use the `EndTimeUnixNano` in trace span as the `_time` field.
+
+The `_time` field is used by [time filter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter) for quickly narrowing down the search to the selected time range.
+
+### Stream fields
+
+As service name and span name usually identify the application instance, VictoriaTraces uses `service.name` in resource attributes and `name` in span
+as the stream fields.
+
+VictoriaTraces optimizes storing and [querying](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter) of individual trace span streams.
+This provides the following benefits:
+- Reduced disk space usage, since a trace span stream from a single application instance is usually compressed better
+  than a mixed trace span stream from multiple distinct applications.
+
+- Increased query performance, since VictoriaTraces needs to scan lower amounts of data
+  when [searching by stream fields](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter).
+
+Every ingested trace span is associated with a trace span stream. Every trace span stream consists of the following special fields:
+
+- `_stream_id` - this is an unique identifier for the trace span stream. All the trace spans for the particular stream can be selected
+  via [`_stream_id:...` filter](https://docs.victoriametrics.com/victorialogs/logsql/#_stream_id-filter).
+
+- `_stream` - this field contains stream labels in the format similar to [labels in Prometheus metrics](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels):
+  ```
+  {resource_attr:service_name="svc name", name="span name"}
+  ```
+  The `_stream` field can be searched with [stream filters](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter).
+
+#### High cardinality
+
+Some fields in the [trace spans](#data-model) may contain big number of unique values across log entries.
+For example, fields with names such as `trace_id`, `span_id` or `ip` tend to contain big number of unique values.
+VictoriaTraces works perfectly with such fields unless they are associated with [trace span streams](#stream-fields).
+
+**Never** associate high-cardinality fields with [trace span streams](#stream-fields), since this may lead to the following issues:
+
+- Performance degradation during [data ingestion](https://docs.victoriametrics.com/victoriatraces/data-ingestion/)
+  and [querying](https://docs.victoriametrics.com/victoriatraces/querying/)
+- Increased memory usage
+- Increased CPU usage
+- Increased disk space usage
+- Increased disk read / write IO
+
+VictoriaTraces exposes `vl_streams_created_total` [metric](https://docs.victoriametrics.com/victorialogs/#monitoring),
+which shows the number of created streams since the last VictoriaTraces restart. If this metric grows at a rapid rate
+during long period of time, then there are high chances of high cardinality issues mentioned above.
+VictoriaTraces can log all the newly registered streams when `-logNewStreams` command-line flag is passed to it.
+This can help narrowing down and eliminating high-cardinality fields from [trace span streams](#stream-fields).
+
+### Other fields
+
+Every ingested log entry may contain arbitrary number of [fields](#data-model) additionally to [`_time`](#time-field).
+For example, `name`, `span_attr:ip`, `span_attr:ip`, etc. Such fields can be used for simplifying and optimizing [search queries](https://docs.victoriametrics.com/victorialogs/logsql/).
+
+See [LogsQL docs](https://docs.victoriametrics.com/victorialogs/logsql/) for more details.

docs/victoriatraces/querying/_index.md

@@ -0,0 +1,16 @@
+---
+title: Querying
+weight: 4
+menu:
+  docs:
+    identifier: victoriatraces-querying
+    parent: "victoriatraces"
+    weight: 4
+tags:
+  - traces
+aliases:
+  - /VictoriaTraces/querying/
+  - /victoriatraces/querying/
+  - /victoriatraces/querying/index.html
+---
+{{% content "README.md" %}}

docs/victoriatraces/querying/grafana.md

@@ -0,0 +1,28 @@
+---
+weight: 4
+title: Visualization in Grafana
+disableToc: true
+menu:
+  docs:
+    parent: "victoriatraces-querying"
+    weight: 4
+tags:
+  - traces
+aliases:
+  - /victoriatraces/querying/grafana.html
+---
+
+> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
+
+[Grafana Jaeger Datasource](https://grafana.com/docs/grafana/latest/datasources/jaeger/) allows you to query and visualize VictoriaTraces data in Grafana.
+
+![Visualization with Grafana](grafana-jaeger.webp)
+
+Simply click "Add new data source" on Grafana, and then fill your VictoriaTraces URL to "Connection.URL". 
+
+The URL format for VictoriaTraces single-node is:
+```
+http://<victoria-traces>:9428/select/jaeger
+```
+
+Finally, click "Save & Test" at the bottom to complete the process.

docs/victoriatraces/querying/jaeger-frontend.md

@@ -0,0 +1,65 @@
+---
+weight: 4
+title: Visualization in Jaeger UI
+disableToc: true
+menu:
+  docs:
+    parent: "victoriatraces-querying"
+    weight: 4
+tags:
+  - traces
+aliases:
+  - /victoriatraces/querying/jaeger-frontend.html
+---
+
+> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
+
+[Jaeger UI](https://github.com/jaegertracing/jaeger-ui) is the official frontend that ships with Jaeger. It queries [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json)
+and visualizes the response trace data.
+
+## Deploy Jaeger UI
+
+You can get Jaeger UI from [release page](https://github.com/jaegertracing/jaeger-ui/releases/tag/v1.70.0). 
+
+As it provides only assets and source code, an HTTP server is needed for serving requests. 
+
+### Nginx Example
+
+Here's an example where we use Nginx to:
+- Serve static content of Jaeger UI.
+- Forward query requests to VictoriaTraces.
+
+Assume you already have:
+1. VictoriaTraces running locally and listening on port `:9428`.
+2. Jaeger UI assets (`index.html` and `/static`) located under `/path/to/jaeger-ui/build/`.
+3. Nginx Installed.
+
+Create the following config file `jaeger.conf` and place it to the Nginx config folder:
+
+```
+server {
+    listen       8080;
+    listen  [::]:8080;
+    server_name  localhost;
+
+    location / {
+        root   /path/to/jaeger-ui/build; # change this path to your asserts location.
+        try_files $uri $uri/ /index.html;
+    }
+
+    location /api {
+        proxy_pass http://127.0.0.1:9428/select/jaeger/api; # change this address to VictoriaTraces' address.
+    }
+}
+```
+
+Here are some common paths of Nginx config folder:
+```sh
+# Ubuntu & Install with apt
+cd /etc/nginx/sites-available/
+
+# MacOS & Install with homebrew
+cd /opt/homebrew/etc/nginx/servers/
+```
+
+After reloading Nginx, you should be able to visit Jaeger UI on: [http://127.0.0.1:8080/](http://127.0.0.1:8080/).

lib/fasttime/fasttime.go

@@ -1,42 +1,5 @@
 package fasttime
 
-import (
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
-)
-
-func init() {
-	go func() {
-		ticker := time.NewTicker(time.Second)
-		defer ticker.Stop()
-		for tm := range ticker.C {
-			t := uint64(tm.Unix())
-			currentTimestamp.Store(t)
-		}
-	}()
-}
-
-var currentTimestamp = func() *atomicutil.Uint64 {
-	var x atomicutil.Uint64
-	x.Store(uint64(time.Now().Unix()))
-	return &x
-}()
-
-// UnixTimestamp returns the current unix timestamp in seconds.
-//
-// It is faster than time.Now().Unix()
-func UnixTimestamp() uint64 {
-	if testing.Testing() {
-		// When executing inside the tests, use the time package directly.
-		// This allows to override time using synctest package.
-		return uint64(time.Now().Unix())
-	}
-
-	return currentTimestamp.Load()
-}
-
 // UnixDate returns date from the current unix timestamp.
 //
 // The date is calculated by dividing unix timestamp by (24*3600)

lib/fasttime/fasttime_normal.go

@@ -0,0 +1,33 @@
+//go:build !goexperiment.synctest
+
+package fasttime
+
+import (
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
+)
+
+func init() {
+	go func() {
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for tm := range ticker.C {
+			t := uint64(tm.Unix())
+			currentTimestamp.Store(t)
+		}
+	}()
+}
+
+var currentTimestamp = func() *atomicutil.Uint64 {
+	var x atomicutil.Uint64
+	x.Store(uint64(time.Now().Unix()))
+	return &x
+}()
+
+// UnixTimestamp returns the current unix timestamp in seconds.
+//
+// It is faster than time.Now().Unix()
+func UnixTimestamp() uint64 {
+	return currentTimestamp.Load()
+}

lib/fasttime/fasttime_synctest.go

@@ -0,0 +1,13 @@
+//go:build goexperiment.synctest
+
+package fasttime
+
+import (
+	"time"
+)
+
+// UnixTimestamp returns the current unix timestamp in seconds.
+func UnixTimestamp() uint64 {
+	// Fall back to time.Now().Unix(), since this is needed for synctest.
+	return uint64(time.Now().Unix())
+}

lib/httpserver/httpserver.go

@@ -54,6 +54,7 @@ var (
 	flagsAuthKey     = flagutil.NewPassword("flagsAuthKey", "Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	pprofAuthKey     = flagutil.NewPassword("pprofAuthKey", "Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides -httpAuth.*")
 
+	disableKeepAlive            = flag.Bool("http.disableKeepAlive", false, "Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr")
 	disableResponseCompression  = flag.Bool("http.disableResponseCompression", false, "Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth")
 	maxGracefulShutdownDuration = flag.Duration("http.maxGracefulShutdownDuration", 7*time.Second, `The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown`)
 	shutdownDelay               = flag.Duration("http.shutdownDelay", 0, `Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers`)
@@ -165,6 +166,7 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler, disableB
 
 		ErrorLog: logger.StdErrorLogger(),
 	}
+	s.s.SetKeepAlivesEnabled(!*disableKeepAlive)
 	if *connTimeout > 0 {
 		s.s.ConnContext = func(ctx context.Context, _ net.Conn) context.Context {
 			timeoutSec := connTimeout.Seconds()

lib/logstorage/pipe_unpack_syslog_test.go

@@ -67,7 +67,7 @@ func TestPipeUnpackSyslog(t *testing.T) {
 	}, [][]Field{
 		{
 			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
-			{"level", "info"},
+			{"level", "notice"},
 			{"foo", "321"},
 			{"priority", "165"},
 			{"facility", "20"},
@@ -94,7 +94,7 @@ func TestPipeUnpackSyslog(t *testing.T) {
 	}, [][]Field{
 		{
 			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
-			{"level", "info"},
+			{"level", "notice"},
 			{"foo", "321"},
 			{"priority", "165"},
 			{"facility", "20"},
@@ -118,7 +118,7 @@ func TestPipeUnpackSyslog(t *testing.T) {
 	}, [][]Field{
 		{
 			{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
-			{"level", "info"},
+			{"level", "notice"},
 			{"priority", "165"},
 			{"facility", "20"},
 			{"facility_keyword", "local4"},
@@ -141,7 +141,7 @@ func TestPipeUnpackSyslog(t *testing.T) {
 	}, [][]Field{
 		{
 			{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
-			{"level", "info"},
+			{"level", "notice"},
 			{"priority", "165"},
 			{"facility", "20"},
 			{"facility_keyword", "local4"},
@@ -175,7 +175,7 @@ func TestPipeUnpackSyslog(t *testing.T) {
 	}, [][]Field{
 		{
 			{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
-			{"level", "info"},
+			{"level", "notice"},
 			{"priority", "165"},
 			{"facility", "20"},
 			{"facility_keyword", "local4"},
@@ -226,7 +226,7 @@ func TestPipeUnpackSyslog(t *testing.T) {
 	}, [][]Field{
 		{
 			{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
-			{"qwe_level", "info"},
+			{"qwe_level", "notice"},
 			{"qwe_priority", "165"},
 			{"qwe_facility", "20"},
 			{"qwe_facility_keyword", "local4"},

lib/logstorage/storage_search.go

@@ -64,7 +64,8 @@ type searchOptions struct {
 
 // WriteDataBlockFunc must process the db.
 //
-// WriteDataBlockFunc cannot hold references to db after returning.
+// WriteDataBlockFunc cannot hold references to db or any of its fields after the function returns.
+// If you need BlockColumn names or values after the function returns, copy them using strings.Clone.
 type WriteDataBlockFunc func(workerID uint, db *DataBlock)
 
 func (f WriteDataBlockFunc) newBlockResultWriter() writeBlockResultFunc {

lib/logstorage/syslog_parser.go

@@ -147,13 +147,19 @@ func syslogSeverityToLevel(severity uint64) string {
 	// See https://en.wikipedia.org/wiki/Syslog#Severity_level
 	// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
 	switch severity {
-	case 0, 1, 2:
+	case 0:
+		return "emerg"
+	case 1:
+		return "alert"
+	case 2:
 		return "critical"
 	case 3:
 		return "error"
 	case 4:
 		return "warning"
-	case 5, 6:
+	case 5:
+		return "notice"
+	case 6:
 		return "info"
 	case 7:
 		return "debug"

lib/logstorage/syslog_parser_test.go

@@ -24,7 +24,7 @@ func TestSyslogParser(t *testing.T) {
 	f("Jun  3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
 		`format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
 	f("<165>Jun  3 12:08:33 abcd systemd[1]: Starting Update the local ESM caches...", time.UTC,
-		`priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
+		`priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc3164 timestamp=2024-06-03T12:08:33.000Z hostname=abcd app_name=systemd proc_id=1 message="Starting Update the local ESM caches..."`)
 	f("Mar 13 12:08:33 abcd systemd: Starting Update the local ESM caches...", time.UTC,
 		`format=rfc3164 timestamp=2024-03-13T12:08:33.000Z hostname=abcd app_name=systemd message="Starting Update the local ESM caches..."`)
 	f("Jun  3 12:08:33 abcd - Starting Update the local ESM caches...", time.UTC,
@@ -35,13 +35,13 @@ func TestSyslogParser(t *testing.T) {
 	// RFC 5424
 	f(`<134>1 2024-12-09T18:25:35.401631+00:00 ps999 account-server - - [sd@51059 project="secret" ] 1.2.3.4 - - [09/Dec/2024:18:25:35 +0000] "PUT someurl" 201 - "-" "-" "container-updater 1283500" 0.0010 "-" 1531 0`, time.UTC, `priority=134 facility_keyword=local0 level=info facility=16 severity=6 format=rfc5424 timestamp=2024-12-09T18:25:35.401631+00:00 hostname=ps999 app_name=account-server proc_id=- msg_id=- sd@51059.project=secret message="1.2.3.4 - - [09/Dec/2024:18:25:35 +0000] \"PUT someurl\" 201 - \"-\" \"-\" \"container-updater 1283500\" 0.0010 \"-\" 1531 0"`)
 	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
-		`priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
+		`priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
 	f(`1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data.`, time.UTC,
 		`format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 message="This is a test message with structured data."`)
 	f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [exampleSDID@32473 iut="3" eventSource="Application 123 = ] 56" eventID="11211"] This is a test message with structured data.`, time.UTC,
-		`priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
+		`priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 exampleSDID@32473.iut=3 exampleSDID@32473.eventSource="Application 123 = ] 56" exampleSDID@32473.eventID=11211 message="This is a test message with structured data."`)
 	f(`<165>1 2023-06-03T17:42:00.000Z mymachine.example.com appname 12345 ID47 [foo@123 iut="3"][bar@456 eventID="11211"][abc=def][x=y z=a q="]= "] This is a test message with structured data.`, time.UTC,
-		`priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 abc=def x=y z=a q="]= " message="This is a test message with structured data."`)
+		`priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:00.000Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123.iut=3 bar@456.eventID=11211 abc=def x=y z=a q="]= " message="This is a test message with structured data."`)
 	f(`<14>1 2025-02-11T12:31:28+01:00 synology Connection - - [synolog@6574 event_id="0x0001" synotype="Connection" username="synouser" luser="synouser" event="User [synouser\] from [192.168.0.10\] logged in successfully via [SSH\]." arg_1="synouser" arg_2="1027" arg_3="192.168.0.10" arg_4="SSH"][meta sequenceId="7"] User [synouser] from [192.168.0.10] logged in successfully via [SSH].`, time.UTC,
 		`priority=14 facility_keyword=user level=info facility=1 severity=6 format=rfc5424 timestamp=2025-02-11T12:31:28+01:00 hostname=synology app_name=Connection proc_id=- msg_id=- synolog@6574.event_id=0x0001 synolog@6574.synotype=Connection synolog@6574.username=synouser synolog@6574.luser=synouser synolog@6574.event="User [synouser] from [192.168.0.10] logged in successfully via [SSH]." synolog@6574.arg_1=synouser synolog@6574.arg_2=1027 synolog@6574.arg_3=192.168.0.10 synolog@6574.arg_4=SSH meta.sequenceId=7 message="User [synouser] from [192.168.0.10] logged in successfully via [SSH]."`)
 	f(`<14>1 2025-02-18T11:37:42+02:00 localhost Test - - [test event="quote \"test\""] Test message`, time.UTC, `priority=14 facility_keyword=user level=info facility=1 severity=6 format=rfc5424 timestamp=2025-02-18T11:37:42+02:00 hostname=localhost app_name=Test proc_id=- msg_id=- test.event="quote \"test\"" message="Test message"`)
@@ -58,11 +58,11 @@ func TestSyslogParser(t *testing.T) {
 	f(`foo bar baz`, time.UTC, `format=rfc3164 message="foo bar baz"`)
 
 	// Incomplete RFC 5424
-	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC, `priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
-	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC, `priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
-	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC, `priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
-	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC, `priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
-	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC, `priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
-	f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC, `priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
-	f(`<165>1 `, time.UTC, `priority=165 facility_keyword=local4 level=info facility=20 severity=5 format=rfc5424`)
+	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 [foo@123]`, time.UTC, `priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47 foo@123=`)
+	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47`, time.UTC, `priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345 msg_id=ID47`)
+	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345`, time.UTC, `priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname proc_id=12345`)
+	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname`, time.UTC, `priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com app_name=appname`)
+	f(`<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com`, time.UTC, `priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z hostname=mymachine.example.com`)
+	f(`<165>1 2023-06-03T17:42:32.123456789Z`, time.UTC, `priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424 timestamp=2023-06-03T17:42:32.123456789Z`)
+	f(`<165>1 `, time.UTC, `priority=165 facility_keyword=local4 level=notice facility=20 severity=5 format=rfc5424`)
 }

lib/storage/index_db_synctest_test.go

@@ -0,0 +1,82 @@
+//go:build goexperiment.synctest
+
+package storage
+
+import (
+	"slices"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestIndexDB_MetricIDsNotMappedToTSIDsAreDeleted(t *testing.T) {
+	defer testRemoveAll(t)
+
+	keys := func(missingMetricIDs map[uint64]uint64) []uint64 {
+		keys := []uint64{}
+		for k := range missingMetricIDs {
+			keys = append(keys, k)
+		}
+		slices.Sort(keys)
+		return keys
+	}
+
+	synctest.Run(func() {
+		s := MustOpenStorage(t.Name(), OpenOptions{})
+		defer s.MustClose()
+		idb, putIndexDB := s.getCurrIndexDB()
+		defer putIndexDB()
+
+		type want struct {
+			missingMetricIDs        []uint64
+			missingTSIDsForMetricID uint64
+			deletedMetricIDs        []uint64
+		}
+		assertGetTSIDsFromMetricIDs := func(metricIDs []uint64, want want) {
+			t.Helper()
+			tsids, err := idb.getTSIDsFromMetricIDs(nil, metricIDs, noDeadline)
+			if err != nil {
+				t.Fatalf("getTSIDsFromMetricIDs() failed unexpectedly: %v", err)
+			}
+			if diff := cmp.Diff([]TSID{}, tsids); diff != "" {
+				t.Fatalf("unexpected tsids (-want, +got):\n%s", diff)
+			}
+			missingMetricIDs := keys(s.missingMetricIDs)
+			if diff := cmp.Diff(want.missingMetricIDs, missingMetricIDs); diff != "" {
+				t.Fatalf("unexpected tsids (-want, +got):\n%s", diff)
+			}
+			if got, want := idb.extDB.missingTSIDsForMetricID.Load(), want.missingTSIDsForMetricID; got != want {
+				t.Fatalf("unexpected missingTSIDsForMetricID metric value: got %d, want %d", got, want)
+			}
+			wantDeletedMetricIDs := &uint64set.Set{}
+			wantDeletedMetricIDs.AddMulti(want.deletedMetricIDs)
+			if !s.getDeletedMetricIDs().Equal(wantDeletedMetricIDs) {
+				t.Fatalf("deleted metricIDs set is different from %v", want.deletedMetricIDs)
+			}
+		}
+
+		metricIDs := []uint64{1, 2, 3, 4}
+
+		// These metricIDs are not mapped to the corresponding TSIDs so they are
+		// expected to be placed in missingMetricIDs cache but not be deleted yet.
+		assertGetTSIDsFromMetricIDs(metricIDs, want{
+			missingMetricIDs:        metricIDs,
+			missingTSIDsForMetricID: 0,
+			deletedMetricIDs:        []uint64{},
+		})
+
+		// If we repeat search after one minute, the get soft-deleted and a
+		// corresponding metric is incremented. The metric will remain in
+		// missingMetricIDs cache for another minute.
+		time.Sleep(61 * time.Second)
+		synctest.Wait()
+		assertGetTSIDsFromMetricIDs(metricIDs, want{
+			missingMetricIDs:        metricIDs,
+			missingTSIDsForMetricID: uint64(len(metricIDs)),
+			deletedMetricIDs:        metricIDs,
+		})
+	})
+}

lib/storage/index_db_test.go

@@ -7,11 +7,9 @@ import (
 	"os"
 	"reflect"
 	"regexp"
-	"slices"
 	"sort"
 	"sync/atomic"
 	"testing"
-	"testing/synctest"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -21,7 +19,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 	"github.com/VictoriaMetrics/fastcache"
-	"github.com/google/go-cmp/cmp"
 )
 
 func TestMarshalUnmarshalMetricIDs(t *testing.T) {
@@ -2091,75 +2088,6 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
 	fs.MustRemoveAll(path)
 }
 
-func TestIndexDB_MetricIDsNotMappedToTSIDsAreDeleted(t *testing.T) {
-	defer testRemoveAll(t)
-
-	keys := func(missingMetricIDs map[uint64]uint64) []uint64 {
-		keys := []uint64{}
-		for k := range missingMetricIDs {
-			keys = append(keys, k)
-		}
-		slices.Sort(keys)
-		return keys
-	}
-
-	synctest.Run(func() {
-		s := MustOpenStorage(t.Name(), OpenOptions{})
-		defer s.MustClose()
-		idb, putIndexDB := s.getCurrIndexDB()
-		defer putIndexDB()
-
-		type want struct {
-			missingMetricIDs        []uint64
-			missingTSIDsForMetricID uint64
-			deletedMetricIDs        []uint64
-		}
-		assertGetTSIDsFromMetricIDs := func(metricIDs []uint64, want want) {
-			t.Helper()
-			tsids, err := idb.getTSIDsFromMetricIDs(nil, metricIDs, noDeadline)
-			if err != nil {
-				t.Fatalf("getTSIDsFromMetricIDs() failed unexpectedly: %v", err)
-			}
-			if diff := cmp.Diff([]TSID{}, tsids); diff != "" {
-				t.Fatalf("unexpected tsids (-want, +got):\n%s", diff)
-			}
-			missingMetricIDs := keys(s.missingMetricIDs)
-			if diff := cmp.Diff(want.missingMetricIDs, missingMetricIDs); diff != "" {
-				t.Fatalf("unexpected tsids (-want, +got):\n%s", diff)
-			}
-			if got, want := idb.extDB.missingTSIDsForMetricID.Load(), want.missingTSIDsForMetricID; got != want {
-				t.Fatalf("unexpected missingTSIDsForMetricID metric value: got %d, want %d", got, want)
-			}
-			wantDeletedMetricIDs := &uint64set.Set{}
-			wantDeletedMetricIDs.AddMulti(want.deletedMetricIDs)
-			if !s.getDeletedMetricIDs().Equal(wantDeletedMetricIDs) {
-				t.Fatalf("deleted metricIDs set is different from %v", want.deletedMetricIDs)
-			}
-		}
-
-		metricIDs := []uint64{1, 2, 3, 4}
-
-		// These metricIDs are not mapped to the corresponding TSIDs so they are
-		// expected to be placed in missingMetricIDs cache but not be deleted yet.
-		assertGetTSIDsFromMetricIDs(metricIDs, want{
-			missingMetricIDs:        metricIDs,
-			missingTSIDsForMetricID: 0,
-			deletedMetricIDs:        []uint64{},
-		})
-
-		// If we repeat search after one minute, the get soft-deleted and a
-		// corresponding metric is incremented. The metric will remain in
-		// missingMetricIDs cache for another minute.
-		time.Sleep(61 * time.Second)
-		synctest.Wait()
-		assertGetTSIDsFromMetricIDs(metricIDs, want{
-			missingMetricIDs:        metricIDs,
-			missingTSIDsForMetricID: uint64(len(metricIDs)),
-			deletedMetricIDs:        metricIDs,
-		})
-	})
-}
-
 func toTFPointers(tfs []tagFilter) []*tagFilter {
 	tfps := make([]*tagFilter, len(tfs))
 	for i := range tfs {

lib/storage/index_db_timing_test.go

@@ -133,10 +133,9 @@ func BenchmarkHeadPostingForMatchers(b *testing.B) {
 
 	benchSearch := func(b *testing.B, tfs *TagFilters, expectedMetricIDs int) {
 		tfss := []*TagFilters{tfs}
-		tr := TimeRange{
-			MinTimestamp: 0,
-			MaxTimestamp: timestampFromTime(time.Now()),
-		}
+		// Use special globalIndexTimeRange to instruct indexDB to search global
+		// index instead of per-day index.
+		tr := globalIndexTimeRange
 		for i := 0; i < b.N; i++ {
 			is := db.getIndexSearch(noDeadline)
 			metricIDs, err := is.searchMetricIDs(nil, tfss, tr, 2e9)

lib/storage/storage.go

@@ -240,7 +240,7 @@ func MustOpenStorage(path string, opts OpenOptions) *Storage {
 	mem := memory.Allowed()
 	s.tsidCache = s.mustLoadCache("metricName_tsid", getTSIDCacheSize())
 	s.metricIDCache = s.mustLoadCache("metricID_tsid", mem/16)
-	s.metricNameCache = s.mustLoadCache("metricID_metricName", mem/10)
+	s.metricNameCache = s.mustLoadCache("metricID_metricName", getMetricNamesCacheSize())
 	s.dateMetricIDCache = newDateMetricIDCache()
 
 	hour := fasttime.UnixHour()
@@ -352,6 +352,20 @@ func getMetricNamesStatsCacheSize() int {
 	return maxMetricNamesStatsCacheSize
 }
 
+var maxMetricNameCacheSize int
+
+// SetMetricNameCacheSize overrides the default size of storage/metricName cache
+func SetMetricNameCacheSize(size int) {
+	maxMetricNameCacheSize = size
+}
+
+func getMetricNamesCacheSize() int {
+	if maxMetricNameCacheSize <= 0 {
+		return memory.Allowed() / 10
+	}
+	return maxMetricNameCacheSize
+}
+
 func (s *Storage) getDeletedMetricIDs() *uint64set.Set {
 	return s.deletedMetricIDs.Load()
 }
@@ -2017,6 +2031,11 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 	hmPrev := s.prevHourMetricIDs.Load()
 	hmCurr := s.currHourMetricIDs.Load()
 	var pendingHourEntries []uint64
+	addToPendingHourEntries := func(hour, metricID uint64) {
+		if hour == hmCurr.hour && !hmCurr.m.Has(metricID) {
+			pendingHourEntries = append(pendingHourEntries, metricID)
+		}
+	}
 
 	mn := GetMetricName()
 	defer PutMetricName(mn)
@@ -2120,9 +2139,7 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 				seriesRepopulated++
 				slowInsertsCount++
 			}
-			if hour == hmCurr.hour && !hmCurr.m.Has(genTSID.TSID.MetricID) {
-				pendingHourEntries = append(pendingHourEntries, genTSID.TSID.MetricID)
-			}
+			addToPendingHourEntries(hour, genTSID.TSID.MetricID)
 			continue
 		}
 
@@ -2168,9 +2185,7 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 			prevTSID = genTSID.TSID
 			prevMetricNameRaw = mr.MetricNameRaw
 
-			if hour == hmCurr.hour && !hmCurr.m.Has(genTSID.TSID.MetricID) {
-				pendingHourEntries = append(pendingHourEntries, genTSID.TSID.MetricID)
-			}
+			addToPendingHourEntries(hour, genTSID.TSID.MetricID)
 
 			continue
 		}
@@ -2193,9 +2208,7 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci
 		prevTSID = r.TSID
 		prevMetricNameRaw = mr.MetricNameRaw
 
-		if hour == hmCurr.hour && !hmCurr.m.Has(genTSID.TSID.MetricID) {
-			pendingHourEntries = append(pendingHourEntries, genTSID.TSID.MetricID)
-		}
+		addToPendingHourEntries(hour, genTSID.TSID.MetricID)
 
 		if logNewSeries {
 			logger.Infof("new series created: %s", mn.String())

lib/storage/storage_synctest_test.go

@@ -0,0 +1,110 @@
+//go:build goexperiment.synctest
+
+package storage
+
+import (
+	"fmt"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestStorageSearchMetricNames_CorruptedIndex(t *testing.T) {
+	defer testRemoveAll(t)
+
+	synctest.Run(func() {
+		s := MustOpenStorage(t.Name(), OpenOptions{})
+		defer s.MustClose()
+
+		now := time.Now().UTC()
+		tr := TimeRange{
+			MinTimestamp: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).UnixMilli(),
+			MaxTimestamp: time.Date(now.Year(), now.Month(), now.Day(), 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+		}
+		const numMetrics = 10
+		date := uint64(tr.MinTimestamp) / msecPerDay
+		idb, putCurrIndexDB := s.getCurrIndexDB()
+		defer putCurrIndexDB()
+		var wantMetricIDs []uint64
+
+		// Symulate corrupted index by inserting `(date, tag) -> metricID`
+		// entries only.
+		for i := range numMetrics {
+			metricName := []byte(fmt.Sprintf("metric_%d", i))
+			metricID := generateUniqueMetricID()
+			wantMetricIDs = append(wantMetricIDs, metricID)
+
+			ii := getIndexItems()
+
+			// Create per-day tag -> metricID entries for every tag in mn.
+			kb := kbPool.Get()
+			kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixDateTagToMetricIDs)
+			kb.B = encoding.MarshalUint64(kb.B, date)
+			ii.B = append(ii.B, kb.B...)
+			ii.B = marshalTagValue(ii.B, nil)
+			ii.B = marshalTagValue(ii.B, metricName)
+			ii.B = encoding.MarshalUint64(ii.B, metricID)
+			ii.Next()
+			kbPool.Put(kb)
+
+			idb.tb.AddItems(ii.Items)
+
+			putIndexItems(ii)
+		}
+		idb.tb.DebugFlush()
+
+		tfsAll := NewTagFilters()
+		if err := tfsAll.Add([]byte("__name__"), []byte(".*"), false, true); err != nil {
+			panic(fmt.Sprintf("unexpected error in TagFilters.Add: %v", err))
+		}
+		tfssAll := []*TagFilters{tfsAll}
+
+		searchMetricIDs := func() []uint64 {
+			metricIDs, err := idb.searchMetricIDs(nil, tfssAll, tr, 1e9, noDeadline)
+			if err != nil {
+				panic(fmt.Sprintf("searchMetricIDs() failed unexpectedly: %v", err))
+			}
+			return metricIDs
+		}
+		searchMetricNames := func() []string {
+			metricNames, err := s.SearchMetricNames(nil, tfssAll, tr, 1e9, noDeadline)
+			if err != nil {
+				panic(fmt.Sprintf("SearchMetricNames() failed unexpectedly: %v", err))
+			}
+			return metricNames
+		}
+
+		// Ensure that metricIDs can be searched.
+		if diff := cmp.Diff(wantMetricIDs, searchMetricIDs()); diff != "" {
+			t.Fatalf("unexpected metricIDs (-want, +got):\n%s", diff)
+		}
+		// Ensure that Storage.SearchMetricNames() returns empty result.
+		// The corrupted index lets to find metricIDs by tag (`__name__` tag in
+		// our case) but it lacks metricID->metricName mapping and hence the
+		// empty search result.
+		// The code detects this and puts such metricIDs into a special cache.
+		if diff := cmp.Diff([]string{}, searchMetricNames()); diff != "" {
+			t.Fatalf("unexpected metric names (-want, +got):\n%s", diff)
+		}
+		// Ensure that the metricIDs still can be searched.
+		if diff := cmp.Diff(wantMetricIDs, searchMetricIDs()); diff != "" {
+			t.Fatalf("unexpected metricIDs (-want, +got):\n%s", diff)
+		}
+
+		time.Sleep(61 * time.Second)
+		synctest.Wait()
+
+		// If the same search is repeated after 1 minute, the metricIDs are
+		// marked as deleted.
+		if diff := cmp.Diff([]string{}, searchMetricNames()); diff != "" {
+			t.Fatalf("unexpected metric names (-want, +got):\n%s", diff)
+		}
+		// As a result they cannot be searched anymore.
+		if diff := cmp.Diff([]uint64{}, searchMetricIDs()); diff != "" {
+			t.Fatalf("unexpected metricIDs (-want, +got):\n%s", diff)
+		}
+	})
+}

lib/storage/storage_test.go

@@ -14,10 +14,8 @@ import (
 	"sync"
 	"testing"
 	"testing/quick"
-	"testing/synctest"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	vmfs "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
@@ -4014,100 +4012,3 @@ func TestStorageSearchTagValueSuffixes_maxTagValueSuffixes(t *testing.T) {
 	wantCount = maxTagValueSuffixes
 	assertSuffixCount(maxTagValueSuffixes, wantCount)
 }
-
-func TestStorageSearchMetricNames_CorruptedIndex(t *testing.T) {
-	defer testRemoveAll(t)
-
-	synctest.Run(func() {
-		s := MustOpenStorage(t.Name(), OpenOptions{})
-		defer s.MustClose()
-
-		now := time.Now().UTC()
-		tr := TimeRange{
-			MinTimestamp: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).UnixMilli(),
-			MaxTimestamp: time.Date(now.Year(), now.Month(), now.Day(), 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
-		}
-		const numMetrics = 10
-		date := uint64(tr.MinTimestamp) / msecPerDay
-		idb, putCurrIndexDB := s.getCurrIndexDB()
-		defer putCurrIndexDB()
-		var wantMetricIDs []uint64
-
-		// Symulate corrupted index by inserting `(date, tag) -> metricID`
-		// entries only.
-		for i := range numMetrics {
-			metricName := []byte(fmt.Sprintf("metric_%d", i))
-			metricID := generateUniqueMetricID()
-			wantMetricIDs = append(wantMetricIDs, metricID)
-
-			ii := getIndexItems()
-
-			// Create per-day tag -> metricID entries for every tag in mn.
-			kb := kbPool.Get()
-			kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixDateTagToMetricIDs)
-			kb.B = encoding.MarshalUint64(kb.B, date)
-			ii.B = append(ii.B, kb.B...)
-			ii.B = marshalTagValue(ii.B, nil)
-			ii.B = marshalTagValue(ii.B, metricName)
-			ii.B = encoding.MarshalUint64(ii.B, metricID)
-			ii.Next()
-			kbPool.Put(kb)
-
-			idb.tb.AddItems(ii.Items)
-
-			putIndexItems(ii)
-		}
-		idb.tb.DebugFlush()
-
-		tfsAll := NewTagFilters()
-		if err := tfsAll.Add([]byte("__name__"), []byte(".*"), false, true); err != nil {
-			panic(fmt.Sprintf("unexpected error in TagFilters.Add: %v", err))
-		}
-		tfssAll := []*TagFilters{tfsAll}
-
-		searchMetricIDs := func() []uint64 {
-			metricIDs, err := idb.searchMetricIDs(nil, tfssAll, tr, 1e9, noDeadline)
-			if err != nil {
-				panic(fmt.Sprintf("searchMetricIDs() failed unexpectedly: %v", err))
-			}
-			return metricIDs
-		}
-		searchMetricNames := func() []string {
-			metricNames, err := s.SearchMetricNames(nil, tfssAll, tr, 1e9, noDeadline)
-			if err != nil {
-				panic(fmt.Sprintf("SearchMetricNames() failed unexpectedly: %v", err))
-			}
-			return metricNames
-		}
-
-		// Ensure that metricIDs can be searched.
-		if diff := cmp.Diff(wantMetricIDs, searchMetricIDs()); diff != "" {
-			t.Fatalf("unexpected metricIDs (-want, +got):\n%s", diff)
-		}
-		// Ensure that Storage.SearchMetricNames() returns empty result.
-		// The corrupted index lets to find metricIDs by tag (`__name__` tag in
-		// our case) but it lacks metricID->metricName mapping and hence the
-		// empty search result.
-		// The code detects this and puts such metricIDs into a special cache.
-		if diff := cmp.Diff([]string{}, searchMetricNames()); diff != "" {
-			t.Fatalf("unexpected metric names (-want, +got):\n%s", diff)
-		}
-		// Ensure that the metricIDs still can be searched.
-		if diff := cmp.Diff(wantMetricIDs, searchMetricIDs()); diff != "" {
-			t.Fatalf("unexpected metricIDs (-want, +got):\n%s", diff)
-		}
-
-		time.Sleep(61 * time.Second)
-		synctest.Wait()
-
-		// If the same search is repeated after 1 minute, the metricIDs are
-		// marked as deleted.
-		if diff := cmp.Diff([]string{}, searchMetricNames()); diff != "" {
-			t.Fatalf("unexpected metric names (-want, +got):\n%s", diff)
-		}
-		// As a result they cannot be searched anymore.
-		if diff := cmp.Diff([]uint64{}, searchMetricIDs()); diff != "" {
-			t.Fatalf("unexpected metricIDs (-want, +got):\n%s", diff)
-		}
-	})
-}

lib/streamaggr/streamaggr_synctest_test.go

@@ -0,0 +1,793 @@
+//go:build goexperiment.synctest
+
+package streamaggr
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
+)
+
+func TestAggregatorsSuccess(t *testing.T) {
+	f := func(inputMetrics []string, interval time.Duration, outputMetricsExpected, config, matchIdxsStrExpected string) {
+		t.Helper()
+		synctest.Run(func() {
+			var matchIdxs []byte
+			var tssOutput []prompbmarshal.TimeSeries
+			var tssOutputLock sync.Mutex
+
+			// Initialize Aggregators
+			pushFunc := func(tss []prompbmarshal.TimeSeries) {
+				tssOutputLock.Lock()
+				tssOutput = appendClonedTimeseries(tssOutput, tss)
+				tssOutputLock.Unlock()
+			}
+			a, err := LoadFromData([]byte(config), pushFunc, nil, "some_alias")
+			if err != nil {
+				t.Fatalf("cannot initialize aggregators: %s", err)
+			}
+			for _, metrics := range inputMetrics {
+				// Push the inputMetrics to Aggregators
+				offsetMsecs := time.Now().UnixMilli()
+				tssInput := prometheus.MustParsePromMetrics(metrics, offsetMsecs)
+				matchIdxs = append(matchIdxs, a.Push(tssInput, nil)...)
+				time.Sleep(interval)
+			}
+
+			a.MustStop()
+
+			// Verify matchIdxs equals to matchIdxsExpected
+			matchIdxsStr := ""
+			for _, v := range matchIdxs {
+				matchIdxsStr += strconv.Itoa(int(v))
+			}
+			if matchIdxsStr != matchIdxsStrExpected {
+				t.Fatalf("unexpected matchIdxs;\ngot\n%s\nwant\n%s", matchIdxsStr, matchIdxsStrExpected)
+			}
+
+			// Verify the tssOutput contains the expected metrics
+			outputMetrics := timeSeriessToString(tssOutput)
+			if outputMetrics != outputMetricsExpected {
+				t.Fatalf("unexpected output metrics;\ngot\n%s\nwant\n%s", outputMetrics, outputMetricsExpected)
+			}
+		})
+	}
+
+	// Empty config
+	f([]string{}, time.Second, ``, ``, "")
+	f([]string{`foo{bar="baz"} 1`}, time.Second, ``, ``, "0")
+	f([]string{"foo 1\nbaz 2"}, time.Second, ``, ``, "00")
+
+	// Empty by list - aggregate only by time
+	f([]string{`
+foo{abc="123"} 4
+bar 5 11
+bar 34 10
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_count_samples 2
+bar:1m_count_series 1
+bar:1m_last 5
+bar:1m_sum_samples 39
+foo:1m_count_samples{abc="123"} 2
+foo:1m_count_samples{abc="456",de="fg"} 1
+foo:1m_count_series{abc="123"} 1
+foo:1m_count_series{abc="456",de="fg"} 1
+foo:1m_last{abc="123"} 8.5
+foo:1m_last{abc="456",de="fg"} 8
+foo:1m_sum_samples{abc="123"} 12.5
+foo:1m_sum_samples{abc="456",de="fg"} 8
+`, `
+- interval: 1m
+  outputs: [count_samples, sum_samples, count_series, last]
+`, "11111")
+
+	// Special case: __name__ in `by` list - this is the same as empty `by` list
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_count_samples 1
+bar:1m_count_series 1
+bar:1m_sum_samples 5
+foo:1m_count_samples 3
+foo:1m_count_series 2
+foo:1m_sum_samples 20.5
+`, `
+- interval: 1m
+  by: [__name__]
+  outputs: [count_samples, sum_samples, count_series]
+`, "1111")
+
+	// Non-empty `by` list with non-existing labels
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_by_bar_foo_count_samples 1
+bar:1m_by_bar_foo_count_series 1
+bar:1m_by_bar_foo_sum_samples 5
+foo:1m_by_bar_foo_count_samples 3
+foo:1m_by_bar_foo_count_series 2
+foo:1m_by_bar_foo_sum_samples 20.5
+`, `
+- interval: 1m
+  by: [foo, bar]
+  outputs: [count_samples, sum_samples, count_series]
+`, "1111")
+
+	// Non-empty `by` list with existing label
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_by_abc_count_samples 1
+bar:1m_by_abc_count_series 1
+bar:1m_by_abc_sum_samples 5
+foo:1m_by_abc_count_samples{abc="123"} 2
+foo:1m_by_abc_count_samples{abc="456"} 1
+foo:1m_by_abc_count_series{abc="123"} 1
+foo:1m_by_abc_count_series{abc="456"} 1
+foo:1m_by_abc_sum_samples{abc="123"} 12.5
+foo:1m_by_abc_sum_samples{abc="456"} 8
+`, `
+- interval: 1m
+  by: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+`, "1111")
+
+	// Non-empty `by` list with duplicate existing label
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_by_abc_count_samples 1
+bar:1m_by_abc_count_series 1
+bar:1m_by_abc_sum_samples 5
+foo:1m_by_abc_count_samples{abc="123"} 2
+foo:1m_by_abc_count_samples{abc="456"} 1
+foo:1m_by_abc_count_series{abc="123"} 1
+foo:1m_by_abc_count_series{abc="456"} 1
+foo:1m_by_abc_sum_samples{abc="123"} 12.5
+foo:1m_by_abc_sum_samples{abc="456"} 8
+`, `
+- interval: 1m
+  by: [abc, abc]
+  outputs: [count_samples, sum_samples, count_series]
+`, "1111")
+
+	// Non-empty `without` list with non-existing labels
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_without_foo_count_samples 1
+bar:1m_without_foo_count_series 1
+bar:1m_without_foo_sum_samples 5
+foo:1m_without_foo_count_samples{abc="123"} 2
+foo:1m_without_foo_count_samples{abc="456",de="fg"} 1
+foo:1m_without_foo_count_series{abc="123"} 1
+foo:1m_without_foo_count_series{abc="456",de="fg"} 1
+foo:1m_without_foo_sum_samples{abc="123"} 12.5
+foo:1m_without_foo_sum_samples{abc="456",de="fg"} 8
+`, `
+- interval: 1m
+  without: [foo]
+  outputs: [count_samples, sum_samples, count_series]
+`, "1111")
+
+	// Non-empty `without` list with existing labels
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_without_abc_count_samples 1
+bar:1m_without_abc_count_series 1
+bar:1m_without_abc_sum_samples 5
+foo:1m_without_abc_count_samples 2
+foo:1m_without_abc_count_samples{de="fg"} 1
+foo:1m_without_abc_count_series 1
+foo:1m_without_abc_count_series{de="fg"} 1
+foo:1m_without_abc_sum_samples 12.5
+foo:1m_without_abc_sum_samples{de="fg"} 8
+`, `
+- interval: 1m
+  without: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+`, "1111")
+
+	// Special case: __name__ in `without` list
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `:1m_count_samples 1
+:1m_count_samples{abc="123"} 2
+:1m_count_samples{abc="456",de="fg"} 1
+:1m_count_series 1
+:1m_count_series{abc="123"} 1
+:1m_count_series{abc="456",de="fg"} 1
+:1m_sum_samples 5
+:1m_sum_samples{abc="123"} 12.5
+:1m_sum_samples{abc="456",de="fg"} 8
+`, `
+- interval: 1m
+  without: [__name__]
+  outputs: [count_samples, sum_samples, count_series]
+`, "1111")
+
+	// drop some input metrics
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_without_abc_count_samples 1
+bar:1m_without_abc_count_series 1
+bar:1m_without_abc_sum_samples 5
+`, `
+- interval: 1m
+  without: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+  input_relabel_configs:
+  - if: 'foo'
+    action: drop
+`, "1111")
+
+	// rename output metrics
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar-1m-without-abc-count-samples 1
+bar-1m-without-abc-count-series 1
+bar-1m-without-abc-sum-samples 5
+foo-1m-without-abc-count-samples 2
+foo-1m-without-abc-count-series 1
+foo-1m-without-abc-sum-samples 12.5
+`, `
+- interval: 1m
+  without: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+  output_relabel_configs:
+  - action: replace_all
+    source_labels: [__name__]
+    regex: ":|_"
+    replacement: "-"
+    target_label: __name__
+  - action: drop
+    source_labels: [de]
+    regex: fg
+`, "1111")
+
+	// match doesn't match anything
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, ``, `
+- interval: 1m
+  without: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+  match: '{non_existing_label!=""}'
+  name: foobar
+`, "0000")
+
+	// match matches foo series with non-empty abc label
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `foo:1m_by_abc_count_samples{abc="123"} 2
+foo:1m_by_abc_count_samples{abc="456"} 1
+foo:1m_by_abc_count_series{abc="123"} 1
+foo:1m_by_abc_count_series{abc="456"} 1
+foo:1m_by_abc_sum_samples{abc="123"} 12.5
+foo:1m_by_abc_sum_samples{abc="456"} 8
+`, `
+- interval: 1m
+  by: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+  name: abcdef
+  match:
+  - foo{abc=~".+"}
+  - '{non_existing_label!=""}'
+`, "1011")
+
+	// total output for non-repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 4.34
+`}, time.Minute, `bar:1m_total{baz="qwe"} 0
+foo:1m_total 0
+`, `
+- interval: 1m
+  outputs: [total]
+`, "11")
+
+	// total output for non-repeated series, ignore first sample 0s
+	f([]string{`
+foo 123
+bar{baz="qwe"} 4.34
+`}, time.Minute, `bar:1m_total{baz="qwe"} 4.34
+foo:1m_total 123
+`, `
+- interval: 1m
+  outputs: [total]
+  ignore_first_sample_interval: 0s
+`, "11")
+
+	// total_prometheus output for non-repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 4.34
+`}, time.Minute, `bar:1m_total_prometheus{baz="qwe"} 0
+foo:1m_total_prometheus 0
+`, `
+- interval: 1m
+  outputs: [total_prometheus]
+`, "11")
+
+	// total output for repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 1.31
+bar{baz="qwe"} 4.34 1
+bar{baz="qwe"} 2
+foo{baz="qwe"} -5
+bar{baz="qwer"} 343
+bar{baz="qwer"} 344
+foo{baz="qwe"} 10
+`}, time.Minute, `bar:1m_total{baz="qwe"} 3.03
+bar:1m_total{baz="qwer"} 1
+foo:1m_total 0
+foo:1m_total{baz="qwe"} 15
+`, `
+- interval: 1m
+  outputs: [total]
+`, "11111111")
+
+	// total_prometheus output for repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 1.32
+bar{baz="qwe"} 4.34
+bar{baz="qwe"} 2
+foo{baz="qwe"} -5
+bar{baz="qwer"} 343
+bar{baz="qwer"} 344
+foo{baz="qwe"} 10
+`}, time.Minute, `bar:1m_total_prometheus{baz="qwe"} 5.02
+bar:1m_total_prometheus{baz="qwer"} 1
+foo:1m_total_prometheus 0
+foo:1m_total_prometheus{baz="qwe"} 15
+`, `
+- interval: 1m
+  outputs: [total_prometheus]
+`, "11111111")
+
+	// total output for repeated series with group by __name__
+	f([]string{`
+foo 123
+bar{baz="qwe"} 1.32
+bar{baz="qwe"} 4.34
+bar{baz="qwe"} 2
+foo{baz="qwe"} -5
+bar{baz="qwer"} 343
+bar{baz="qwer"} 344
+foo{baz="qwe"} 10
+`}, time.Minute, `bar:1m_total 6.02
+foo:1m_total 15
+`, `
+- interval: 1m
+  by: [__name__]
+  outputs: [total]
+`, "11111111")
+
+	// total_prometheus output for repeated series with group by __name__
+	f([]string{`
+foo 123
+bar{baz="qwe"} 1.32
+bar{baz="qwe"} 4.34
+bar{baz="qwe"} 2
+foo{baz="qwe"} -5
+bar{baz="qwer"} 343
+bar{baz="qwer"} 344
+foo{baz="qwe"} 10
+`}, time.Minute, `bar:1m_total_prometheus 6.02
+foo:1m_total_prometheus 15
+`, `
+- interval: 1m
+  by: [__name__]
+  outputs: [total_prometheus]
+`, "11111111")
+
+	// increase output for non-repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 4.34
+`}, time.Minute, `bar:1m_increase{baz="qwe"} 0
+foo:1m_increase 0
+`, `
+- interval: 1m
+  outputs: [increase]
+`, "11")
+
+	// increase_prometheus output for non-repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 4.34
+`}, time.Minute, `bar:1m_increase_prometheus{baz="qwe"} 0
+foo:1m_increase_prometheus 0
+`, `
+- interval: 1m
+  outputs: [increase_prometheus]
+`, "11")
+
+	// increase output for repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 1.32
+bar{baz="qwe"} 4.34
+bar{baz="qwe"} 2
+foo{baz="qwe"} -5
+bar{baz="qwer"} 343
+bar{baz="qwer"} 344
+foo{baz="qwe"} 10
+`}, time.Minute, `bar:1m_increase{baz="qwe"} 5.02
+bar:1m_increase{baz="qwer"} 1
+foo:1m_increase 0
+foo:1m_increase{baz="qwe"} 15
+`, `
+- interval: 1m
+  outputs: [increase]
+`, "11111111")
+
+	// increase_prometheus output for repeated series
+	f([]string{`
+foo 123
+bar{baz="qwe"} 1.32
+bar{baz="qwe"} 4.34
+bar{baz="qwe"} 2
+foo{baz="qwe"} -5
+bar{baz="qwer"} 343
+bar{baz="qwer"} 344
+foo{baz="qwe"} 10
+`}, time.Minute, `bar:1m_increase_prometheus{baz="qwe"} 5.02
+bar:1m_increase_prometheus{baz="qwer"} 1
+foo:1m_increase_prometheus 0
+foo:1m_increase_prometheus{baz="qwe"} 15
+`, `
+- interval: 1m
+  outputs: [increase_prometheus]
+`, "11111111")
+
+	// multiple aggregate configs
+	f([]string{`
+foo 1
+foo{bar="baz"} 2
+foo 3.3
+`, ``, ``, ``, ``}, time.Minute, `foo:1m_count_series 1
+foo:1m_count_series{bar="baz"} 1
+foo:1m_sum_samples 0
+foo:1m_sum_samples 0
+foo:1m_sum_samples 4.3
+foo:1m_sum_samples{bar="baz"} 0
+foo:1m_sum_samples{bar="baz"} 0
+foo:1m_sum_samples{bar="baz"} 2
+foo:5m_by_bar_sum_samples 4.3
+foo:5m_by_bar_sum_samples{bar="baz"} 2
+`, `
+- interval: 1m
+  outputs: [count_series, sum_samples]
+- interval: 5m
+  by: [bar]
+  outputs: [sum_samples]
+`, "111")
+
+	// min and max outputs
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_max 5
+bar:1m_min 5
+foo:1m_max{abc="123"} 8.5
+foo:1m_max{abc="456",de="fg"} 8
+foo:1m_min{abc="123"} 4
+foo:1m_min{abc="456",de="fg"} 8
+`, `
+- interval: 1m
+  outputs: [min, max]
+`, "1111")
+
+	// avg output
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_avg 5
+foo:1m_avg{abc="123"} 6.25
+foo:1m_avg{abc="456",de="fg"} 8
+`, `
+- interval: 1m
+  outputs: [avg]
+`, "1111")
+
+	// stddev output
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_stddev 0
+foo:1m_stddev{abc="123"} 2.25
+foo:1m_stddev{abc="456",de="fg"} 0
+`, `
+- interval: 1m
+  outputs: [stddev]
+`, "1111")
+
+	// stdvar output
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar:1m_stdvar 0
+foo:1m_stdvar{abc="123"} 5.0625
+foo:1m_stdvar{abc="456",de="fg"} 0
+`, `
+- interval: 1m
+  outputs: [stdvar]
+`, "1111")
+
+	// histogram_bucket output
+	f([]string{`
+cpu_usage{cpu="1"} 12.5
+cpu_usage{cpu="1"} 13.3
+cpu_usage{cpu="1"} 13
+cpu_usage{cpu="1"} 12
+cpu_usage{cpu="1"} 14
+cpu_usage{cpu="1"} 25
+cpu_usage{cpu="2"} 90
+`}, time.Minute, `cpu_usage:1m_histogram_bucket{cpu="1",vmrange="1.136e+01...1.292e+01"} 2
+cpu_usage:1m_histogram_bucket{cpu="1",vmrange="1.292e+01...1.468e+01"} 3
+cpu_usage:1m_histogram_bucket{cpu="1",vmrange="2.448e+01...2.783e+01"} 1
+cpu_usage:1m_histogram_bucket{cpu="2",vmrange="8.799e+01...1.000e+02"} 1
+`, `
+- interval: 1m
+  outputs: [histogram_bucket]
+`, "1111111")
+
+	// histogram_bucket output without cpu
+	f([]string{`
+cpu_usage{cpu="1"} 12.5
+cpu_usage{cpu="1"} 13.3
+cpu_usage{cpu="1"} 13
+cpu_usage{cpu="1"} 12
+cpu_usage{cpu="1"} 14
+cpu_usage{cpu="1"} 25
+cpu_usage{cpu="2"} 90
+`}, time.Minute, `cpu_usage:1m_without_cpu_histogram_bucket{vmrange="1.136e+01...1.292e+01"} 2
+cpu_usage:1m_without_cpu_histogram_bucket{vmrange="1.292e+01...1.468e+01"} 3
+cpu_usage:1m_without_cpu_histogram_bucket{vmrange="2.448e+01...2.783e+01"} 1
+cpu_usage:1m_without_cpu_histogram_bucket{vmrange="8.799e+01...1.000e+02"} 1
+`, `
+- interval: 1m
+  without: [cpu]
+  outputs: [histogram_bucket]
+`, "1111111")
+
+	// quantiles output
+	f([]string{`
+cpu_usage{cpu="1"} 12.5
+cpu_usage{cpu="1"} 13.3
+cpu_usage{cpu="1"} 13
+cpu_usage{cpu="1"} 12
+cpu_usage{cpu="1"} 14
+cpu_usage{cpu="1"} 25
+cpu_usage{cpu="2"} 90
+`}, time.Minute, `cpu_usage:1m_quantiles{cpu="1",quantile="0"} 12
+cpu_usage:1m_quantiles{cpu="1",quantile="0.5"} 13.3
+cpu_usage:1m_quantiles{cpu="1",quantile="1"} 25
+cpu_usage:1m_quantiles{cpu="2",quantile="0"} 90
+cpu_usage:1m_quantiles{cpu="2",quantile="0.5"} 90
+cpu_usage:1m_quantiles{cpu="2",quantile="1"} 90
+`, `
+- interval: 1m
+  outputs: ["quantiles(0, 0.5, 1)"]
+`, "1111111")
+
+	// quantiles output without cpu
+	f([]string{`
+cpu_usage{cpu="1"} 12.5
+cpu_usage{cpu="1"} 13.3
+cpu_usage{cpu="1"} 13
+cpu_usage{cpu="1"} 12
+cpu_usage{cpu="1"} 14
+cpu_usage{cpu="1"} 25
+cpu_usage{cpu="2"} 90
+`}, time.Minute, `cpu_usage:1m_without_cpu_quantiles{quantile="0"} 12
+cpu_usage:1m_without_cpu_quantiles{quantile="0.5"} 13.3
+cpu_usage:1m_without_cpu_quantiles{quantile="1"} 90
+`, `
+- interval: 1m
+  without: [cpu]
+  outputs: ["quantiles(0, 0.5, 1)"]
+`, "1111111")
+
+	// append additional label
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5 10
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar-1m-without-abc-count-samples{new_label="must_keep_metric_name"} 1
+bar-1m-without-abc-count-series{new_label="must_keep_metric_name"} 1
+bar-1m-without-abc-sum-samples{new_label="must_keep_metric_name"} 5
+foo-1m-without-abc-count-samples{new_label="must_keep_metric_name"} 2
+foo-1m-without-abc-count-series{new_label="must_keep_metric_name"} 1
+foo-1m-without-abc-sum-samples{new_label="must_keep_metric_name"} 12.5
+`, `
+- interval: 1m
+  without: [abc]
+  outputs: [count_samples, sum_samples, count_series]
+  output_relabel_configs:
+  - action: replace_all
+    source_labels: [__name__]
+    regex: ":|_"
+    replacement: "-"
+    target_label: __name__
+  - action: drop
+    source_labels: [de]
+    regex: fg
+  - target_label: new_label
+    replacement: must_keep_metric_name
+`, "1111")
+
+	// test rate_sum and rate_avg
+	f([]string{`
+foo{abc="123", cde="1"} 3
+foo{abc="456", cde="1"} 8.5
+foo 12 34
+`, `
+foo{abc="123", cde="1"} 8
+foo{abc="456", cde="1"} 11
+`}, time.Minute, `foo:1m_by_cde_rate_avg{cde="1"} 0.0625
+foo:1m_by_cde_rate_sum{cde="1"} 0.125
+`, `
+- interval: 1m
+  by: [cde]
+  outputs: [rate_sum, rate_avg]
+`, "11111")
+
+	// test rate_sum and rate_avg, when two aggregation intervals are empty
+	f([]string{`
+foo{abc="123", cde="1"} 2
+foo{abc="456", cde="1"} 8
+foo{abc="777", cde="1"} 9 -10
+`, ``, ``, `
+foo{abc="123", cde="1"} 20
+foo{abc="456", cde="1"} 26
+foo{abc="777", cde="1"} 27 -10
+`}, time.Minute, `foo:1m_by_cde_rate_avg{cde="1"} 0.1
+foo:1m_by_cde_rate_sum{cde="1"} 0.2
+`, `            
+- interval: 1m
+  by: [cde]
+  outputs: [rate_sum, rate_avg]
+  enable_windows: true
+`, "111111")
+
+	// rate_sum and rate_avg with duplicated events
+	f([]string{`
+foo{abc="123", cde="1"} 4  10
+foo{abc="123", cde="1"} 4  10
+`}, time.Minute, ``, `
+- interval: 1m
+  outputs: [rate_sum, rate_avg]
+`, "11")
+
+	// rate_sum and rate_avg for a single sample
+	f([]string{`
+foo 4  10
+bar 5  10
+`}, time.Minute, ``, `
+- interval: 1m
+  outputs: [rate_sum, rate_avg]
+`, "11")
+
+	// unique_samples output
+	f([]string{`
+foo 1  10
+foo 2  20
+foo 1  10
+foo 2  20
+foo 3  20
+`}, time.Minute, `foo:1m_unique_samples 3
+`, `
+- interval: 1m
+  outputs: [unique_samples]
+`, "11111")
+
+	// keep_metric_names
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+bar -34.3
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar 2
+foo{abc="123"} 2
+foo{abc="456",de="fg"} 1
+`, `
+- interval: 1m
+  keep_metric_names: true
+  outputs: [count_samples]
+`, "11111")
+
+	// drop_input_labels
+	f([]string{`
+foo{abc="123"} 4
+bar 5
+foo{abc="123"} 8.5
+bar -34.3
+foo{abc="456",de="fg"} 8
+`}, time.Minute, `bar 2
+foo 2
+foo{de="fg"} 1
+`, `
+- interval: 1m
+  drop_input_labels: [abc]
+  keep_metric_names: true
+  outputs: [count_samples]
+`, "11111")
+
+	f([]string{`
+foo 123
+bar 567
+`, ``, ``}, 30*time.Second, `bar:1m_sum_samples 567
+foo:1m_sum_samples 123
+`, `
+- interval: 1m
+  outputs: [sum_samples]
+  dedup_interval: 30s
+`, "11")
+
+	f([]string{`
+foo 123
+bar{baz="qwe"} 1.32
+bar{baz="qwe"} 4.34
+bar{baz="qwe"} 2
+foo{baz="qwe"} -5
+bar{baz="qwer"} 343
+bar{baz="qwer"} 344
+foo{baz="qwe"} 10
+`, ``, ``}, 30*time.Second, `bar:1m_sum_samples{baz="qwe"} 4.34
+bar:1m_sum_samples{baz="qwer"} 344
+foo:1m_sum_samples 123
+foo:1m_sum_samples{baz="qwe"} 10
+`, `
+- interval: 1m
+  dedup_interval: 30s
+  outputs: [sum_samples]
+`, "11111111")
+}

lib/streamaggr/streamaggr_test.go

@@ -3,16 +3,11 @@ package streamaggr
 import (
 	"fmt"
 	"sort"
-	"strconv"
 	"strings"
-	"sync"
 	"testing"
-	"testing/synctest"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 )
 
 func TestAggregatorsFailure(t *testing.T) {
@@ -256,785 +251,6 @@ func TestAggregatorsEqual(t *testing.T) {
   ignore_first_intervals: 4`, false)
 }
 
-func TestAggregatorsSuccess(t *testing.T) {
-	f := func(inputMetrics []string, interval time.Duration, outputMetricsExpected, config, matchIdxsStrExpected string) {
-		t.Helper()
-		synctest.Run(func() {
-			var matchIdxs []byte
-			var tssOutput []prompbmarshal.TimeSeries
-			var tssOutputLock sync.Mutex
-
-			// Initialize Aggregators
-			pushFunc := func(tss []prompbmarshal.TimeSeries) {
-				tssOutputLock.Lock()
-				tssOutput = appendClonedTimeseries(tssOutput, tss)
-				tssOutputLock.Unlock()
-			}
-			a, err := LoadFromData([]byte(config), pushFunc, nil, "some_alias")
-			if err != nil {
-				t.Fatalf("cannot initialize aggregators: %s", err)
-			}
-			for _, metrics := range inputMetrics {
-				// Push the inputMetrics to Aggregators
-				offsetMsecs := time.Now().UnixMilli()
-				tssInput := prometheus.MustParsePromMetrics(metrics, offsetMsecs)
-				matchIdxs = append(matchIdxs, a.Push(tssInput, nil)...)
-				time.Sleep(interval)
-			}
-
-			a.MustStop()
-
-			// Verify matchIdxs equals to matchIdxsExpected
-			matchIdxsStr := ""
-			for _, v := range matchIdxs {
-				matchIdxsStr += strconv.Itoa(int(v))
-			}
-			if matchIdxsStr != matchIdxsStrExpected {
-				t.Fatalf("unexpected matchIdxs;\ngot\n%s\nwant\n%s", matchIdxsStr, matchIdxsStrExpected)
-			}
-
-			// Verify the tssOutput contains the expected metrics
-			outputMetrics := timeSeriessToString(tssOutput)
-			if outputMetrics != outputMetricsExpected {
-				t.Fatalf("unexpected output metrics;\ngot\n%s\nwant\n%s", outputMetrics, outputMetricsExpected)
-			}
-		})
-	}
-
-	// Empty config
-	f([]string{}, time.Second, ``, ``, "")
-	f([]string{`foo{bar="baz"} 1`}, time.Second, ``, ``, "0")
-	f([]string{"foo 1\nbaz 2"}, time.Second, ``, ``, "00")
-
-	// Empty by list - aggregate only by time
-	f([]string{`
-foo{abc="123"} 4
-bar 5 11
-bar 34 10
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_count_samples 2
-bar:1m_count_series 1
-bar:1m_last 5
-bar:1m_sum_samples 39
-foo:1m_count_samples{abc="123"} 2
-foo:1m_count_samples{abc="456",de="fg"} 1
-foo:1m_count_series{abc="123"} 1
-foo:1m_count_series{abc="456",de="fg"} 1
-foo:1m_last{abc="123"} 8.5
-foo:1m_last{abc="456",de="fg"} 8
-foo:1m_sum_samples{abc="123"} 12.5
-foo:1m_sum_samples{abc="456",de="fg"} 8
-`, `
-- interval: 1m
-  outputs: [count_samples, sum_samples, count_series, last]
-`, "11111")
-
-	// Special case: __name__ in `by` list - this is the same as empty `by` list
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_count_samples 1
-bar:1m_count_series 1
-bar:1m_sum_samples 5
-foo:1m_count_samples 3
-foo:1m_count_series 2
-foo:1m_sum_samples 20.5
-`, `
-- interval: 1m
-  by: [__name__]
-  outputs: [count_samples, sum_samples, count_series]
-`, "1111")
-
-	// Non-empty `by` list with non-existing labels
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_by_bar_foo_count_samples 1
-bar:1m_by_bar_foo_count_series 1
-bar:1m_by_bar_foo_sum_samples 5
-foo:1m_by_bar_foo_count_samples 3
-foo:1m_by_bar_foo_count_series 2
-foo:1m_by_bar_foo_sum_samples 20.5
-`, `
-- interval: 1m
-  by: [foo, bar]
-  outputs: [count_samples, sum_samples, count_series]
-`, "1111")
-
-	// Non-empty `by` list with existing label
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_by_abc_count_samples 1
-bar:1m_by_abc_count_series 1
-bar:1m_by_abc_sum_samples 5
-foo:1m_by_abc_count_samples{abc="123"} 2
-foo:1m_by_abc_count_samples{abc="456"} 1
-foo:1m_by_abc_count_series{abc="123"} 1
-foo:1m_by_abc_count_series{abc="456"} 1
-foo:1m_by_abc_sum_samples{abc="123"} 12.5
-foo:1m_by_abc_sum_samples{abc="456"} 8
-`, `
-- interval: 1m
-  by: [abc]
-  outputs: [count_samples, sum_samples, count_series]
-`, "1111")
-
-	// Non-empty `by` list with duplicate existing label
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_by_abc_count_samples 1
-bar:1m_by_abc_count_series 1
-bar:1m_by_abc_sum_samples 5
-foo:1m_by_abc_count_samples{abc="123"} 2
-foo:1m_by_abc_count_samples{abc="456"} 1
-foo:1m_by_abc_count_series{abc="123"} 1
-foo:1m_by_abc_count_series{abc="456"} 1
-foo:1m_by_abc_sum_samples{abc="123"} 12.5
-foo:1m_by_abc_sum_samples{abc="456"} 8
-`, `
-- interval: 1m
-  by: [abc, abc]
-  outputs: [count_samples, sum_samples, count_series]
-`, "1111")
-
-	// Non-empty `without` list with non-existing labels
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_without_foo_count_samples 1
-bar:1m_without_foo_count_series 1
-bar:1m_without_foo_sum_samples 5
-foo:1m_without_foo_count_samples{abc="123"} 2
-foo:1m_without_foo_count_samples{abc="456",de="fg"} 1
-foo:1m_without_foo_count_series{abc="123"} 1
-foo:1m_without_foo_count_series{abc="456",de="fg"} 1
-foo:1m_without_foo_sum_samples{abc="123"} 12.5
-foo:1m_without_foo_sum_samples{abc="456",de="fg"} 8
-`, `
-- interval: 1m
-  without: [foo]
-  outputs: [count_samples, sum_samples, count_series]
-`, "1111")
-
-	// Non-empty `without` list with existing labels
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_without_abc_count_samples 1
-bar:1m_without_abc_count_series 1
-bar:1m_without_abc_sum_samples 5
-foo:1m_without_abc_count_samples 2
-foo:1m_without_abc_count_samples{de="fg"} 1
-foo:1m_without_abc_count_series 1
-foo:1m_without_abc_count_series{de="fg"} 1
-foo:1m_without_abc_sum_samples 12.5
-foo:1m_without_abc_sum_samples{de="fg"} 8
-`, `
-- interval: 1m
-  without: [abc]
-  outputs: [count_samples, sum_samples, count_series]
-`, "1111")
-
-	// Special case: __name__ in `without` list
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `:1m_count_samples 1
-:1m_count_samples{abc="123"} 2
-:1m_count_samples{abc="456",de="fg"} 1
-:1m_count_series 1
-:1m_count_series{abc="123"} 1
-:1m_count_series{abc="456",de="fg"} 1
-:1m_sum_samples 5
-:1m_sum_samples{abc="123"} 12.5
-:1m_sum_samples{abc="456",de="fg"} 8
-`, `
-- interval: 1m
-  without: [__name__]
-  outputs: [count_samples, sum_samples, count_series]
-`, "1111")
-
-	// drop some input metrics
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_without_abc_count_samples 1
-bar:1m_without_abc_count_series 1
-bar:1m_without_abc_sum_samples 5
-`, `
-- interval: 1m
-  without: [abc]
-  outputs: [count_samples, sum_samples, count_series]
-  input_relabel_configs:
-  - if: 'foo'
-    action: drop
-`, "1111")
-
-	// rename output metrics
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar-1m-without-abc-count-samples 1
-bar-1m-without-abc-count-series 1
-bar-1m-without-abc-sum-samples 5
-foo-1m-without-abc-count-samples 2
-foo-1m-without-abc-count-series 1
-foo-1m-without-abc-sum-samples 12.5
-`, `
-- interval: 1m
-  without: [abc]
-  outputs: [count_samples, sum_samples, count_series]
-  output_relabel_configs:
-  - action: replace_all
-    source_labels: [__name__]
-    regex: ":|_"
-    replacement: "-"
-    target_label: __name__
-  - action: drop
-    source_labels: [de]
-    regex: fg
-`, "1111")
-
-	// match doesn't match anything
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, ``, `
-- interval: 1m
-  without: [abc]
-  outputs: [count_samples, sum_samples, count_series]
-  match: '{non_existing_label!=""}'
-  name: foobar
-`, "0000")
-
-	// match matches foo series with non-empty abc label
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `foo:1m_by_abc_count_samples{abc="123"} 2
-foo:1m_by_abc_count_samples{abc="456"} 1
-foo:1m_by_abc_count_series{abc="123"} 1
-foo:1m_by_abc_count_series{abc="456"} 1
-foo:1m_by_abc_sum_samples{abc="123"} 12.5
-foo:1m_by_abc_sum_samples{abc="456"} 8
-`, `
-- interval: 1m
-  by: [abc]
-  outputs: [count_samples, sum_samples, count_series]
-  name: abcdef
-  match:
-  - foo{abc=~".+"}
-  - '{non_existing_label!=""}'
-`, "1011")
-
-	// total output for non-repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 4.34
-`}, time.Minute, `bar:1m_total{baz="qwe"} 0
-foo:1m_total 0
-`, `
-- interval: 1m
-  outputs: [total]
-`, "11")
-
-	// total output for non-repeated series, ignore first sample 0s
-	f([]string{`
-foo 123
-bar{baz="qwe"} 4.34
-`}, time.Minute, `bar:1m_total{baz="qwe"} 4.34
-foo:1m_total 123
-`, `
-- interval: 1m
-  outputs: [total]
-  ignore_first_sample_interval: 0s
-`, "11")
-
-	// total_prometheus output for non-repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 4.34
-`}, time.Minute, `bar:1m_total_prometheus{baz="qwe"} 0
-foo:1m_total_prometheus 0
-`, `
-- interval: 1m
-  outputs: [total_prometheus]
-`, "11")
-
-	// total output for repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 1.31
-bar{baz="qwe"} 4.34 1
-bar{baz="qwe"} 2
-foo{baz="qwe"} -5
-bar{baz="qwer"} 343
-bar{baz="qwer"} 344
-foo{baz="qwe"} 10
-`}, time.Minute, `bar:1m_total{baz="qwe"} 3.03
-bar:1m_total{baz="qwer"} 1
-foo:1m_total 0
-foo:1m_total{baz="qwe"} 15
-`, `
-- interval: 1m
-  outputs: [total]
-`, "11111111")
-
-	// total_prometheus output for repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 1.32
-bar{baz="qwe"} 4.34
-bar{baz="qwe"} 2
-foo{baz="qwe"} -5
-bar{baz="qwer"} 343
-bar{baz="qwer"} 344
-foo{baz="qwe"} 10
-`}, time.Minute, `bar:1m_total_prometheus{baz="qwe"} 5.02
-bar:1m_total_prometheus{baz="qwer"} 1
-foo:1m_total_prometheus 0
-foo:1m_total_prometheus{baz="qwe"} 15
-`, `
-- interval: 1m
-  outputs: [total_prometheus]
-`, "11111111")
-
-	// total output for repeated series with group by __name__
-	f([]string{`
-foo 123
-bar{baz="qwe"} 1.32
-bar{baz="qwe"} 4.34
-bar{baz="qwe"} 2
-foo{baz="qwe"} -5
-bar{baz="qwer"} 343
-bar{baz="qwer"} 344
-foo{baz="qwe"} 10
-`}, time.Minute, `bar:1m_total 6.02
-foo:1m_total 15
-`, `
-- interval: 1m
-  by: [__name__]
-  outputs: [total]
-`, "11111111")
-
-	// total_prometheus output for repeated series with group by __name__
-	f([]string{`
-foo 123
-bar{baz="qwe"} 1.32
-bar{baz="qwe"} 4.34
-bar{baz="qwe"} 2
-foo{baz="qwe"} -5
-bar{baz="qwer"} 343
-bar{baz="qwer"} 344
-foo{baz="qwe"} 10
-`}, time.Minute, `bar:1m_total_prometheus 6.02
-foo:1m_total_prometheus 15
-`, `
-- interval: 1m
-  by: [__name__]
-  outputs: [total_prometheus]
-`, "11111111")
-
-	// increase output for non-repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 4.34
-`}, time.Minute, `bar:1m_increase{baz="qwe"} 0
-foo:1m_increase 0
-`, `
-- interval: 1m
-  outputs: [increase]
-`, "11")
-
-	// increase_prometheus output for non-repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 4.34
-`}, time.Minute, `bar:1m_increase_prometheus{baz="qwe"} 0
-foo:1m_increase_prometheus 0
-`, `
-- interval: 1m
-  outputs: [increase_prometheus]
-`, "11")
-
-	// increase output for repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 1.32
-bar{baz="qwe"} 4.34
-bar{baz="qwe"} 2
-foo{baz="qwe"} -5
-bar{baz="qwer"} 343
-bar{baz="qwer"} 344
-foo{baz="qwe"} 10
-`}, time.Minute, `bar:1m_increase{baz="qwe"} 5.02
-bar:1m_increase{baz="qwer"} 1
-foo:1m_increase 0
-foo:1m_increase{baz="qwe"} 15
-`, `
-- interval: 1m
-  outputs: [increase]
-`, "11111111")
-
-	// increase_prometheus output for repeated series
-	f([]string{`
-foo 123
-bar{baz="qwe"} 1.32
-bar{baz="qwe"} 4.34
-bar{baz="qwe"} 2
-foo{baz="qwe"} -5
-bar{baz="qwer"} 343
-bar{baz="qwer"} 344
-foo{baz="qwe"} 10
-`}, time.Minute, `bar:1m_increase_prometheus{baz="qwe"} 5.02
-bar:1m_increase_prometheus{baz="qwer"} 1
-foo:1m_increase_prometheus 0
-foo:1m_increase_prometheus{baz="qwe"} 15
-`, `
-- interval: 1m
-  outputs: [increase_prometheus]
-`, "11111111")
-
-	// multiple aggregate configs
-	f([]string{`
-foo 1
-foo{bar="baz"} 2
-foo 3.3
-`, ``, ``, ``, ``}, time.Minute, `foo:1m_count_series 1
-foo:1m_count_series{bar="baz"} 1
-foo:1m_sum_samples 0
-foo:1m_sum_samples 0
-foo:1m_sum_samples 4.3
-foo:1m_sum_samples{bar="baz"} 0
-foo:1m_sum_samples{bar="baz"} 0
-foo:1m_sum_samples{bar="baz"} 2
-foo:5m_by_bar_sum_samples 4.3
-foo:5m_by_bar_sum_samples{bar="baz"} 2
-`, `
-- interval: 1m
-  outputs: [count_series, sum_samples]
-- interval: 5m
-  by: [bar]
-  outputs: [sum_samples]
-`, "111")
-
-	// min and max outputs
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_max 5
-bar:1m_min 5
-foo:1m_max{abc="123"} 8.5
-foo:1m_max{abc="456",de="fg"} 8
-foo:1m_min{abc="123"} 4
-foo:1m_min{abc="456",de="fg"} 8
-`, `
-- interval: 1m
-  outputs: [min, max]
-`, "1111")
-
-	// avg output
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_avg 5
-foo:1m_avg{abc="123"} 6.25
-foo:1m_avg{abc="456",de="fg"} 8
-`, `
-- interval: 1m
-  outputs: [avg]
-`, "1111")
-
-	// stddev output
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_stddev 0
-foo:1m_stddev{abc="123"} 2.25
-foo:1m_stddev{abc="456",de="fg"} 0
-`, `
-- interval: 1m
-  outputs: [stddev]
-`, "1111")
-
-	// stdvar output
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar:1m_stdvar 0
-foo:1m_stdvar{abc="123"} 5.0625
-foo:1m_stdvar{abc="456",de="fg"} 0
-`, `
-- interval: 1m
-  outputs: [stdvar]
-`, "1111")
-
-	// histogram_bucket output
-	f([]string{`
-cpu_usage{cpu="1"} 12.5
-cpu_usage{cpu="1"} 13.3
-cpu_usage{cpu="1"} 13
-cpu_usage{cpu="1"} 12
-cpu_usage{cpu="1"} 14
-cpu_usage{cpu="1"} 25
-cpu_usage{cpu="2"} 90
-`}, time.Minute, `cpu_usage:1m_histogram_bucket{cpu="1",vmrange="1.136e+01...1.292e+01"} 2
-cpu_usage:1m_histogram_bucket{cpu="1",vmrange="1.292e+01...1.468e+01"} 3
-cpu_usage:1m_histogram_bucket{cpu="1",vmrange="2.448e+01...2.783e+01"} 1
-cpu_usage:1m_histogram_bucket{cpu="2",vmrange="8.799e+01...1.000e+02"} 1
-`, `
-- interval: 1m
-  outputs: [histogram_bucket]
-`, "1111111")
-
-	// histogram_bucket output without cpu
-	f([]string{`
-cpu_usage{cpu="1"} 12.5
-cpu_usage{cpu="1"} 13.3
-cpu_usage{cpu="1"} 13
-cpu_usage{cpu="1"} 12
-cpu_usage{cpu="1"} 14
-cpu_usage{cpu="1"} 25
-cpu_usage{cpu="2"} 90
-`}, time.Minute, `cpu_usage:1m_without_cpu_histogram_bucket{vmrange="1.136e+01...1.292e+01"} 2
-cpu_usage:1m_without_cpu_histogram_bucket{vmrange="1.292e+01...1.468e+01"} 3
-cpu_usage:1m_without_cpu_histogram_bucket{vmrange="2.448e+01...2.783e+01"} 1
-cpu_usage:1m_without_cpu_histogram_bucket{vmrange="8.799e+01...1.000e+02"} 1
-`, `
-- interval: 1m
-  without: [cpu]
-  outputs: [histogram_bucket]
-`, "1111111")
-
-	// quantiles output
-	f([]string{`
-cpu_usage{cpu="1"} 12.5
-cpu_usage{cpu="1"} 13.3
-cpu_usage{cpu="1"} 13
-cpu_usage{cpu="1"} 12
-cpu_usage{cpu="1"} 14
-cpu_usage{cpu="1"} 25
-cpu_usage{cpu="2"} 90
-`}, time.Minute, `cpu_usage:1m_quantiles{cpu="1",quantile="0"} 12
-cpu_usage:1m_quantiles{cpu="1",quantile="0.5"} 13.3
-cpu_usage:1m_quantiles{cpu="1",quantile="1"} 25
-cpu_usage:1m_quantiles{cpu="2",quantile="0"} 90
-cpu_usage:1m_quantiles{cpu="2",quantile="0.5"} 90
-cpu_usage:1m_quantiles{cpu="2",quantile="1"} 90
-`, `
-- interval: 1m
-  outputs: ["quantiles(0, 0.5, 1)"]
-`, "1111111")
-
-	// quantiles output without cpu
-	f([]string{`
-cpu_usage{cpu="1"} 12.5
-cpu_usage{cpu="1"} 13.3
-cpu_usage{cpu="1"} 13
-cpu_usage{cpu="1"} 12
-cpu_usage{cpu="1"} 14
-cpu_usage{cpu="1"} 25
-cpu_usage{cpu="2"} 90
-`}, time.Minute, `cpu_usage:1m_without_cpu_quantiles{quantile="0"} 12
-cpu_usage:1m_without_cpu_quantiles{quantile="0.5"} 13.3
-cpu_usage:1m_without_cpu_quantiles{quantile="1"} 90
-`, `
-- interval: 1m
-  without: [cpu]
-  outputs: ["quantiles(0, 0.5, 1)"]
-`, "1111111")
-
-	// append additional label
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5 10
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar-1m-without-abc-count-samples{new_label="must_keep_metric_name"} 1
-bar-1m-without-abc-count-series{new_label="must_keep_metric_name"} 1
-bar-1m-without-abc-sum-samples{new_label="must_keep_metric_name"} 5
-foo-1m-without-abc-count-samples{new_label="must_keep_metric_name"} 2
-foo-1m-without-abc-count-series{new_label="must_keep_metric_name"} 1
-foo-1m-without-abc-sum-samples{new_label="must_keep_metric_name"} 12.5
-`, `
-- interval: 1m
-  without: [abc]
-  outputs: [count_samples, sum_samples, count_series]
-  output_relabel_configs:
-  - action: replace_all
-    source_labels: [__name__]
-    regex: ":|_"
-    replacement: "-"
-    target_label: __name__
-  - action: drop
-    source_labels: [de]
-    regex: fg
-  - target_label: new_label
-    replacement: must_keep_metric_name
-`, "1111")
-
-	// test rate_sum and rate_avg
-	f([]string{`
-foo{abc="123", cde="1"} 3
-foo{abc="456", cde="1"} 8.5
-foo 12 34
-`, `
-foo{abc="123", cde="1"} 8
-foo{abc="456", cde="1"} 11
-`}, time.Minute, `foo:1m_by_cde_rate_avg{cde="1"} 0.0625
-foo:1m_by_cde_rate_sum{cde="1"} 0.125
-`, `
-- interval: 1m
-  by: [cde]
-  outputs: [rate_sum, rate_avg]
-`, "11111")
-
-	// test rate_sum and rate_avg, when two aggregation intervals are empty
-	f([]string{`
-foo{abc="123", cde="1"} 2
-foo{abc="456", cde="1"} 8
-foo{abc="777", cde="1"} 9 -10
-`, ``, ``, `
-foo{abc="123", cde="1"} 20
-foo{abc="456", cde="1"} 26
-foo{abc="777", cde="1"} 27 -10
-`}, time.Minute, `foo:1m_by_cde_rate_avg{cde="1"} 0.1
-foo:1m_by_cde_rate_sum{cde="1"} 0.2
-`, `            
-- interval: 1m
-  by: [cde]
-  outputs: [rate_sum, rate_avg]
-  enable_windows: true
-`, "111111")
-
-	// rate_sum and rate_avg with duplicated events
-	f([]string{`
-foo{abc="123", cde="1"} 4  10
-foo{abc="123", cde="1"} 4  10
-`}, time.Minute, ``, `
-- interval: 1m
-  outputs: [rate_sum, rate_avg]
-`, "11")
-
-	// rate_sum and rate_avg for a single sample
-	f([]string{`
-foo 4  10
-bar 5  10
-`}, time.Minute, ``, `
-- interval: 1m
-  outputs: [rate_sum, rate_avg]
-`, "11")
-
-	// unique_samples output
-	f([]string{`
-foo 1  10
-foo 2  20
-foo 1  10
-foo 2  20
-foo 3  20
-`}, time.Minute, `foo:1m_unique_samples 3
-`, `
-- interval: 1m
-  outputs: [unique_samples]
-`, "11111")
-
-	// keep_metric_names
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-bar -34.3
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar 2
-foo{abc="123"} 2
-foo{abc="456",de="fg"} 1
-`, `
-- interval: 1m
-  keep_metric_names: true
-  outputs: [count_samples]
-`, "11111")
-
-	// drop_input_labels
-	f([]string{`
-foo{abc="123"} 4
-bar 5
-foo{abc="123"} 8.5
-bar -34.3
-foo{abc="456",de="fg"} 8
-`}, time.Minute, `bar 2
-foo 2
-foo{de="fg"} 1
-`, `
-- interval: 1m
-  drop_input_labels: [abc]
-  keep_metric_names: true
-  outputs: [count_samples]
-`, "11111")
-
-	f([]string{`
-foo 123
-bar 567
-`, ``, ``}, 30*time.Second, `bar:1m_sum_samples 567
-foo:1m_sum_samples 123
-`, `
-- interval: 1m
-  outputs: [sum_samples]
-  dedup_interval: 30s
-`, "11")
-
-	f([]string{`
-foo 123
-bar{baz="qwe"} 1.32
-bar{baz="qwe"} 4.34
-bar{baz="qwe"} 2
-foo{baz="qwe"} -5
-bar{baz="qwer"} 343
-bar{baz="qwer"} 344
-foo{baz="qwe"} 10
-`, ``, ``}, 30*time.Second, `bar:1m_sum_samples{baz="qwe"} 4.34
-bar:1m_sum_samples{baz="qwer"} 344
-foo:1m_sum_samples 123
-foo:1m_sum_samples{baz="qwe"} 10
-`, `
-- interval: 1m
-  dedup_interval: 30s
-  outputs: [sum_samples]
-`, "11111111")
-}
-
 func timeSeriessToString(tss []prompbmarshal.TimeSeries) string {
 	a := make([]string, len(tss))
 	for i, ts := range tss {