lib/streamaggr: made input key compression optional

Automated changes by
[create-pull-request](https://github.com/peter-evans/create-pull-request)
GitHub action

Signed-off-by: Github Actions <133988544+victoriametrics-bot@users.noreply.github.com>
Co-authored-by: AndrewChubatiuk <3162380+AndrewChubatiuk@users.noreply.github.com>
Co-authored-by: Hui Wang <haley@victoriametrics.com>

app/victoria-logs/main.go

@@ -92,6 +92,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if vlselect.RequestHandler(w, r) {
 		return true
 	}
+	if vlstorage.RequestHandler(w, r) {
+		return true
+	}
 	return false
 }
 

app/vlinsert/elasticsearch/elasticsearch_test.go

@@ -86,10 +86,10 @@ func TestReadBulkRequest_Success(t *testing.T) {
 	msgField := "message"
 	rowsExpected := 4
 	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000, 1686026893000000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}
-{"_msg":"qwe rty","@timestamp":""}`
+	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"_msg":"baz"}
+{"_msg":"xyz","x":"y"}
+{"_msg":"qwe rty"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlinsert/jsonline/jsonline_test.go

@@ -30,9 +30,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	msgField := "message"
 	rowsExpected := 3
 	timestampsExpected := []int64{1686026891735000000, 1686023292735000000, 1686026893735000000}
-	resultExpected := `{"@timestamp":"","log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
-{"@timestamp":"","_msg":"baz"}
-{"_msg":"xyz","@timestamp":"","x":"y"}`
+	resultExpected := `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"_msg":"baz"}
+{"_msg":"xyz","x":"y"}`
 	f(data, timeField, msgField, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlinsert/syslog/syslog_test.go

@@ -101,9 +101,9 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	currentYear := 2023
 	rowsExpected := 3
 	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
-	resultExpected := `{"format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility":"20","severity":"5","format":"rfc3164","timestamp":"","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility":"15","severity":"3","format":"rfc5424","timestamp":"","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
+{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
 	f(data, currentYear, rowsExpected, timestampsExpected, resultExpected)
 }
 

app/vlselect/logsql/logsql.go

@@ -394,7 +394,9 @@ func ProcessLiveTailRequest(ctx context.Context, w http.ResponseWriter, r *http.
 		return
 	}
 	if !q.CanLiveTail() {
-		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
+		httpserver.Errorf(w, r, "the query [%s] cannot be used in live tailing; "+
+			"see https://docs.victoriametrics.com/victorialogs/querying/#live-tailing for details", q)
+		return
 	}
 	q.Optimize()
 

app/vlselect/logsql/query_response.qtpl

@@ -6,15 +6,31 @@
 
 // JSONRow creates JSON row from the given fields.
 {% func JSONRow(columns []logstorage.BlockColumn, rowIdx int) %}
-{
-	{% code c := &columns[0] %}
+	{% code
+		i := 0
+		for i < len(columns) && columns[i].Values[rowIdx] == "" {
+			i++
+		}
+		columns = columns[i:]
+	%}
+	{% if len(columns) == 0 %}
+		{% return %}
+	{% endif %}
+	{
+	{% code	c := &columns[0] %}
 	{%q= c.Name %}:{%q= c.Values[rowIdx] %}
 	{% code columns = columns[1:] %}
 	{% for colIdx := range columns %}
-		{% code c := &columns[colIdx] %}
+		{% code
+			c := &columns[colIdx]
+			v := c.Values[rowIdx]
+		%}
+		{% if v == "" %}
+			{% continue %}
+		{% endif %}
 		,{%q= c.Name %}:{%q= c.Values[rowIdx] %}
 	{% endfor %}
-}{% newline %}
+	}{% newline %}
 {% endfunc %}
 
 // JSONRows prints formatted rows
@@ -23,7 +39,11 @@
 		{% return %}
 	{% endif %}
 	{% for _, fields := range rows %}
-	{
+		{% code fields = logstorage.SkipLeadingFieldsWithoutValues(fields) %}
+		{% if len(fields) == 0 %}
+			{% continue %}
+		{% endif %}
+		{
 		{% if len(fields) > 0 %}
 			{% code
 				f := fields[0]
@@ -31,10 +51,13 @@
 			%}
 			{%q= f.Name %}:{%q= f.Value %}
 			{% for _, f := range fields %}
+				{% if f.Value == "" %}
+					{% continue %}
+				{% endif %}
 				,{%q= f.Name %}:{%q= f.Value %}
 			{% endfor %}
 		{% endif %}
-	}{% newline %}
+		}{% newline %}
 	{% endfor %}
 {% endfunc %}
 

app/vlselect/logsql/query_response.qtpl.go

@@ -26,141 +26,176 @@ var (
 
 //line app/vlselect/logsql/query_response.qtpl:8
 func StreamJSONRow(qw422016 *qt422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:8
-	qw422016.N().S(`{`)
 //line app/vlselect/logsql/query_response.qtpl:10
+	i := 0
+	for i < len(columns) && columns[i].Values[rowIdx] == "" {
+		i++
+	}
+	columns = columns[i:]
+
+//line app/vlselect/logsql/query_response.qtpl:16
+	if len(columns) == 0 {
+//line app/vlselect/logsql/query_response.qtpl:17
+		return
+//line app/vlselect/logsql/query_response.qtpl:18
+	}
+//line app/vlselect/logsql/query_response.qtpl:18
+	qw422016.N().S(`{`)
+//line app/vlselect/logsql/query_response.qtpl:20
 	c := &columns[0]
 
-//line app/vlselect/logsql/query_response.qtpl:11
+//line app/vlselect/logsql/query_response.qtpl:21
 	qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:11
+//line app/vlselect/logsql/query_response.qtpl:21
 	qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:11
+//line app/vlselect/logsql/query_response.qtpl:21
 	qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:12
+//line app/vlselect/logsql/query_response.qtpl:22
 	columns = columns[1:]
 
-//line app/vlselect/logsql/query_response.qtpl:13
+//line app/vlselect/logsql/query_response.qtpl:23
 	for colIdx := range columns {
-//line app/vlselect/logsql/query_response.qtpl:14
+//line app/vlselect/logsql/query_response.qtpl:25
 		c := &columns[colIdx]
+		v := c.Values[rowIdx]
 
-//line app/vlselect/logsql/query_response.qtpl:14
+//line app/vlselect/logsql/query_response.qtpl:28
+		if v == "" {
+//line app/vlselect/logsql/query_response.qtpl:29
+			continue
+//line app/vlselect/logsql/query_response.qtpl:30
+		}
+//line app/vlselect/logsql/query_response.qtpl:30
 		qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:15
+//line app/vlselect/logsql/query_response.qtpl:31
 		qw422016.N().Q(c.Name)
-//line app/vlselect/logsql/query_response.qtpl:15
+//line app/vlselect/logsql/query_response.qtpl:31
 		qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:15
+//line app/vlselect/logsql/query_response.qtpl:31
 		qw422016.N().Q(c.Values[rowIdx])
-//line app/vlselect/logsql/query_response.qtpl:16
+//line app/vlselect/logsql/query_response.qtpl:32
 	}
-//line app/vlselect/logsql/query_response.qtpl:16
+//line app/vlselect/logsql/query_response.qtpl:32
 	qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:17
+//line app/vlselect/logsql/query_response.qtpl:33
 	qw422016.N().S(`
 `)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 }
 
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 func WriteJSONRow(qq422016 qtio422016.Writer, columns []logstorage.BlockColumn, rowIdx int) {
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	StreamJSONRow(qw422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 }
 
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 func JSONRow(columns []logstorage.BlockColumn, rowIdx int) string {
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	WriteJSONRow(qb422016, columns, rowIdx)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:18
+//line app/vlselect/logsql/query_response.qtpl:34
 }
 
 // JSONRows prints formatted rows
 
-//line app/vlselect/logsql/query_response.qtpl:21
+//line app/vlselect/logsql/query_response.qtpl:37
 func StreamJSONRows(qw422016 *qt422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:22
+//line app/vlselect/logsql/query_response.qtpl:38
 	if len(rows) == 0 {
-//line app/vlselect/logsql/query_response.qtpl:23
+//line app/vlselect/logsql/query_response.qtpl:39
 		return
-//line app/vlselect/logsql/query_response.qtpl:24
+//line app/vlselect/logsql/query_response.qtpl:40
 	}
-//line app/vlselect/logsql/query_response.qtpl:25
+//line app/vlselect/logsql/query_response.qtpl:41
 	for _, fields := range rows {
-//line app/vlselect/logsql/query_response.qtpl:25
+//line app/vlselect/logsql/query_response.qtpl:42
+		fields = logstorage.SkipLeadingFieldsWithoutValues(fields)
+
+//line app/vlselect/logsql/query_response.qtpl:43
+		if len(fields) == 0 {
+//line app/vlselect/logsql/query_response.qtpl:44
+			continue
+//line app/vlselect/logsql/query_response.qtpl:45
+		}
+//line app/vlselect/logsql/query_response.qtpl:45
 		qw422016.N().S(`{`)
-//line app/vlselect/logsql/query_response.qtpl:27
+//line app/vlselect/logsql/query_response.qtpl:47
 		if len(fields) > 0 {
-//line app/vlselect/logsql/query_response.qtpl:29
+//line app/vlselect/logsql/query_response.qtpl:49
 			f := fields[0]
 			fields = fields[1:]
 
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:52
 			qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:52
 			qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:32
+//line app/vlselect/logsql/query_response.qtpl:52
 			qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:33
+//line app/vlselect/logsql/query_response.qtpl:53
 			for _, f := range fields {
-//line app/vlselect/logsql/query_response.qtpl:33
+//line app/vlselect/logsql/query_response.qtpl:54
+				if f.Value == "" {
+//line app/vlselect/logsql/query_response.qtpl:55
+					continue
+//line app/vlselect/logsql/query_response.qtpl:56
+				}
+//line app/vlselect/logsql/query_response.qtpl:56
 				qw422016.N().S(`,`)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:57
 				qw422016.N().Q(f.Name)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:57
 				qw422016.N().S(`:`)
-//line app/vlselect/logsql/query_response.qtpl:34
+//line app/vlselect/logsql/query_response.qtpl:57
 				qw422016.N().Q(f.Value)
-//line app/vlselect/logsql/query_response.qtpl:35
+//line app/vlselect/logsql/query_response.qtpl:58
 			}
-//line app/vlselect/logsql/query_response.qtpl:36
+//line app/vlselect/logsql/query_response.qtpl:59
 		}
-//line app/vlselect/logsql/query_response.qtpl:36
+//line app/vlselect/logsql/query_response.qtpl:59
 		qw422016.N().S(`}`)
-//line app/vlselect/logsql/query_response.qtpl:37
+//line app/vlselect/logsql/query_response.qtpl:60
 		qw422016.N().S(`
 `)
-//line app/vlselect/logsql/query_response.qtpl:38
+//line app/vlselect/logsql/query_response.qtpl:61
 	}
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 }
 
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 func WriteJSONRows(qq422016 qtio422016.Writer, rows [][]logstorage.Field) {
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	StreamJSONRows(qw422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	qt422016.ReleaseWriter(qw422016)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 }
 
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 func JSONRows(rows [][]logstorage.Field) string {
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	WriteJSONRows(qb422016, rows)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	qs422016 := string(qb422016.B)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 	return qs422016
-//line app/vlselect/logsql/query_response.qtpl:39
+//line app/vlselect/logsql/query_response.qtpl:62
 }

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.cbbca000.css",
-    "main.js": "./static/js/main.3d2eb957.js",
+    "main.css": "./static/css/main.faf86aa5.css",
+    "main.js": "./static/js/main.2810cc52.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.cbbca000.css",
-    "static/js/main.3d2eb957.js"
+    "static/css/main.faf86aa5.css",
+    "static/js/main.2810cc52.js"
   ]
 }

app/vlselect/vmui/config.json

@@ -0,0 +1,5 @@
+{
+  "license": {
+    "type": "opensource"
+  }
+}

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.3d2eb957.js"></script><link href="./static/css/main.cbbca000.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.2810cc52.js"></script><link href="./static/css/main.faf86aa5.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlstorage/main.go

@@ -18,12 +18,12 @@ import (
 )
 
 var (
-	retentionPeriod = flagutil.NewDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
+	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "7d", "Log entries with timestamps older than now-retentionPeriod are automatically deleted; "+
 		"log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); "+
 		"see https://docs.victoriametrics.com/victorialogs/#retention ; see also -retention.maxDiskSpaceUsageBytes")
 	maxDiskSpaceUsageBytes = flagutil.NewBytes("retention.maxDiskSpaceUsageBytes", 0, "The maximum disk space usage at -storageDataPath before older per-day "+
 		"partitions are automatically dropped; see https://docs.victoriametrics.com/victorialogs/#retention-by-disk-space-usage ; see also -retentionPeriod")
-	futureRetention = flagutil.NewDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
+	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; "+
 		"see https://docs.victoriametrics.com/victorialogs/#retention")
 	storageDataPath = flag.String("storageDataPath", "victoria-logs-data", "Path to directory where to store VictoriaLogs data; "+
 		"see https://docs.victoriametrics.com/victorialogs/#storage")
@@ -37,6 +37,8 @@ var (
 		"see https://docs.victoriametrics.com/victorialogs/data-ingestion/ ; see also -logNewStreams")
 	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
 		"the storage stops accepting new data")
+
+	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 )
 
 // Init initializes vlstorage.
@@ -87,6 +89,28 @@ func Stop() {
 	strg = nil
 }
 
+// RequestHandler is a storage request handler.
+func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	path := r.URL.Path
+	if path == "/internal/force_merge" {
+		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
+			return true
+		}
+		// Run force merge in background
+		partitionNamePrefix := r.FormValue("partition_prefix")
+		go func() {
+			activeForceMerges.Inc()
+			defer activeForceMerges.Dec()
+			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
+			startTime := time.Now()
+			strg.MustForceMerge(partitionNamePrefix)
+			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
+		}()
+		return true
+	}
+	return false
+}
+
 var strg *logstorage.Storage
 var storageMetrics *metrics.Set
 
@@ -205,3 +229,5 @@ func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_big_timestamp"}`, ss.RowsDroppedTooBigTimestamp)
 	metrics.WriteCounterUint64(w, `vl_rows_dropped_total{reason="too_small_timestamp"}`, ss.RowsDroppedTooSmallTimestamp)
 }
+
+var activeForceMerges = metrics.NewCounter("vl_active_force_merges")

app/vmagent/influx/request_handler.go

@@ -36,7 +36,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(at *auth.Token, r io.Reader, isGzipped bool) error {
-	return stream.Parse(r, isGzipped, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, true, isGzipped, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, nil)
 	})
 }
@@ -50,11 +50,12 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	isStreamMode := req.Header.Get("Stream-Mode") == "1"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(at, db, rows, extraLabels)
 	})
 }

app/vmagent/remotewrite/streamaggr.go

@@ -24,7 +24,7 @@ var (
 	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation/")
-	streamAggrGlobalDedupInterval = flagutil.NewDuration("streamAggr.dedupInterval", "0s", "Input samples are de-duplicated with this interval on "+
+	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
 		"aggregator before optional aggregation with -streamAggr.config . "+
 		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/stream-aggregation/#deduplication")
 	streamAggrGlobalIgnoreOldSamples = flag.Bool("streamAggr.ignoreOldSamples", false, "Whether to ignore input samples with old timestamps outside the "+
@@ -133,7 +133,7 @@ func initStreamAggrConfigGlobal() {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
 	}
-	dedupInterval := streamAggrGlobalDedupInterval.Duration()
+	dedupInterval := *streamAggrGlobalDedupInterval
 	if dedupInterval > 0 {
 		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 	}
@@ -202,7 +202,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 	}
 
 	opts := &streamaggr.Options{
-		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
+		DedupInterval:        *streamAggrGlobalDedupInterval,
 		DropInputLabels:      *streamAggrGlobalDropInputLabels,
 		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,

app/vmalert-tool/unittest/input.go

@@ -43,18 +43,33 @@ func httpWrite(address string, r io.Reader) {
 // writeInputSeries send input series to vmstorage and flush them
 func writeInputSeries(input []series, interval *promutils.Duration, startStamp time.Time, dst string) error {
 	r := testutil.WriteRequest{}
+	var err error
+	r.Timeseries, err = parseInputSeries(input, interval, startStamp)
+	if err != nil {
+		return err
+	}
+
+	data := testutil.Compress(r)
+	// write input series to vm
+	httpWrite(dst, bytes.NewBuffer(data))
+	vmstorage.Storage.DebugFlush()
+	return nil
+}
+
+func parseInputSeries(input []series, interval *promutils.Duration, startStamp time.Time) ([]testutil.TimeSeries, error) {
+	var res []testutil.TimeSeries
 	for _, data := range input {
 		expr, err := metricsql.Parse(data.Series)
 		if err != nil {
-			return fmt.Errorf("failed to parse series %s: %v", data.Series, err)
+			return res, fmt.Errorf("failed to parse series %s: %v", data.Series, err)
 		}
 		promvals, err := parseInputValue(data.Values, true)
 		if err != nil {
-			return fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
+			return res, fmt.Errorf("failed to parse input series value %s: %v", data.Values, err)
 		}
 		metricExpr, ok := expr.(*metricsql.MetricExpr)
-		if !ok {
-			return fmt.Errorf("failed to parse series %s to metric expr: %v", data.Series, err)
+		if !ok || len(metricExpr.LabelFilterss) != 1 {
+			return res, fmt.Errorf("got invalid input series %s: %v", data.Series, err)
 		}
 		samples := make([]testutil.Sample, 0, len(promvals))
 		ts := startStamp
@@ -71,14 +86,9 @@ func writeInputSeries(input []series, interval *promutils.Duration, startStamp t
 		for _, filter := range metricExpr.LabelFilterss[0] {
 			ls = append(ls, testutil.Label{Name: filter.Label, Value: filter.Value})
 		}
-		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
+		res = append(res, testutil.TimeSeries{Labels: ls, Samples: samples})
 	}
-
-	data := testutil.Compress(r)
-	// write input series to vm
-	httpWrite(dst, bytes.NewBuffer(data))
-	vmstorage.Storage.DebugFlush()
-	return nil
+	return res, nil
 }
 
 // parseInputValue support input like "1", "1+1x1 _ -4 3+20x1", see more examples in test.

app/vmalert-tool/unittest/input_test.go

@@ -2,8 +2,10 @@ package unittest
 
 import (
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestParseInputValue_Failure(t *testing.T) {
@@ -43,7 +45,7 @@ func TestParseInputValue_Success(t *testing.T) {
 				if decimal.IsStaleNaN(outputExpected[i].Value) && decimal.IsStaleNaN(output[i].Value) {
 					continue
 				}
-				t.Fatalf("unexpeccted Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
+				t.Fatalf("unexpected Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}
 		}
 	}
@@ -64,3 +66,34 @@ func TestParseInputValue_Success(t *testing.T) {
 
 	f("1+1x1 _ -4 stale 3+20x1", []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}})
 }
+
+func TestParseInputSeries_Success(t *testing.T) {
+	f := func(input []series) {
+		t.Helper()
+		var interval promutils.Duration
+		_, err := parseInputSeries(input, &interval, time.Now())
+		if err != nil {
+			t.Fatalf("expect to see no error: %v", err)
+		}
+	}
+
+	f([]series{{Series: "test", Values: "1"}})
+	f([]series{{Series: "test{}", Values: "1"}})
+	f([]series{{Series: "test{env=\"prod\",job=\"a\" }", Values: "1"}})
+	f([]series{{Series: "{__name__=\"test\",env=\"prod\",job=\"a\" }", Values: "1"}})
+}
+
+func TestParseInputSeries_Fail(t *testing.T) {
+	f := func(input []series) {
+		t.Helper()
+		var interval promutils.Duration
+		_, err := parseInputSeries(input, &interval, time.Now())
+		if err == nil {
+			t.Fatalf("expect to see error: %v", err)
+		}
+	}
+
+	f([]series{{Series: "", Values: "1"}})
+	f([]series{{Series: "{}", Values: "1"}})
+	f([]series{{Series: "{env=\"prod\",job=\"a\" or env=\"dev\",job=\"b\"}", Values: "1"}})
+}

app/vmalert-tool/unittest/recording.go

@@ -57,16 +57,18 @@ Outer:
 					continue Outer
 				}
 				metricsqlMetricExpr, ok := metricsqlExpr.(*metricsql.MetricExpr)
-				if !ok {
+				if !ok || len(metricsqlMetricExpr.LabelFilterss) > 1 {
 					checkErrs = append(checkErrs, fmt.Errorf("\n    expr: %q, time: %s, err: %v", mt.Expr,
-						mt.EvalTime.Duration().String(), fmt.Errorf("got unsupported metricsql type")))
+						mt.EvalTime.Duration().String(), fmt.Errorf("got invalid exp_samples: %q", s.Labels)))
 					continue Outer
 				}
-				for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
-					expLb = append(expLb, datasource.Label{
-						Name:  l.Label,
-						Value: l.Value,
-					})
+				if len(metricsqlMetricExpr.LabelFilterss) > 0 {
+					for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
+						expLb = append(expLb, datasource.Label{
+							Name:  l.Label,
+							Value: l.Value,
+						})
+					}
 				}
 			}
 			sort.Slice(expLb, func(i, j int) bool {

app/vmalert-tool/unittest/unittest.go

@@ -250,7 +250,7 @@ checkCheck:
 			if readyCheckFunc() {
 				break checkCheck
 			}
-			time.Sleep(3 * time.Second)
+			time.Sleep(100 * time.Millisecond)
 		}
 	}
 }

app/vmalert/remotewrite/client.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/golang/snappy"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -22,11 +23,12 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 )
 
+var defaultConcurrency = cgroup.AvailableCPUs() * 2
+
 const (
-	defaultConcurrency   = 4
-	defaultMaxBatchSize  = 1e3
-	defaultMaxQueueSize  = 1e5
-	defaultFlushInterval = 5 * time.Second
+	defaultMaxBatchSize  = 1e4
+	defaultMaxQueueSize  = 1e6
+	defaultFlushInterval = 2 * time.Second
 	defaultWriteTimeout  = 30 * time.Second
 )
 

app/vmalert/remotewrite/init.go

@@ -34,10 +34,10 @@ var (
 
 	idleConnectionTimeout = flag.Duration("remoteWrite.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider settings this value less to the value of "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
 
-	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", 1e6, "Defines the max number of pending datapoints to remote write endpoint")
-	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", 1e4, "Defines max number of timeseries to be flushed at once")
-	concurrency   = flag.Int("remoteWrite.concurrency", 4, "Defines number of writers for concurrent writing into remote write endpoint")
-	flushInterval = flag.Duration("remoteWrite.flushInterval", 5*time.Second, "Defines interval of flushes to remote write endpoint")
+	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
+	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
+	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint")
+	flushInterval = flag.Duration("remoteWrite.flushInterval", defaultFlushInterval, "Defines interval of flushes to remote write endpoint")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")
 	tlsCertFile           = flag.String("remoteWrite.tlsCertFile", "", "Optional path to client-side TLS certificate file to use when connecting to -remoteWrite.url")

app/vmalert/rule/group.go

@@ -324,14 +324,28 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 		g.infof("will start in %v", sleepBeforeStart)
 
 		sleepTimer := time.NewTimer(sleepBeforeStart)
-		select {
-		case <-ctx.Done():
-			sleepTimer.Stop()
-			return
-		case <-g.doneCh:
-			sleepTimer.Stop()
-			return
-		case <-sleepTimer.C:
+	randSleep:
+		for {
+			select {
+			case <-ctx.Done():
+				sleepTimer.Stop()
+				return
+			case <-g.doneCh:
+				sleepTimer.Stop()
+				return
+			case ng := <-g.updateCh:
+				g.mu.Lock()
+				err := g.updateWith(ng)
+				if err != nil {
+					logger.Errorf("group %q: failed to update: %s", g.Name, err)
+					g.mu.Unlock()
+					continue
+				}
+				g.mu.Unlock()
+				g.infof("reload successfully")
+			case <-sleepTimer.C:
+				break randSleep
+			}
 		}
 		evalTS = evalTS.Add(sleepBeforeStart)
 	}

app/vmalert/rule/group_test.go

@@ -175,6 +175,74 @@ func TestUpdateWith(t *testing.T) {
 	})
 }
 
+func TestUpdateDuringRandSleep(t *testing.T) {
+	// enable rand sleep to test group update during sleep
+	SkipRandSleepOnGroupStart = false
+	defer func() {
+		SkipRandSleepOnGroupStart = true
+	}()
+	rule := AlertingRule{
+		Name: "jobDown",
+		Expr: "up==0",
+		Labels: map[string]string{
+			"foo": "bar",
+		},
+	}
+	g := &Group{
+		Name: "test",
+		Rules: []Rule{
+			&rule,
+		},
+		// big interval ensures big enough randSleep during start process
+		Interval: 100 * time.Hour,
+		updateCh: make(chan *Group),
+	}
+	go g.Start(context.Background(), nil, nil, nil)
+
+	rule1 := AlertingRule{
+		Name: "jobDown",
+		Expr: "up{job=\"vmagent\"}==0",
+		Labels: map[string]string{
+			"foo": "bar",
+		},
+	}
+	g1 := &Group{
+		Rules: []Rule{
+			&rule1,
+		},
+	}
+	g.updateCh <- g1
+	time.Sleep(10 * time.Millisecond)
+	g.mu.RLock()
+	if g.Rules[0].(*AlertingRule).Expr != "up{job=\"vmagent\"}==0" {
+		t.Fatalf("expected to have updated rule expr")
+	}
+	g.mu.RUnlock()
+
+	rule2 := AlertingRule{
+		Name: "jobDown",
+		Expr: "up{job=\"vmagent\"}==0",
+		Labels: map[string]string{
+			"foo": "bar",
+			"baz": "qux",
+		},
+	}
+	g2 := &Group{
+		Rules: []Rule{
+			&rule2,
+		},
+	}
+	g.updateCh <- g2
+	time.Sleep(10 * time.Millisecond)
+	g.mu.RLock()
+	if len(g.Rules[0].(*AlertingRule).Labels) != 2 {
+		t.Fatalf("expected to have updated labels")
+	}
+	g.mu.RUnlock()
+
+	g.Close()
+}
+
 func TestGroupStart(t *testing.T) {
 	const (
 		rules = `

app/vmalert/web_types.go

@@ -215,8 +215,10 @@ func recordingToAPI(rr *rule.RecordingRule) apiRule {
 		Updates:           rule.GetAllRuleState(rr),
 
 		// encode as strings to avoid rounding
-		ID:      fmt.Sprintf("%d", rr.ID()),
-		GroupID: fmt.Sprintf("%d", rr.GroupID),
+		ID:        fmt.Sprintf("%d", rr.ID()),
+		GroupID:   fmt.Sprintf("%d", rr.GroupID),
+		GroupName: rr.GroupName,
+		File:      rr.File,
 	}
 	if lastState.Err != nil {
 		r.LastError = lastState.Err.Error()

app/vmalert/web_types_test.go

@@ -1,9 +1,56 @@
 package main
 
 import (
+	"fmt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
+	"reflect"
 	"testing"
 )
 
+func TestRecordingToApi(t *testing.T) {
+	fq := &datasource.FakeQuerier{}
+	fq.Add(datasource.Metric{
+		Values: []float64{1}, Timestamps: []int64{0},
+	})
+	g := &rule.Group{
+		Name:        "group",
+		File:        "rules.yaml",
+		Concurrency: 1,
+	}
+
+	entriesLimit := 44
+	rr := rule.NewRecordingRule(fq, g, config.Rule{
+		ID:                 1248,
+		Record:             "record_name",
+		Expr:               "up",
+		Labels:             map[string]string{"label": "value"},
+		UpdateEntriesLimit: &entriesLimit,
+	})
+
+	expectedRes := apiRule{
+		Name:           "record_name",
+		Query:          "up",
+		Labels:         map[string]string{"label": "value"},
+		Health:         "ok",
+		Type:           ruleTypeRecording,
+		DatasourceType: "prometheus",
+		ID:             "1248",
+		GroupID:        fmt.Sprintf("%d", g.ID()),
+		GroupName:      "group",
+		File:           "rules.yaml",
+		MaxUpdates:     44,
+		Updates:        make([]rule.StateEntry, 0),
+	}
+
+	res := recordingToAPI(rr)
+
+	if !reflect.DeepEqual(res, expectedRes) {
+		t.Fatalf("expected to have: \n%v;\ngot: \n%v", expectedRes, res)
+	}
+}
+
 func TestUrlValuesToStrings(t *testing.T) {
 	mapQueryParams := map[string][]string{
 		"param1": {"param1"},

app/vmctl/opentsdb.go

@@ -158,7 +158,7 @@ func (op *otsdbProcessor) do(s queryObj) error {
 	if len(data.Timestamps) < 1 || len(data.Values) < 1 {
 		return nil
 	}
-	labels := make([]vm.LabelPair, len(data.Tags))
+	labels := make([]vm.LabelPair, 0, len(data.Tags))
 	for k, v := range data.Tags {
 		labels = append(labels, vm.LabelPair{Name: k, Value: v})
 	}

app/vminsert/influx/request_handler.go

@@ -34,7 +34,7 @@ var (
 //
 // See https://github.com/influxdata/telegraf/tree/master/plugins/inputs/socket_listener/
 func InsertHandlerForReader(r io.Reader) error {
-	return stream.Parse(r, false, "", "", func(db string, rows []parser.Row) error {
+	return stream.Parse(r, true, false, "", "", func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, nil)
 	})
 }
@@ -48,11 +48,12 @@ func InsertHandlerForHTTP(req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	isStreamMode := req.Header.Get("Stream-Mode") == "1"
 	q := req.URL.Query()
 	precision := q.Get("precision")
 	// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
 	db := q.Get("db")
-	return stream.Parse(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
+	return stream.Parse(req.Body, isStreamMode, isGzipped, precision, db, func(db string, rows []parser.Row) error {
 		return insertRows(db, rows, extraLabels)
 	})
 }

app/vmselect/main.go

@@ -60,6 +60,7 @@ func Init() {
 	fs.RemoveDirContents(tmpDirPath)
 	netstorage.InitTmpBlocksDir(tmpDirPath)
 	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
+	prometheus.InitMaxUniqueTimeseries(*maxConcurrentRequests)
 
 	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
 	initVMAlertProxy()
@@ -82,6 +83,9 @@ var (
 	_ = metrics.NewGauge(`vm_concurrent_select_current`, func() float64 {
 		return float64(len(concurrencyLimitCh))
 	})
+	_ = metrics.NewGauge(`vm_search_max_unique_timeseries`, func() float64 {
+		return float64(prometheus.GetMaxUniqueTimeSeries())
+	})
 )
 
 //go:embed vmui

app/vmselect/prometheus/prometheus.go

@@ -12,6 +12,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/metricsql"
+	"github.com/valyala/fastjson/fastfloat"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats"
@@ -23,11 +27,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/VictoriaMetrics/metricsql"
-	"github.com/valyala/fastjson/fastfloat"
 )
 
 var (
@@ -47,7 +50,8 @@ var (
 	maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+
 		"points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data")
 
-	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 300e3, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage")
+	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage. "+
+		"When set to zero, the limit is automatically calculated based on -search.maxConcurrentRequests (inversely proportional) and memory available to the process (proportional).")
 	maxFederateSeries   = flag.Int("search.maxFederateSeries", 1e6, "The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage")
 	maxExportSeries     = flag.Int("search.maxExportSeries", 10e6, "The maximum number of time series, which can be returned from /api/v1/export* APIs. This option allows limiting memory usage")
 	maxTSDBStatusSeries = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
@@ -792,7 +796,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		End:                 start,
 		Step:                step,
 		MaxPointsPerSeries:  *maxPointsPerTimeseries,
-		MaxSeries:           *maxUniqueTimeseries,
+		MaxSeries:           GetMaxUniqueTimeSeries(),
 		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
 		Deadline:            deadline,
 		MayCache:            mayCache,
@@ -900,7 +904,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		End:                 end,
 		Step:                step,
 		MaxPointsPerSeries:  *maxPointsPerTimeseries,
-		MaxSeries:           *maxUniqueTimeseries,
+		MaxSeries:           GetMaxUniqueTimeSeries(),
 		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
 		Deadline:            deadline,
 		MayCache:            mayCache,
@@ -1246,3 +1250,40 @@ func (sw *scalableWriter) flush() error {
 	})
 	return sw.bw.Flush()
 }
+
+var (
+	maxUniqueTimeseriesValueOnce sync.Once
+	maxUniqueTimeseriesValue     int
+)
+
+// InitMaxUniqueTimeseries init the max metrics limit calculated by available resources.
+// The calculation is split into calculateMaxUniqueTimeSeriesForResource for unit testing.
+func InitMaxUniqueTimeseries(maxConcurrentRequests int) {
+	maxUniqueTimeseriesValueOnce.Do(func() {
+		maxUniqueTimeseriesValue = *maxUniqueTimeseries
+		if maxUniqueTimeseriesValue <= 0 {
+			maxUniqueTimeseriesValue = calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, memory.Remaining())
+		}
+	})
+}
+
+// calculateMaxUniqueTimeSeriesForResource calculate the max metrics limit calculated by available resources.
+func calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, remainingMemory int) int {
+	if maxConcurrentRequests <= 0 {
+		// This line should NOT be reached unless the user has set an incorrect `search.maxConcurrentRequests`.
+		// In such cases, fallback to unlimited.
+		logger.Warnf("limiting -search.maxUniqueTimeseries to %v because -search.maxConcurrentRequests=%d.", 2e9, maxConcurrentRequests)
+		return 2e9
+	}
+
+	// Calculate the max metrics limit for a single request in the worst-case concurrent scenario.
+	// The approximate size of 1 unique series that could occupy in the vmstorage is 200 bytes.
+	mts := remainingMemory / 200 / maxConcurrentRequests
+	logger.Infof("limiting -search.maxUniqueTimeseries to %d according to -search.maxConcurrentRequests=%d and remaining memory=%d bytes. To increase the limit, reduce -search.maxConcurrentRequests or increase memory available to the process.", mts, maxConcurrentRequests, remainingMemory)
+	return mts
+}
+
+// GetMaxUniqueTimeSeries returns the max metrics limit calculated by available resources.
+func GetMaxUniqueTimeSeries() int {
+	return maxUniqueTimeseriesValue
+}

app/vmselect/prometheus/prometheus_test.go

@@ -4,6 +4,7 @@ import (
 	"math"
 	"net/http"
 	"reflect"
+	"runtime"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
@@ -229,3 +230,29 @@ func TestGetLatencyOffsetMillisecondsFailure(t *testing.T) {
 	}
 	f("http://localhost?latency_offset=foobar")
 }
+
+func TestCalculateMaxMetricsLimitByResource(t *testing.T) {
+	f := func(maxConcurrentRequest, remainingMemory, expect int) {
+		t.Helper()
+		maxMetricsLimit := calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequest, remainingMemory)
+		if maxMetricsLimit != expect {
+			t.Fatalf("unexpected max metrics limit: got %d, want %d", maxMetricsLimit, expect)
+		}
+	}
+
+	// Skip when GOARCH=386
+	if runtime.GOARCH != "386" {
+		// 8 CPU & 32 GiB
+		f(16, int(math.Round(32*1024*1024*1024*0.4)), 4294967)
+		// 4 CPU & 32 GiB
+		f(8, int(math.Round(32*1024*1024*1024*0.4)), 8589934)
+	}
+
+	// 2 CPU & 4 GiB
+	f(4, int(math.Round(4*1024*1024*1024*0.4)), 2147483)
+
+	// other edge cases
+	f(0, int(math.Round(4*1024*1024*1024*0.4)), 2e9)
+	f(4, 0, 0)
+
+}

app/vmselect/promql/eval.go

@@ -44,7 +44,7 @@ var (
 		"See also -search.logSlowQueryDuration and -search.maxMemoryPerQuery")
 	noStaleMarkers = flag.Bool("search.noStaleMarkers", false, "Set this flag to true if the database doesn't contain Prometheus stale markers, "+
 		"so there is no need in spending additional CPU time on its handling. Staleness markers may exist only in data obtained from Prometheus scrape targets")
-	minWindowForInstantRollupOptimization = flagutil.NewDuration("search.minWindowForInstantRollupOptimization", "3h", "Enable cache-based optimization for repeated queries "+
+	minWindowForInstantRollupOptimization = flag.Duration("search.minWindowForInstantRollupOptimization", time.Hour*3, "Enable cache-based optimization for repeated queries "+
 		"to /api/v1/query (aka instant queries), which contain rollup functions with lookbehind window exceeding the given value")
 )
 

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.d871147a.css",
-    "main.js": "./static/js/main.621c4b4d.js",
+    "main.css": "./static/css/main.d781989c.css",
+    "main.js": "./static/js/main.68e2aae8.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.d871147a.css",
-    "static/js/main.621c4b4d.js"
+    "static/css/main.d781989c.css",
+    "static/js/main.68e2aae8.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.621c4b4d.js"></script><link href="./static/css/main.d871147a.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.68e2aae8.js"></script><link href="./static/css/main.d781989c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmstorage/main.go

@@ -27,11 +27,11 @@ import (
 )
 
 var (
-	retentionPeriod   = flagutil.NewDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
+	retentionPeriod   = flagutil.NewRetentionDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
-	snapshotsMaxAge   = flagutil.NewDuration("snapshotsMaxAge", "0", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
+	snapshotsMaxAge   = flagutil.NewRetentionDuration("snapshotsMaxAge", "0", "Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted")
 	_                 = flag.Duration("snapshotCreateTimeout", 0, "Deprecated: this flag does nothing")
 
 	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss")

app/vmui/packages/vmui/public/config.json

@@ -0,0 +1,5 @@
+{
+  "license": {
+    "type": "opensource"
+  }
+}

app/vmui/packages/vmui/src/App.tsx

@@ -15,6 +15,8 @@ import WithTemplate from "./pages/WithTemplate";
 import Relabel from "./pages/Relabel";
 import ActiveQueries from "./pages/ActiveQueries";
 import QueryAnalyzer from "./pages/QueryAnalyzer";
+import DownsamplingFilters from "./pages/DownsamplingFilters";
+import RetentionFilters from "./pages/RetentionFilters";
 
 const App: FC = () => {
   const [loadedTheme, setLoadedTheme] = useState(false);
@@ -74,6 +76,14 @@ const App: FC = () => {
                   path={router.icons}
                   element={<PreviewIcons/>}
                 />
+                <Route
+                  path={router.downsamplingDebug}
+                  element={<DownsamplingFilters/>}
+                />
+                <Route
+                  path={router.retentionDebug}
+                  element={<RetentionFilters/>}
+                />
               </Route>
             </Routes>
           )}

app/vmui/packages/vmui/src/api/downsampling-filters-debug.ts

@@ -0,0 +1,7 @@
+export const getDownsamplingFiltersDebug = (server: string, flags: string, metrics: string): string => {
+  const params = [
+    `flags=${encodeURIComponent(flags)}`,
+    `metrics=${encodeURIComponent(metrics)}`
+  ];
+  return `${server}/downsampling-filters-debug?${params.join("&")}`;
+};

app/vmui/packages/vmui/src/api/retention-filters-debug.ts

@@ -0,0 +1,7 @@
+export const getRetentionFiltersDebug = (server: string, flags: string, metrics: string): string => {
+  const params = [
+    `flags=${encodeURIComponent(flags)}`,
+    `metrics=${encodeURIComponent(metrics)}`
+  ];
+  return `${server}/retention-filters-debug?${params.join("&")}`;
+};

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsChart.tsx

@@ -33,12 +33,15 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
     graphStyle: GRAPH_STYLES.LINE_STEPPED,
     stacked: false,
     fill: false,
+    hideChart: false,
   });
 
   const { xRange, setPlotScale } = usePlotScale({ period, setPeriod });
   const { onReadyChart, isPanning } = useReadyChart(setPlotScale);
   useZoomChart({ uPlotInst, xRange, setPlotScale });
 
+  const isEmptyData = useMemo(() => _data.every(d => d.length === 0), [_data]);
+
   const { data, bands } = useMemo(() => {
     return graphOptions.stacked ? stack(_data, () => false) : { data: _data, bands: [] };
   }, [graphOptions, _data]);
@@ -88,26 +91,33 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
   }, [data]);
 
   return (
-    <div className="vm-bar-hits-chart__wrapper">
-      <div
-        className={classNames({
-          "vm-bar-hits-chart": true,
-          "vm-bar-hits-chart_panning": isPanning
-        })}
-        ref={containerRef}
-      >
+    <div
+      className={classNames({
+        "vm-bar-hits-chart__wrapper": true,
+        "vm-bar-hits-chart__wrapper_hidden": graphOptions.hideChart
+      })}
+    >
+      {!graphOptions.hideChart && (
         <div
-          className="vm-line-chart__u-plot"
-          ref={uPlotRef}
-        />
-        <BarHitsTooltip
-          uPlotInst={uPlotInst}
-          data={_data}
-          focusDataIdx={focusDataIdx}
-        />
-      </div>
+          className={classNames({
+            "vm-bar-hits-chart": true,
+            "vm-bar-hits-chart_panning": isPanning
+          })}
+          ref={containerRef}
+        >
+          <div
+            className="vm-line-chart__u-plot"
+            ref={uPlotRef}
+          />
+          <BarHitsTooltip
+            uPlotInst={uPlotInst}
+            data={_data}
+            focusDataIdx={focusDataIdx}
+          />
+        </div>
+      )}
       <BarHitsOptions onChange={setGraphOptions}/>
-      {uPlotInst && (
+      {uPlotInst && !isEmptyData && !graphOptions.hideChart && (
         <BarHitsLegend
           uPlotInst={uPlotInst}
           onApplyFilter={onApplyFilter}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/BarHitsOptions.tsx

@@ -6,7 +6,7 @@ import useStateSearchParams from "../../../../hooks/useStateSearchParams";
 import { useSearchParams } from "react-router-dom";
 import Button from "../../../Main/Button/Button";
 import classNames from "classnames";
-import { SettingsIcon } from "../../../Main/Icons";
+import { SettingsIcon, VisibilityIcon, VisibilityOffIcon } from "../../../Main/Icons";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import Popper from "../../../Main/Popper/Popper";
 import useBoolean from "../../../../hooks/useBoolean";
@@ -27,12 +27,14 @@ const BarHitsOptions: FC<Props> = ({ onChange }) => {
   const [graphStyle, setGraphStyle] = useStateSearchParams(GRAPH_STYLES.LINE_STEPPED, "graph");
   const [stacked, setStacked] = useStateSearchParams(false, "stacked");
   const [fill, setFill] = useStateSearchParams(false, "fill");
+  const [hideChart, setHideChart] = useStateSearchParams(false, "hide_chart");
 
   const options: GraphOptions = useMemo(() => ({
     graphStyle,
     stacked,
     fill,
-  }), [graphStyle, stacked, fill]);
+    hideChart,
+  }), [graphStyle, stacked, fill, hideChart]);
 
   const handleChangeGraphStyle = (val: string) => () => {
     setGraphStyle(val as GRAPH_STYLES);
@@ -52,24 +54,41 @@ const BarHitsOptions: FC<Props> = ({ onChange }) => {
     setSearchParams(searchParams);
   };
 
+  const toggleHideChart = () => {
+    setHideChart(prev => {
+      const newVal = !prev;
+      newVal ? searchParams.set("hide_chart", "true") : searchParams.delete("hide_chart");
+      setSearchParams(searchParams);
+      return newVal;
+    });
+  };
+
   useEffect(() => {
     onChange(options);
   }, [options]);
 
   return (
-    <div
-      className="vm-bar-hits-options"
-      ref={optionsButtonRef}
-    >
-      <Tooltip title="Graph settings">
+    <div className="vm-bar-hits-options">
+      <Tooltip title={hideChart ? "Show chart and resume hits updates" : "Hide chart and pause hits updates"}>
         <Button
           variant="text"
           color="primary"
-          startIcon={<SettingsIcon/>}
-          onClick={toggleOpenOptions}
+          startIcon={hideChart ? <VisibilityOffIcon/> : <VisibilityIcon/>}
+          onClick={toggleHideChart}
           ariaLabel="settings"
         />
       </Tooltip>
+      <div ref={optionsButtonRef}>
+        <Tooltip title="Graph settings">
+          <Button
+            variant="text"
+            color="primary"
+            startIcon={<SettingsIcon/>}
+            onClick={toggleOpenOptions}
+            ariaLabel="settings"
+          />
+        </Tooltip>
+      </div>
       <Popper
         open={openOptions}
         placement="bottom-right"

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/style.scss

@@ -1,6 +1,8 @@
 @use "src/styles/variables" as *;
 
 .vm-bar-hits-options {
+  display: flex;
+  align-items: center;
   position: absolute;
   top: $padding-small;
   right: $padding-small;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/style.scss

@@ -10,6 +10,10 @@
     flex-direction: column;
     width: 100%;
     height: 100%;
+
+    &_hidden {
+      min-height: 90px;
+    }
   }
 
   &_panning {

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/types.ts

@@ -9,4 +9,5 @@ export interface GraphOptions {
   graphStyle: GRAPH_STYLES;
   stacked: boolean;
   fill: boolean;
+  hideChart: boolean;
 }

app/vmui/packages/vmui/src/components/Main/Alert/style.scss

@@ -51,6 +51,8 @@
   &__content {
     filter: brightness(0.6);
     white-space: pre-line;
+    text-wrap: balance;
+    overflow-wrap: anywhere;
   }
 
   &_success {

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -553,3 +553,20 @@ export const SearchIcon = () => (
     ></path>
   </svg>
 );
+
+export const SpinnerIcon = () => (
+  <svg viewBox="0 0 24 24">
+    <path
+      fill="currentColor"
+      d="M12,4a8,8,0,0,1,7.89,6.7A1.53,1.53,0,0,0,21.38,12h0a1.5,1.5,0,0,0,1.48-1.75,11,11,0,0,0-21.72,0A1.5,1.5,0,0,0,2.62,12h0a1.53,1.53,0,0,0,1.49-1.3A8,8,0,0,1,12,4Z"
+    >
+      <animateTransform
+        attributeName="transform"
+        dur="0.75s"
+        repeatCount="indefinite"
+        type="rotate"
+        values="0 12 12;360 12 12"
+      />
+    </path>
+  </svg>
+);

app/vmui/packages/vmui/src/components/Main/LineLoader/LineLoader.tsx

@@ -0,0 +1,13 @@
+import React, { FC } from "preact/compat";
+import "./style.scss";
+
+const LineLoader: FC = () => {
+  return (
+    <div className="vm-line-loader">
+      <div className="vm-line-loader__background"></div>
+      <div className="vm-line-loader__line"></div>
+    </div>
+  );
+};
+
+export default LineLoader;

app/vmui/packages/vmui/src/components/Main/LineLoader/style.scss

@@ -0,0 +1,39 @@
+@use "src/styles/variables" as *;
+
+.vm-line-loader {
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  height: 2px;
+  z-index: 2;
+  overflow: hidden;
+
+  &__background {
+    position: absolute;
+    left: 0;
+    right: 0;
+    top: 0;
+    bottom: 0;
+    background-color: $color-text;
+    opacity: 0.1;
+  }
+
+  &__line {
+    position: absolute;
+    width: 10%;
+    height: 100%;
+    background-color: $color-primary;
+    animation: slide 2s infinite linear;
+    opacity: 0.8;
+  }
+}
+
+@keyframes slide {
+  0% {
+    left: 0;
+  }
+  100% {
+    left: 100%;
+  }
+}

app/vmui/packages/vmui/src/constants/navigation.ts

@@ -1,81 +0,0 @@
-import router, { routerOptions } from "../router";
-
-export enum NavigationItemType {
-  internalLink,
-  externalLink,
-}
-
-export interface NavigationItem {
-  label?: string,
-  value?: string,
-  hide?: boolean
-  submenu?: NavigationItem[],
-  type?: NavigationItemType,
-}
-
-const explore = {
-  label: "Explore",
-  submenu: [
-    {
-      label: routerOptions[router.metrics].title,
-      value: router.metrics,
-    },
-    {
-      label: routerOptions[router.cardinality].title,
-      value: router.cardinality,
-    },
-    {
-      label: routerOptions[router.topQueries].title,
-      value: router.topQueries,
-    },
-    {
-      label: routerOptions[router.activeQueries].title,
-      value: router.activeQueries,
-    },
-  ]
-};
-
-const tools = {
-  label: "Tools",
-  submenu: [
-    {
-      label: routerOptions[router.trace].title,
-      value: router.trace,
-    },
-    {
-      label: routerOptions[router.queryAnalyzer].title,
-      value: router.queryAnalyzer,
-    },
-    {
-      label: routerOptions[router.withTemplate].title,
-      value: router.withTemplate,
-    },
-    {
-      label: routerOptions[router.relabel].title,
-      value: router.relabel,
-    },
-  ]
-};
-
-export const logsNavigation: NavigationItem[] = [
-  {
-    label: routerOptions[router.logs].title,
-    value: router.home,
-  },
-];
-
-export const anomalyNavigation: NavigationItem[] = [
-  {
-    label: routerOptions[router.anomaly].title,
-    value: router.home,
-  }
-];
-
-export const defaultNavigation: NavigationItem[] = [
-  {
-    label: routerOptions[router.home].title,
-    value: router.home,
-  },
-  explore,
-  tools,
-];

app/vmui/packages/vmui/src/hooks/useFetchAppConfig.ts

@@ -0,0 +1,34 @@
+import { useAppDispatch } from "../state/common/StateContext";
+import { useEffect, useState } from "preact/compat";
+import { ErrorTypes } from "../types";
+
+const useFetchFlags = () => {
+  const dispatch = useAppDispatch();
+
+  const [isLoading, setIsLoading] = useState(false);
+  const [error, setError] = useState<ErrorTypes | string>("");
+
+  useEffect(() => {
+    const fetchAppConfig = async () => {
+      if (process.env.REACT_APP_TYPE) return;
+      setError("");
+      setIsLoading(true);
+
+      try {
+        const data = await fetch("./config.json");
+        const config = await data.json();
+        dispatch({ type: "SET_APP_CONFIG", payload: config || {} });
+      } catch (e) {
+        setIsLoading(false);
+        if (e instanceof Error) setError(`${e.name}: ${e.message}`);
+      }
+    };
+
+    fetchAppConfig();
+  }, []);
+
+  return { isLoading, error };
+};
+
+export default useFetchFlags;
+

app/vmui/packages/vmui/src/hooks/useFetchQuery.ts

@@ -34,7 +34,8 @@ interface FetchQueryReturn {
   queryStats: QueryStats[],
   warning?: string,
   traces?: Trace[],
-  isHistogram: boolean
+  isHistogram: boolean,
+  abortFetch: () => void
 }
 
 interface FetchDataParams {
@@ -160,6 +161,7 @@ export const useFetchQuery = ({
       const error = e as Error;
       if (error.name === "AbortError") {
         // Aborts are expected, don't show an error for them.
+        setIsLoading(false);
         return;
       }
       const helperText = "Please check your serverURL settings and confirm server availability.";
@@ -197,6 +199,13 @@ export const useFetchQuery = ({
   },
   [serverUrl, period, displayType, customStep, hideQuery]);
 
+  const abortFetch = useCallback(() => {
+    fetchQueue.map(f => f.abort());
+    setFetchQueue([]);
+    setGraphData([]);
+    setLiveData([]);
+  }, [fetchQueue]);
+
   const [prevUrl, setPrevUrl] = useState<string[]>([]);
 
   useEffect(() => {
@@ -238,6 +247,7 @@ export const useFetchQuery = ({
     queryStats,
     warning,
     traces,
-    isHistogram
+    isHistogram,
+    abortFetch,
   };
 };

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/HeaderNav.tsx

@@ -1,16 +1,12 @@
-import React, { FC, useMemo, useState } from "preact/compat";
-import router, { routerOptions } from "../../../router";
-import { getAppModeEnable } from "../../../utils/app-mode";
+import React, { FC, useState } from "preact/compat";
 import { useLocation } from "react-router-dom";
-import { useDashboardsState } from "../../../state/dashboards/DashboardsStateContext";
 import { useEffect } from "react";
 import "./style.scss";
 import NavItem from "./NavItem";
 import NavSubItem from "./NavSubItem";
 import classNames from "classnames";
-import { anomalyNavigation, defaultNavigation, logsNavigation, NavigationItemType } from "../../../constants/navigation";
-import { AppType } from "../../../types/appType";
-import { useAppState } from "../../../state/common/StateContext";
+import useNavigationMenu from "../../../router/useNavigationMenu";
+import { NavigationItemType } from "../../../router/navigation";
 
 interface HeaderNavProps {
   color: string
@@ -19,43 +15,14 @@ interface HeaderNavProps {
 }
 
 const HeaderNav: FC<HeaderNavProps> = ({ color, background, direction }) => {
-  const appModeEnable = getAppModeEnable();
-  const { dashboardsSettings } = useDashboardsState();
   const { pathname } = useLocation();
-  const { serverUrl, flags } = useAppState();
-
   const [activeMenu, setActiveMenu] = useState(pathname);
-
-  const menu = useMemo(() => {
-    switch (process.env.REACT_APP_TYPE) {
-      case AppType.logs:
-        return logsNavigation;
-      case AppType.anomaly:
-        return anomalyNavigation;
-      default:
-        return ([
-          ...defaultNavigation,
-          {
-            label: routerOptions[router.dashboards].title,
-            value: router.dashboards,
-            hide: appModeEnable || !dashboardsSettings.length,
-          },
-          {
-            // see more https://docs.victoriametrics.com/cluster-victoriametrics/?highlight=vmalertproxyurl#vmalert
-            label: "Alerts",
-            value: `${serverUrl}/vmalert`,
-            type: NavigationItemType.externalLink,
-            hide: !Object.keys(flags).includes("vmalert.proxyURL"),
-          },
-        ].filter(r => !r.hide));
-    }
-  }, [appModeEnable, dashboardsSettings]);
+  const menu = useNavigationMenu();
 
   useEffect(() => {
     setActiveMenu(pathname);
   }, [pathname]);
 
-
   return (
     <nav
       className={classNames({

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/NavItem.tsx

@@ -1,7 +1,7 @@
 import React, { FC } from "preact/compat";
 import { NavLink } from "react-router-dom";
 import classNames from "classnames";
-import { NavigationItemType } from "../../../constants/navigation";
+import { NavigationItemType } from "../../../router/navigation";
 
 interface NavItemProps {
   activeMenu: string,

app/vmui/packages/vmui/src/layouts/Header/HeaderNav/NavSubItem.tsx

@@ -6,7 +6,7 @@ import Popper from "../../../components/Main/Popper/Popper";
 import NavItem from "./NavItem";
 import { useEffect } from "react";
 import useBoolean from "../../../hooks/useBoolean";
-import { NavigationItem, NavigationItemType } from "../../../constants/navigation";
+import { NavigationItem, NavigationItemType } from "../../../router/navigation";
 
 interface NavItemProps {
   activeMenu: string,

app/vmui/packages/vmui/src/layouts/MainLayout/MainLayout.tsx

@@ -12,6 +12,7 @@ import useDeviceDetect from "../../hooks/useDeviceDetect";
 import ControlsMainLayout from "./ControlsMainLayout";
 import useFetchDefaultTimezone from "../../hooks/useFetchDefaultTimezone";
 import useFetchFlags from "../../hooks/useFetchFlags";
+import useFetchAppConfig from "../../hooks/useFetchAppConfig";
 
 const MainLayout: FC = () => {
   const appModeEnable = getAppModeEnable();
@@ -21,6 +22,7 @@ const MainLayout: FC = () => {
 
   useFetchDashboards();
   useFetchDefaultTimezone();
+  useFetchAppConfig();
   useFetchFlags();
 
   const setDocumentTitle = () => {

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -41,10 +41,10 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
       value: totalSeries.toLocaleString("en-US"),
       dynamic: (!totalSeries || !totalSeriesPrev || isPrometheus) ? "" : `${dynamic.toFixed(2)}%`,
       display: !focusLabel,
-      info: `The total number of active time series. 
+      info: `The total number of unique time series for a selected day.
              A time series is uniquely identified by its name plus a set of its labels. 
              For example, temperature{city="NY",country="US"} and temperature{city="SF",country="US"} 
-             are two distinct series, since they differ by the city label.`
+             are two distinct series, since they differ by the "city" label.`
     },
     {
       title: "Percentage from total",

app/vmui/packages/vmui/src/pages/CustomPanel/QueryConfigurator/QueryConfigurator.tsx

@@ -10,6 +10,7 @@ import {
   PlayIcon,
   PlusIcon,
   Prettify,
+  SpinnerIcon,
   VisibilityIcon,
   VisibilityOffIcon
 } from "../../../components/Main/Icons";
@@ -30,8 +31,10 @@ export interface QueryConfiguratorProps {
   setQueryErrors: Dispatch<SetStateAction<string[]>>;
   setHideError: Dispatch<SetStateAction<boolean>>;
   stats: QueryStats[];
+  isLoading?: boolean;
   onHideQuery?: (queries: number[]) => void
   onRunQuery: () => void;
+  abortFetch?: () => void;
   hideButtons?: {
     addQuery?: boolean;
     prettify?: boolean;
@@ -46,8 +49,10 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
   setQueryErrors,
   setHideError,
   stats,
+  isLoading,
   onHideQuery,
   onRunQuery,
+  abortFetch,
   hideButtons
 }) => {
 
@@ -84,6 +89,10 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
   };
 
   const handleRunQuery = () => {
+    if (isLoading) {
+      abortFetch && abortFetch();
+      return;
+    }
     updateHistory();
     queryDispatch({ type: "SET_QUERY", payload: stateQuery });
     timeDispatch({ type: "RUN_QUERY" });
@@ -271,9 +280,9 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
         <Button
           variant="contained"
           onClick={handleRunQuery}
-          startIcon={<PlayIcon/>}
+          startIcon={isLoading ? <SpinnerIcon/> : <PlayIcon/>}
         >
-          {isMobile ? "Execute" : "Execute Query"}
+          {`${isLoading ? "Cancel" : "Execute"} ${isMobile ? "" : "Query"}`}
         </Button>
       </div>
     </div>

app/vmui/packages/vmui/src/pages/CustomPanel/index.tsx

@@ -3,7 +3,7 @@ import QueryConfigurator from "./QueryConfigurator/QueryConfigurator";
 import { useFetchQuery } from "../../hooks/useFetchQuery";
 import { DisplayTypeSwitch } from "./DisplayTypeSwitch";
 import { useGraphDispatch, useGraphState } from "../../state/graph/GraphStateContext";
-import Spinner from "../../components/Main/Spinner/Spinner";
+import LineLoader from "../../components/Main/LineLoader/LineLoader";
 import { useCustomPanelState } from "../../state/customPanel/CustomPanelStateContext";
 import { useQueryState } from "../../state/query/QueryStateContext";
 import { useSetQueryParams } from "./hooks/useSetQueryParams";
@@ -45,7 +45,8 @@ const CustomPanel: FC = () => {
     queryStats,
     warning,
     traces,
-    isHistogram
+    isHistogram,
+    abortFetch,
   } = useFetchQuery({
     visible: true,
     customStep,
@@ -80,14 +81,15 @@ const CustomPanel: FC = () => {
         setQueryErrors={setQueryErrors}
         setHideError={setHideError}
         stats={queryStats}
+        isLoading={isLoading}
         onHideQuery={handleHideQuery}
         onRunQuery={handleRunQuery}
+        abortFetch={abortFetch}
       />
       <CustomPanelTraces
         traces={traces}
         displayType={displayType}
       />
-      {isLoading && <Spinner />}
       {showError && <Alert variant="error">{error}</Alert>}
       {showInstantQueryTip && <Alert variant="info"><InstantQueryTip/></Alert>}
       {warning && (
@@ -105,6 +107,7 @@ const CustomPanel: FC = () => {
           "vm-block_mobile": isMobile,
         })}
       >
+        {isLoading && <LineLoader />}
         <div
           className="vm-custom-panel-body-header"
           ref={controlsRef}

app/vmui/packages/vmui/src/pages/DownsamplingFilters/hooks/useDebugDownsamplingFilters.ts

@@ -0,0 +1,53 @@
+import { useAppState } from "../../../state/common/StateContext";
+import { useState } from "react";
+import { ErrorTypes } from "../../../types";
+import { useSearchParams } from "react-router-dom";
+import { getDownsamplingFiltersDebug } from "../../../api/downsampling-filters-debug";
+import { useCallback } from "preact/compat";
+
+export const useDebugDownsamplingFilters = () => {
+  const { serverUrl } = useAppState();
+  const [searchParams, setSearchParams] = useSearchParams();
+
+  const [data, setData] = useState<Map<string, string[]>>(new Map());
+  const [loading, setLoading] = useState(false);
+  const [metricsError, setMetricsError] = useState<ErrorTypes | string>();
+  const [flagsError, setFlagsError] = useState<ErrorTypes | string>();
+  const [error, setError] = useState<ErrorTypes | string>();
+
+  const fetchData = useCallback(async (flags: string, metrics: string) => {
+    metrics ? setMetricsError("") : setMetricsError("metrics are required");
+    flags ? setFlagsError("") : setFlagsError("flags are required");
+    if (!metrics || !flags) return;
+
+    searchParams.set("flags", flags);
+    searchParams.set("metrics", metrics);
+    setSearchParams(searchParams);
+    const fetchUrl = getDownsamplingFiltersDebug(serverUrl, flags, metrics);
+    setLoading(true);
+    try {
+      const response = await fetch(fetchUrl);
+
+      const resp = await response.json();
+      setData(new Map(Object.entries(resp.result || {})));
+      setMetricsError(resp.error?.metrics || "");
+      setFlagsError(resp.error?.flags || "");
+      setError("");
+
+    } catch (e) {
+      if (e instanceof Error && e.name !== "AbortError") {
+        setError(`${e.name}: ${e.message}`);
+      }
+    }
+    setLoading(false);
+  }, [serverUrl]);
+
+  return {
+    data,
+    error: error,
+    metricsError: metricsError,
+    flagsError: flagsError,
+    loading,
+    applyFilters: fetchData
+  };
+};

app/vmui/packages/vmui/src/pages/DownsamplingFilters/index.tsx

@@ -0,0 +1,137 @@
+import React, { FC, useEffect } from "preact/compat";
+import "./style.scss";
+import TextField from "../../components/Main/TextField/TextField";
+import { useCallback, useState } from "react";
+import Button from "../../components/Main/Button/Button";
+import { PlayIcon, WikiIcon } from "../../components/Main/Icons";
+import { useDebugDownsamplingFilters } from "./hooks/useDebugDownsamplingFilters";
+import Spinner from "../../components/Main/Spinner/Spinner";
+import { useSearchParams } from "react-router-dom";
+
+const example = {
+  flags: `-downsampling.period={env="dev"}:7d:5m,{env="dev"}:30d:30m
+-downsampling.period=30d:1m
+-downsampling.period=60d:5m
+`,
+  metrics: `up
+up{env="dev"}
+up{env="prod"}`,
+};
+
+const DownsamplingFilters: FC = () => {
+  const [searchParams] = useSearchParams();
+
+  const { data, loading, error, metricsError, flagsError, applyFilters } = useDebugDownsamplingFilters();
+  const [metrics, setMetrics] = useState(searchParams.get("metrics") || "");
+  const [flags, setFlags] = useState(searchParams.get("flags") || "");
+
+  const handleMetricsChangeInput = useCallback((val: string) => {
+    setMetrics(val);
+  }, [setMetrics]);
+
+  const handleFlagsChangeInput = useCallback((val: string) => {
+    setFlags(val);
+  }, [setFlags]);
+
+  const handleApplyFilters = useCallback(() => {
+    applyFilters(flags, metrics);
+  }, [applyFilters, flags, metrics]);
+
+  const handleRunExample = useCallback(() => {
+    const { flags, metrics } = example;
+    setFlags(flags);
+    setMetrics(metrics);
+    applyFilters(flags, metrics);
+    searchParams.set("flags", flags);
+    searchParams.set("metrics", metrics);
+  }, [example, setFlags, setMetrics, searchParams]);
+
+  useEffect(() => {
+    if (flags && metrics) handleApplyFilters();
+  }, []);
+
+  const rows = [];
+  for (const [key, value] of data) {
+    rows.push(<tr className="vm-table__row">
+      <td className="vm-table-cell">{key}</td>
+      <td className="vm-table-cell">{value.join(" ")}</td>
+    </tr>);
+  }
+  return (
+    <section className="vm-downsampling-filters">
+      {loading && <Spinner/>}
+
+      <div className="vm-downsampling-filters-body vm-block">
+        <div className="vm-downsampling-filters-body__expr">
+          <div className="vm-retention-filters-body__title">
+            <p>Provide a list of flags for downsampling configuration. Note that
+              only <code>-downsampling.period</code> and <code>-dedup.minScrapeInterval</code> flags are supported</p>
+          </div>
+          <TextField
+            type="textarea"
+            label="Flags"
+            value={flags}
+            error={error || flagsError}
+            autofocus
+            onEnter={handleApplyFilters}
+            onChange={handleFlagsChangeInput}
+            placeholder={"-downsampling.period=30d:1m -downsampling.period=7d:5m -dedup.minScrapeInterval=30s"}
+          />
+        </div>
+        <div className="vm-downsampling-filters-body__expr">
+          <div className="vm-retention-filters-body__title">
+            <p>Provide a list of metrics to check downsampling configuration.</p>
+          </div>
+          <TextField
+            type="textarea"
+            label="Metrics"
+            value={metrics}
+            error={error || metricsError}
+            onEnter={handleApplyFilters}
+            onChange={handleMetricsChangeInput}
+            placeholder={"up{env=\"dev\"}\nup{env=\"prod\"}\n"}
+          />
+        </div>
+        <div className="vm-downsampling-filters-body__result">
+          <table className="vm-table">
+            <thead className="vm-table-header">
+              <tr>
+                <th className="vm-table-cell vm-table-cell_header">Metric</th>
+                <th className="vm-table-cell vm-table-cell_header">Applied downsampling rules</th>
+              </tr>
+            </thead>
+            <tbody className="vm-table-body">
+              {rows}
+            </tbody>
+          </table>
+        </div>
+        <div className="vm-downsampling-filters-body-top">
+          <a
+            className="vm-link vm-link_with-icon"
+            target="_blank"
+            href="https://docs.victoriametrics.com/#downsampling"
+            rel="help noreferrer"
+          >
+            <WikiIcon/>
+            Documentation
+          </a>
+          <Button
+            variant="text"
+            onClick={handleRunExample}
+          >
+            Try example
+          </Button>
+          <Button
+            variant="contained"
+            onClick={handleApplyFilters}
+            startIcon={<PlayIcon/>}
+          >
+            Apply
+          </Button>
+        </div>
+      </div>
+    </section>
+  );
+};
+
+export default DownsamplingFilters;

app/vmui/packages/vmui/src/pages/DownsamplingFilters/style.scss

@@ -0,0 +1,46 @@
+@use "src/styles/variables" as *;
+
+.vm-downsampling-filters {
+  display: grid;
+  gap: $padding-medium;
+
+  &-body {
+    display: grid;
+    gap: $padding-global;
+    align-items: flex-start;
+    width: 100%;
+
+    &__title {
+      margin-bottom: $padding-medium;
+    }
+
+    &-top {
+      display: flex;
+      gap: $padding-small;
+      align-items: center;
+      justify-content: flex-end;
+    }
+
+    &__expr textarea {
+      min-height: 200px;
+    }
+
+    &__result textarea {
+      min-height: 60px;
+    }
+
+    code {
+      background-color: var(--color-hover-black);
+      border-radius: 6px;
+      font-size: 85%;
+      padding: .2em .4em;
+    }
+
+    textarea {
+      font-family: $font-family-monospace;
+      overflow: auto;
+      width: 100%;
+      height: 100%;
+    }
+  }
+}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -1,10 +1,9 @@
-import React, { FC, useCallback, useEffect, useState } from "preact/compat";
+import React, { FC, useCallback, useEffect, useMemo, useState } from "preact/compat";
 import ExploreLogsBody from "./ExploreLogsBody/ExploreLogsBody";
 import useStateSearchParams from "../../hooks/useStateSearchParams";
 import useSearchParamsFromObject from "../../hooks/useSearchParamsFromObject";
 import { useFetchLogs } from "./hooks/useFetchLogs";
 import { useAppState } from "../../state/common/StateContext";
-import Spinner from "../../components/Main/Spinner/Spinner";
 import Alert from "../../components/Main/Alert/Alert";
 import ExploreLogsHeader from "./ExploreLogsHeader/ExploreLogsHeader";
 import "./style.scss";
@@ -15,6 +14,7 @@ import ExploreLogsBarChart from "./ExploreLogsBarChart/ExploreLogsBarChart";
 import { useFetchLogHits } from "./hooks/useFetchLogHits";
 import { LOGS_ENTRIES_LIMIT } from "../../constants/logs";
 import { getTimeperiodForDuration, relativeTimeOptions } from "../../utils/time";
+import { useSearchParams } from "react-router-dom";
 
 const storageLimit = Number(getFromStorage("LOGS_LIMIT"));
 const defaultLimit = isNaN(storageLimit) ? LOGS_ENTRIES_LIMIT : storageLimit;
@@ -23,6 +23,8 @@ const ExploreLogs: FC = () => {
   const { serverUrl } = useAppState();
   const { duration, relativeTime, period: periodState } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
+  const [searchParams] = useSearchParams();
+  const hideChart = useMemo(() => searchParams.get("hide_chart"), [searchParams]);
 
   const [limit, setLimit] = useStateSearchParams(defaultLimit, "limit");
   const [query, setQuery] = useStateSearchParams("*", "query");
@@ -30,7 +32,7 @@ const ExploreLogs: FC = () => {
   const [period, setPeriod] = useState<TimeParams>(periodState);
   const [queryError, setQueryError] = useState<ErrorTypes | string>("");
 
-  const { logs, isLoading, error, fetchLogs } = useFetchLogs(serverUrl, query, limit);
+  const { logs, isLoading, error, fetchLogs, abortController } = useFetchLogs(serverUrl, query, limit);
   const { fetchLogHits, ...dataLogHits } = useFetchLogHits(serverUrl, query);
 
   const getPeriod = useCallback(() => {
@@ -50,7 +52,7 @@ const ExploreLogs: FC = () => {
     const newPeriod = getPeriod();
     setPeriod(newPeriod);
     fetchLogs(newPeriod).then((isSuccess) => {
-      isSuccess && fetchLogHits(newPeriod);
+      isSuccess && !hideChart && fetchLogHits(newPeriod);
     }).catch(e => e);
     setSearchParamsFromKeys( {
       query,
@@ -70,10 +72,15 @@ const ExploreLogs: FC = () => {
     setQuery(prev => `_stream: ${val === "other" ? "{}" : val} AND (${prev})`);
   };
 
-  const handleUpdateQuery = () => {
-    setQuery(tmpQuery);
-    handleRunQuery();
-  };
+  const handleUpdateQuery = useCallback(() => {
+    if (isLoading || dataLogHits.isLoading) {
+      abortController.abort && abortController.abort();
+      dataLogHits.abortController.abort && dataLogHits.abortController.abort();
+    } else {
+      setQuery(tmpQuery);
+      handleRunQuery();
+    }
+  }, [isLoading, dataLogHits.isLoading]);
 
   useEffect(() => {
     if (query) handleRunQuery();
@@ -84,6 +91,10 @@ const ExploreLogs: FC = () => {
     setTmpQuery(query);
   }, [query]);
 
+  useEffect(() => {
+    !hideChart && fetchLogHits(period);
+  }, [hideChart]);
+
   return (
     <div className="vm-explore-logs">
       <ExploreLogsHeader
@@ -93,8 +104,8 @@ const ExploreLogs: FC = () => {
         onChange={setTmpQuery}
         onChangeLimit={handleChangeLimit}
         onRun={handleUpdateQuery}
+        isLoading={isLoading || dataLogHits.isLoading}
       />
-      {isLoading && <Spinner message={"Loading logs..."}/>}
       {error && <Alert variant="error">{error}</Alert>}
       {!error && (
         <ExploreLogsBarChart
@@ -102,10 +113,12 @@ const ExploreLogs: FC = () => {
           query={query}
           period={period}
           onApplyFilter={handleApplyFilter}
-          isLoading={isLoading ? false : dataLogHits.isLoading}
         />
       )}
-      <ExploreLogsBody data={logs}/>
+      <ExploreLogsBody
+        data={logs}
+        isLoading={isLoading}
+      />
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBarChart/ExploreLogsBarChart.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useMemo } from "preact/compat";
+import React, { FC, useCallback, useMemo } from "preact/compat";
 import "./style.scss";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import classNames from "classnames";
@@ -9,7 +9,9 @@ import { AlignedData } from "uplot";
 import BarHitsChart from "../../../components/Chart/BarHitsChart/BarHitsChart";
 import Alert from "../../../components/Main/Alert/Alert";
 import { TimeParams } from "../../../types";
-import Spinner from "../../../components/Main/Spinner/Spinner";
+import LineLoader from "../../../components/Main/LineLoader/LineLoader";
+import { useSearchParams } from "react-router-dom";
+import { getHitsTimeParams } from "../../../utils/logs";
 
 interface Props {
   query: string;
@@ -23,27 +25,46 @@ interface Props {
 const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading, onApplyFilter }) => {
   const { isMobile } = useDeviceDetect();
   const timeDispatch = useTimeDispatch();
+  const [searchParams] = useSearchParams();
+  const hideChart = useMemo(() => searchParams.get("hide_chart"), [searchParams]);
 
-  const getXAxis = (timestamps: string[]): number[] => {
-    return (timestamps.map(t => t ? dayjs(t).unix() : null)
-      .filter(Boolean) as number[])
-      .sort((a, b) => a - b);
-  };
-
-  const getYAxes = (logHits: LogHits[], timestamps: string[]) => {
+  const getYAxes = (logHits: LogHits[], timestamps: number[]) => {
     return logHits.map(hits => {
-      return timestamps.map(t => {
-        const index = hits.timestamps.findIndex(ts => ts === t);
-        return index === -1 ? null : hits.values[index] || null;
+      const timestampValueMap = new Map();
+      hits.timestamps.forEach((ts, idx) => {
+        const unixTime = dayjs(ts).unix();
+        timestampValueMap.set(unixTime, hits.values[idx] || null);
       });
+
+      return timestamps.map(t => timestampValueMap.get(t) || null);
     });
   };
 
+  const generateTimestamps = useCallback((date: dayjs.Dayjs) => {
+    const result: number[] = [];
+    const { start, end, step } = getHitsTimeParams(period);
+    const stepsToFirstTimestamp = Math.ceil(start.diff(date, "milliseconds") / step);
+    let firstTimestamp = date.add(stepsToFirstTimestamp * step, "milliseconds");
+
+    // If the first timestamp is before 'start', set it to 'start'
+    if (firstTimestamp.isBefore(start)) {
+      firstTimestamp = start.clone();
+    }
+
+    // Calculate the total number of steps from 'firstTimestamp' to 'end'
+    const totalSteps = Math.floor(end.diff(firstTimestamp, "milliseconds") / step);
+
+    for (let i = 0; i <= totalSteps; i++) {
+      result.push(firstTimestamp.add(i * step, "milliseconds").unix());
+    }
+
+    return result;
+  }, [period]);
+
   const data = useMemo(() => {
     if (!logHits.length) return [[], []] as AlignedData;
-    const timestamps = Array.from(new Set(logHits.map(l => l.timestamps).flat()));
-    const xAxis = getXAxis(timestamps);
-    const yAxes = getYAxes(logHits, timestamps);
+    const xAxis = generateTimestamps(dayjs(logHits[0].timestamps[0]));
+    const yAxes = getYAxes(logHits, xAxis);
     return [xAxis, ...yAxes] as AlignedData;
   }, [logHits]);
 
@@ -51,14 +72,16 @@ const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading, onA
     const noData = data.every(d => d.length === 0);
     const noTimestamps = data[0].length === 0;
     const noValues = data[1].length === 0;
-    if (noData) {
+    if (hideChart) {
+      return "Chart hidden. Hits updates paused.";
+    } else if (noData) {
       return "No logs volume available\nNo volume information available for the current queries and time range.";
     } else if (noTimestamps) {
       return "No timestamp information available for the current queries and time range.";
     } else if (noValues) {
       return "No value information available for the current queries and time range.";
     } return "";
-  }, [data]);
+  }, [data, hideChart]);
 
   const setPeriod = ({ from, to }: {from: Date, to: Date}) => {
     timeDispatch({ type: "SET_PERIOD", payload: { from, to } });
@@ -72,10 +95,7 @@ const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading, onA
         "vm-block_mobile": isMobile,
       })}
     >
-      {isLoading && <Spinner
-        message={"Loading hits stats..."}
-        containerStyles={{ position: "absolute" }}
-      />}
+      {isLoading && <LineLoader/>}
       {!error && noDataMessage && (
         <div className="vm-explore-logs-chart__empty">
           <Alert variant="info">{noDataMessage}</Alert>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBarChart/style.scss

@@ -13,8 +13,12 @@
   }
 
   &__empty {
+    display: flex;
+    align-items: center;
+    justify-content: center;
     position: absolute;
-    transform: translateY(-25px);
+    top: 0;
+    bottom: 0;
     z-index: 2;
   }
 }

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx

@@ -16,9 +16,11 @@ import TableLogs from "./TableLogs";
 import GroupLogs from "../GroupLogs/GroupLogs";
 import { DATE_TIME_FORMAT } from "../../../constants/date";
 import { marked } from "marked";
+import LineLoader from "../../../components/Main/LineLoader/LineLoader";
 
 export interface ExploreLogBodyProps {
   data: Logs[];
+  isLoading: boolean;
 }
 
 enum DisplayType {
@@ -33,7 +35,7 @@ const tabs = [
   { label: "JSON", value: DisplayType.json, icon: <CodeIcon/> },
 ];
 
-const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data }) => {
+const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, isLoading }) => {
   const { isMobile } = useDeviceDetect();
   const { timezone } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
@@ -75,6 +77,7 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data }) => {
         "vm-block_mobile": isMobile,
       })}
     >
+      {isLoading && <LineLoader/>}
       <div
         className={classNames({
           "vm-explore-logs-body-header": true,

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/style.scss

@@ -1,6 +1,8 @@
 @use "src/styles/variables" as *;
 
 .vm-explore-logs-body {
+  position: relative;
+
   &-header {
     margin: -$padding-medium 0-$padding-medium 0;
 

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx

@@ -1,5 +1,5 @@
 import React, { FC, useEffect, useState } from "preact/compat";
-import { InfoIcon, PlayIcon, WikiIcon } from "../../../components/Main/Icons";
+import { InfoIcon, PlayIcon, SpinnerIcon, WikiIcon } from "../../../components/Main/Icons";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
@@ -11,6 +11,7 @@ export interface ExploreLogHeaderProps {
   query: string;
   limit: number;
   error?: string;
+  isLoading: boolean;
   onChange: (val: string) => void;
   onChangeLimit: (val: number) => void;
   onRun: () => void;
@@ -20,6 +21,7 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
   query,
   limit,
   error,
+  isLoading,
   onChange,
   onChangeLimit,
   onRun,
@@ -94,13 +96,16 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
             Documentation
           </a>
         </div>
-        <div className="vm-explore-logs-header-bottom__execute">
+        <div className="vm-explore-logs-header-bottom-execute">
           <Button
-            startIcon={<PlayIcon/>}
+            startIcon={isLoading ? <SpinnerIcon/> : <PlayIcon/>}
             onClick={onRun}
             fullWidth
           >
-            Execute Query
+            <span className="vm-explore-logs-header-bottom-execute__text">
+              {isLoading ? "Cancel Query" : "Execute Query"}
+            </span>
+            <span className="vm-explore-logs-header-bottom-execute__text_hidden">Execute Query</span>
           </Button>
         </div>
       </div>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/style.scss

@@ -29,8 +29,18 @@
       flex-grow: 1;
     }
 
-    &__execute {
+    &-execute {
+      position: relative;
       display: grid;
+
+      &__text {
+        position: absolute;
+
+        &_hidden {
+          position: relative;
+          visibility: hidden;
+        }
+      }
     }
 
     &-helpful {

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss

@@ -7,7 +7,6 @@
     display: flex;
     align-items: center;
     justify-content: flex-end;
-    gap: $padding-global;
 
     &-keys {
       max-height: 300px;

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogHits.ts

@@ -2,9 +2,8 @@ import { useCallback, useMemo, useRef, useState } from "preact/compat";
 import { getLogHitsUrl } from "../../../api/logs";
 import { ErrorTypes, TimeParams } from "../../../types";
 import { LogHits } from "../../../api/types";
-import dayjs from "dayjs";
-import { LOGS_BARS_VIEW } from "../../../constants/logs";
 import { useSearchParams } from "react-router-dom";
+import { getHitsTimeParams } from "../../../utils/logs";
 
 export const useFetchLogHits = (server: string, query: string) => {
   const [searchParams] = useSearchParams();
@@ -17,10 +16,7 @@ export const useFetchLogHits = (server: string, query: string) => {
   const url = useMemo(() => getLogHitsUrl(server), [server]);
 
   const getOptions = (query: string, period: TimeParams, signal: AbortSignal) => {
-    const start = dayjs(period.start * 1000);
-    const end = dayjs(period.end * 1000);
-    const totalSeconds = end.diff(start, "milliseconds");
-    const step = Math.ceil(totalSeconds / LOGS_BARS_VIEW) || 1;
+    const { start, end, step } = getHitsTimeParams(period);
 
     return {
       signal,
@@ -118,5 +114,6 @@ export const useFetchLogHits = (server: string, query: string) => {
     isLoading: Object.values(isLoading).some(s => s),
     error,
     fetchLogHits,
+    abortController: abortControllerRef.current
   };
 };

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogs.ts

@@ -81,5 +81,6 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
     isLoading: Object.values(isLoading).some(s => s),
     error,
     fetchLogs,
+    abortController: abortControllerRef.current
   };
 };

app/vmui/packages/vmui/src/pages/RetentionFilters/hooks/useDebugRetentionFilters.ts

@@ -0,0 +1,53 @@
+import { useAppState } from "../../../state/common/StateContext";
+import { useState } from "react";
+import { ErrorTypes } from "../../../types";
+import { useSearchParams } from "react-router-dom";
+import { getRetentionFiltersDebug } from "../../../api/retention-filters-debug";
+import { useCallback } from "preact/compat";
+
+export const useDebugRetentionFilters = () => {
+  const { serverUrl } = useAppState();
+  const [searchParams, setSearchParams] = useSearchParams();
+
+  const [data, setData] = useState<Map<string, string>>(new Map());
+  const [loading, setLoading] = useState(false);
+  const [metricsError, setMetricsError] = useState<ErrorTypes | string>();
+  const [flagsError, setFlagsError] = useState<ErrorTypes | string>();
+  const [error, setError] = useState<ErrorTypes | string>();
+
+  const fetchData = useCallback(async (flags: string, metrics: string) => {
+    metrics ? setMetricsError("") : setMetricsError("metrics are required");
+    flags ? setFlagsError("") : setFlagsError("flags are required");
+    if (!metrics || !flags) return;
+
+    searchParams.set("flags", flags);
+    searchParams.set("metrics", metrics);
+    setSearchParams(searchParams);
+    const fetchUrl = getRetentionFiltersDebug(serverUrl, flags, metrics);
+    setLoading(true);
+    try {
+      const response = await fetch(fetchUrl);
+
+      const resp = await response.json();
+      setData(new Map(Object.entries(resp.result || {})));
+      setMetricsError(resp.error?.metrics || "");
+      setFlagsError(resp.error?.flags || "");
+      setError("");
+
+    } catch (e) {
+      if (e instanceof Error && e.name !== "AbortError") {
+        setError(`${e.name}: ${e.message}`);
+      }
+    }
+    setLoading(false);
+  }, [serverUrl]);
+
+  return {
+    data,
+    error: error,
+    metricsError: metricsError,
+    flagsError: flagsError,
+    loading,
+    applyFilters: fetchData
+  };
+};

app/vmui/packages/vmui/src/pages/RetentionFilters/index.tsx

@@ -0,0 +1,137 @@
+import React, { FC, useEffect } from "preact/compat";
+import "./style.scss";
+import TextField from "../../components/Main/TextField/TextField";
+import { useCallback, useState } from "react";
+import Button from "../../components/Main/Button/Button";
+import { PlayIcon, WikiIcon } from "../../components/Main/Icons";
+import { useDebugRetentionFilters } from "./hooks/useDebugRetentionFilters";
+import Spinner from "../../components/Main/Spinner/Spinner";
+import { useSearchParams } from "react-router-dom";
+
+const example = {
+  flags: `-retentionPeriod=1y
+-retentionFilters={env!="prod"}:2w
+`,
+  metrics: `up
+up{env="dev"}
+up{env="prod"}`,
+};
+
+const RetentionFilters: FC = () => {
+  const [searchParams] = useSearchParams();
+
+  const { data, loading, error, metricsError, flagsError, applyFilters } = useDebugRetentionFilters();
+  const [metrics, setMetrics] = useState(searchParams.get("metrics") || "");
+  const [flags, setFlags] = useState(searchParams.get("flags") || "");
+
+  const handleMetricsChangeInput = useCallback((val: string) => {
+    setMetrics(val);
+  }, [setMetrics]);
+
+  const handleFlagsChangeInput = useCallback((val: string) => {
+    setFlags(val);
+  }, [setFlags]);
+
+  const handleApplyFilters = useCallback(() => {
+    applyFilters(flags, metrics);
+  }, [applyFilters, flags, metrics]);
+
+  const handleRunExample = useCallback(() => {
+    const { flags, metrics } = example;
+    setFlags(flags);
+    setMetrics(metrics);
+    applyFilters(flags, metrics);
+    searchParams.set("flags", flags);
+    searchParams.set("metrics", metrics);
+  }, [example, setFlags, setMetrics, searchParams]);
+
+  useEffect(() => {
+    if (flags && metrics) handleApplyFilters();
+  }, []);
+
+  const rows = [];
+  for (const [key, value] of data) {
+    rows.push(<tr className="vm-table__row">
+      <td className="vm-table-cell">{key}</td>
+      <td className="vm-table-cell">{value}</td>
+    </tr>);
+  }
+  return (
+    <section className="vm-retention-filters">
+      {loading && <Spinner/>}
+
+      <div className="vm-retention-filters-body vm-block">
+        <div className="vm-retention-filters-body__expr">
+          <div className="vm-retention-filters-body__title">
+            <p>Provide a list of flags for retention configuration. Note that
+              only <code>-retentionPeriod</code> and <code>-retentionFilters</code> flags are
+              supported.</p>
+          </div>
+          <TextField
+            type="textarea"
+            label="Flags"
+            value={flags}
+            error={error || flagsError}
+            autofocus
+            onEnter={handleApplyFilters}
+            onChange={handleFlagsChangeInput}
+            placeholder={"-retentionPeriod=4w -retentionFilters=up{env=\"dev\"}:2w"}
+          />
+        </div>
+        <div className="vm-retention-filters-body__expr">
+          <div className="vm-retention-filters-body__title">
+            <p>Provide a list of metrics to check retention configuration.</p>
+          </div>
+          <TextField
+            type="textarea"
+            label="Metrics"
+            value={metrics}
+            error={error || metricsError}
+            onEnter={handleApplyFilters}
+            onChange={handleMetricsChangeInput}
+            placeholder={"up{env=\"dev\"}\nup{env=\"prod\"}\n"}
+          />
+        </div>
+        <div className="vm-retention-filters-body__result">
+          <table className="vm-table">
+            <thead className="vm-table-header">
+              <tr>
+                <th className="vm-table-cell vm-table-cell_header">Metric</th>
+                <th className="vm-table-cell vm-table-cell_header">Applied retention</th>
+              </tr>
+            </thead>
+            <tbody className="vm-table-body">
+              {rows}
+            </tbody>
+          </table>
+        </div>
+        <div className="vm-retention-filters-body-top">
+          <a
+            className="vm-link vm-link_with-icon"
+            target="_blank"
+            href="https://docs.victoriametrics.com/#retention-filters"
+            rel="help noreferrer"
+          >
+            <WikiIcon/>
+            Documentation
+          </a>
+          <Button
+            variant="text"
+            onClick={handleRunExample}
+          >
+            Try example
+          </Button>
+          <Button
+            variant="contained"
+            onClick={handleApplyFilters}
+            startIcon={<PlayIcon/>}
+          >
+            Apply
+          </Button>
+        </div>
+      </div>
+    </section>
+  );
+};
+
+export default RetentionFilters;

app/vmui/packages/vmui/src/pages/RetentionFilters/style.scss

@@ -0,0 +1,46 @@
+@use "src/styles/variables" as *;
+
+.vm-retention-filters {
+  display: grid;
+  gap: $padding-medium;
+
+  &-body {
+    display: grid;
+    gap: $padding-global;
+    align-items: flex-start;
+    width: 100%;
+
+    &__title {
+      margin-bottom: $padding-medium;
+    }
+
+    &-top {
+      display: flex;
+      gap: $padding-small;
+      align-items: center;
+      justify-content: flex-end;
+    }
+
+    &__expr textarea {
+      min-height: 200px;
+    }
+
+    &__result textarea {
+      min-height: 60px;
+    }
+
+    code {
+      background-color: var(--color-hover-black);
+      border-radius: 6px;
+      font-size: 85%;
+      padding: .2em .4em;
+    }
+
+    textarea {
+      font-family: $font-family-monospace;
+      overflow: auto;
+      width: 100%;
+      height: 100%;
+    }
+  }
+}

app/vmui/packages/vmui/src/router/index.ts

@@ -15,6 +15,8 @@ const router = {
   icons: "/icons",
   anomaly: "/anomaly",
   query: "/query",
+  downsamplingDebug: "/downsampling-filters-debug",
+  retentionDebug: "/retention-filters-debug",
 };
 
 export interface RouterOptionsHeader {
@@ -108,6 +110,14 @@ export const routerOptions: {[key: string]: RouterOptions} = {
   [router.query]: {
     title: "Query",
     ...routerOptionsDefault
+  },
+  [router.downsamplingDebug]: {
+    title: "Downsampling filters debug",
+    header: {}
+  },
+  [router.retentionDebug]: {
+    title: "Retention filters debug",
+    header: {}
   }
 };
 

app/vmui/packages/vmui/src/router/navigation.ts

@@ -0,0 +1,92 @@
+import router, { routerOptions } from "./index";
+
+export enum NavigationItemType {
+  internalLink,
+  externalLink,
+}
+
+export interface NavigationItem {
+  label?: string,
+  value?: string,
+  hide?: boolean
+  submenu?: NavigationItem[],
+  type?: NavigationItemType,
+}
+
+interface NavigationConfig {
+  serverUrl: string,
+  isEnterpriseLicense: boolean,
+  showPredefinedDashboards: boolean,
+  showAlertLink: boolean,
+}
+
+/**
+ * Special case for alert link
+ */
+const getAlertLink = (url: string, showAlertLink: boolean) => {
+  // see more https://docs.victoriametrics.com/cluster-victoriametrics/?highlight=vmalertproxyurl#vmalert
+  return {
+    label: "Alerts",
+    value: `${url}/vmalert`,
+    type: NavigationItemType.externalLink,
+    hide: !showAlertLink,
+  };
+};
+
+/**
+ * Submenu for Tools tab
+ */
+const getToolsNav = (isEnterpriseLicense: boolean) => [
+  { value: router.trace },
+  { value: router.queryAnalyzer },
+  { value: router.withTemplate },
+  { value: router.relabel },
+  { value: router.downsamplingDebug, hide: !isEnterpriseLicense },
+  { value: router.retentionDebug, hide: !isEnterpriseLicense },
+];
+
+/**
+ * Submenu for Explore tab
+ */
+const getExploreNav = () => [
+  { value: router.metrics },
+  { value: router.cardinality },
+  { value: router.topQueries },
+  { value: router.activeQueries },
+];
+
+/**
+ * Default navigation menu
+ */
+export const getDefaultNavigation = ({
+  serverUrl,
+  isEnterpriseLicense,
+  showPredefinedDashboards,
+  showAlertLink,
+}: NavigationConfig): NavigationItem[] => [
+  { value: router.home },
+  { label: "Explore", submenu: getExploreNav() },
+  { label: "Tools", submenu: getToolsNav(isEnterpriseLicense) },
+  { value: router.dashboards, hide: !showPredefinedDashboards },
+  getAlertLink(serverUrl, showAlertLink),
+];
+
+/**
+ * VictoriaLogs navigation menu
+ */
+export const getLogsNavigation = (): NavigationItem[] => [
+  {
+    label: routerOptions[router.logs].title,
+    value: router.home,
+  },
+];
+
+/**
+ * vmanomaly navigation menu
+ */
+export const getAnomalyNavigation = (): NavigationItem[] => [
+  {
+    label: routerOptions[router.anomaly].title,
+    value: router.home,
+  },
+];

app/vmui/packages/vmui/src/router/useNavigationMenu.ts

@@ -0,0 +1,43 @@
+import { getAppModeEnable } from "../utils/app-mode";
+import { useDashboardsState } from "../state/dashboards/DashboardsStateContext";
+import { useAppState } from "../state/common/StateContext";
+import { useMemo } from "preact/compat";
+import { AppType } from "../types/appType";
+import { processNavigationItems } from "./utils";
+import { getAnomalyNavigation, getDefaultNavigation, getLogsNavigation } from "./navigation";
+
+const appType = process.env.REACT_APP_TYPE;
+
+const useNavigationMenu = () => {
+  const appModeEnable = getAppModeEnable();
+  const { dashboardsSettings } = useDashboardsState();
+  const { serverUrl, flags, appConfig } = useAppState();
+  const isEnterpriseLicense = appConfig.license?.type === "enterprise";
+  const showAlertLink = Boolean(flags["vmalert.proxyURL"]);
+  const showPredefinedDashboards = Boolean(!appModeEnable && dashboardsSettings.length);
+
+  const navigationConfig = useMemo(() => ({
+    serverUrl,
+    isEnterpriseLicense,
+    showAlertLink,
+    showPredefinedDashboards
+  }), [serverUrl, isEnterpriseLicense, showAlertLink, showPredefinedDashboards]);
+
+
+  const menu = useMemo(() => {
+    switch (appType) {
+      case AppType.logs:
+        return getLogsNavigation();
+      case AppType.anomaly:
+        return getAnomalyNavigation();
+      default:
+        return getDefaultNavigation(navigationConfig);
+    }
+  }, [navigationConfig]);
+
+  return processNavigationItems(menu);
+};
+
+export default useNavigationMenu;
+
+

app/vmui/packages/vmui/src/router/utils.ts

@@ -0,0 +1,30 @@
+import { routerOptions } from "./index";
+import { NavigationItem } from "./navigation";
+
+const routePathToTitle = (path: string): string => {
+  try {
+    return path
+      .replace(/^\/+/, "") // Remove leading slashes
+      .replace(/-/g, " ") // Replace hyphens with spaces
+      .trim() // Trim whitespace from both ends
+      .replace(/^\w/, (c) => c.toUpperCase()); // Capitalize the first character
+  } catch (e) {
+    return path;
+  }
+};
+
+export const processNavigationItems = (items: NavigationItem[]): NavigationItem[] => {
+  return items.filter((item) => !item.hide).map((item) => {
+    const newItem: NavigationItem = { ...item };
+
+    if (newItem.value && !newItem.label) {
+      newItem.label = routerOptions[newItem.value]?.title || routePathToTitle(newItem.value);
+    }
+
+    if (newItem.submenu && newItem.submenu.length > 0) {
+      newItem.submenu = processNavigationItems(newItem.submenu);
+    }
+
+    return newItem;
+  });
+};

app/vmui/packages/vmui/src/state/common/reducer.ts

@@ -1,7 +1,7 @@
 import { getDefaultServer } from "../../utils/default-server-url";
 import { getQueryStringValue } from "../../utils/query-string";
 import { getFromStorage, saveToStorage } from "../../utils/storage";
-import { Theme } from "../../types";
+import { AppConfig, Theme } from "../../types";
 import { isDarkTheme } from "../../utils/theme";
 import { removeTrailingSlash } from "../../utils/url";
 
@@ -11,6 +11,7 @@ export interface AppState {
   theme: Theme;
   isDarkTheme: boolean | null;
   flags: Record<string, string | null>;
+  appConfig: AppConfig
 }
 
 export type Action =
@@ -18,6 +19,7 @@ export type Action =
   | { type: "SET_THEME", payload: Theme }
   | { type: "SET_TENANT_ID", payload: string }
   | { type: "SET_FLAGS", payload: Record<string, string | null> }
+  | { type: "SET_APP_CONFIG", payload: AppConfig }
   | { type: "SET_DARK_THEME" }
 
 const tenantId = getQueryStringValue("g0.tenantID", "") as string;
@@ -28,6 +30,7 @@ export const initialState: AppState = {
   theme: (getFromStorage("THEME") || Theme.system) as Theme,
   isDarkTheme: null,
   flags: {},
+  appConfig: {}
 };
 
 export function reducer(state: AppState, action: Action): AppState {
@@ -58,6 +61,11 @@ export function reducer(state: AppState, action: Action): AppState {
         ...state,
         flags: action.payload
       };
+    case "SET_APP_CONFIG":
+      return {
+        ...state,
+        appConfig: action.payload
+      };
     default:
       throw new Error();
   }

app/vmui/packages/vmui/src/types/index.ts

@@ -165,3 +165,9 @@ export enum QueryContextType {
   label = "label",
   labelValue = "labelValue",
 }
+
+export interface AppConfig {
+  license?: {
+    type?: "enterprise" | "opensource";
+  }
+}

app/vmui/packages/vmui/src/utils/logs.ts

@@ -1,4 +1,16 @@
+import { TimeParams } from "../types";
+import dayjs from "dayjs";
+import { LOGS_BARS_VIEW } from "../constants/logs";
+
 export const getStreamPairs = (value: string): string[] => {
   const pairs = /^{.+}$/.test(value) ? value.slice(1, -1).split(",") : [value];
   return pairs.filter(Boolean);
 };
+
+export const getHitsTimeParams = (period: TimeParams) => {
+  const start = dayjs(period.start * 1000);
+  const end = dayjs(period.end * 1000);
+  const totalSeconds = end.diff(start, "milliseconds");
+  const step = Math.ceil(totalSeconds / LOGS_BARS_VIEW) || 1;
+  return { start, end, step };
+};

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -137,14 +137,14 @@ export const barDisp = (stroke: Stroke, fill: Fill): Disp => {
 
 export const delSeries = (u: uPlot) => {
   for (let i = u.series.length - 1; i >= 0; i--) {
-    u.delSeries(i);
+    i && u.delSeries(i);
   }
 };
 
 export const addSeries = (u: uPlot, series: uPlotSeries[], spanGaps = false) => {
-  series.forEach((s) => {
+  series.forEach((s,i) => {
     if (s.label) s.spanGaps = spanGaps;
-    u.addSeries(s);
+    i && u.addSeries(s);
   });
 };
 

dashboards/victoriametrics-cluster.json

@@ -6200,7 +6200,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second. The index datapoints value in general is much lower.",
+          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `3*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every two seconds. The index datapoints value in general is much lower.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/victoriametrics.json

@@ -4607,7 +4607,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second.",
+          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `3*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every two seconds.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/victoriametrics-cluster.json

@@ -6201,7 +6201,7 @@
             "type": "victoriametrics-datasource",
             "uid": "$ds"
           },
-          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second. The index datapoints value in general is much lower.",
+          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `3*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every two seconds. The index datapoints value in general is much lower.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/victoriametrics.json

@@ -4608,7 +4608,7 @@
             "type": "victoriametrics-datasource",
             "uid": "$ds"
           },
-          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `2*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every second.",
+          "description": "How many datapoints are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `3*<ingestion_rate>`, since VictoriaMetrics pushes pending data to persistent storage every two seconds.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/vmagent.json

@@ -951,7 +951,7 @@
         "type": "victoriametrics-datasource",
         "uid": "$ds"
       },
-      "description": "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show CPU usage per instance.\n",
+      "description": "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show persistent queue size per instance.\n",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -2834,7 +2834,7 @@
             "type": "victoriametrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows saturation persistent queue for writes. If the threshold of 0.9sec is reached, then persistent is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
+          "description": "Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -2941,7 +2941,7 @@
             "type": "victoriametrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows saturation persistent queue for reads. If the threshold of 0.9sec is reached, then persistent is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
+          "description": "Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -7338,4 +7338,4 @@
   "uid": "G7Z9GzMGz_vm",
   "version": 1,
   "weekStart": ""
-}
+}

dashboards/vmagent.json

@@ -950,7 +950,7 @@
         "type": "prometheus",
         "uid": "$ds"
       },
-      "description": "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show CPU usage per instance.\n",
+      "description": "Shows the persistent queue size of pending samples in bytes >2MB which hasn't been flushed to remote storage yet. \n\nIncreasing of value might be a sign of connectivity issues. In such cases, vmagent starts to flush pending data on disk with attempt to send it later once connection is restored.\n\nRemote write URLs are hidden by default but might be unveiled once `-remoteWrite.showURL` is set to true.\n\nClick on the line and choose Drilldown to show the persistent queue size per instance.\n",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -2833,7 +2833,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows saturation persistent queue for writes. If the threshold of 0.9sec is reached, then persistent is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
+          "description": "Shows write saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -2940,7 +2940,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows saturation persistent queue for reads. If the threshold of 0.9sec is reached, then persistent is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
+          "description": "Shows read saturation of the persistent queue. If the threshold of 0.9sec is reached, then the persistent queue is saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vmagent or improve the disk throughput.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -7337,4 +7337,4 @@
   "uid": "G7Z9GzMGz",
   "version": 1,
   "weekStart": ""
-}
+}

deployment/docker/README.md

@@ -165,6 +165,8 @@ The list of alerting rules is the following:
   alerting rules related to [vmalert](https://docs.victoriametrics.com/vmalert/) component;
 * [alerts-vmauth.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vmauth.yml):
   alerting rules related to [vmauth](https://docs.victoriametrics.com/vmauth/) component;
+* [alerts-vlogs.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alerts-vlogs.yml):
+    alerting rules related to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/);
 
 Please, also see [how to monitor](https://docs.victoriametrics.com/single-server-victoriametrics/#monitoring) 
 VictoriaMetrics installations.

deployment/docker/alerts-cluster.yml

@@ -40,7 +40,7 @@ groups:
         labels:
           severity: critical
         annotations:
-          dashboard: http://localhost:3000/d/oS7Bi_0Wz?viewPanel=200&var-instance={{ $labels.instance }}"
+          dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=200&var-instance={{ $labels.instance }}"
           summary: "Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon"
           description: "Disk utilisation on instance {{ $labels.instance }} is more than 80%.\n
             Having less than 20% of free disk space could cripple merges processes and overall performance.
@@ -164,4 +164,4 @@ groups:
           description: "The connection between vminsert (instance {{ $labels.instance }}) and vmstorage (instance {{ $labels.addr }})
             is saturated by more than 90% and vminsert won't be able to keep up.\n
             This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase
-            the total number of vminsert -> vmstorage links."
+            the total number of vminsert -> vmstorage links."

deployment/docker/alerts-vlogs.yml

@@ -0,0 +1,43 @@
+# File contains default list of alerts for VictoriaLogs single server.
+# The alerts below are just recommendations and may require some updates
+# and threshold calibration according to every specific setup.
+groups:
+  - name: vlogs
+    interval: 30s
+    concurrency: 2
+    rules:
+      - alert: DiskRunsOutOfSpace
+        expr: |
+          sum(vl_data_size_bytes) by(job, instance) /
+          (
+           sum(vl_free_disk_space_bytes) by(job, instance) +
+           sum(vl_data_size_bytes) by(job, instance)
+          ) > 0.8
+        for: 30m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon"
+          description: "Disk utilisation on instance {{ $labels.instance }} is more than 80%.\n
+            Having less than 20% of free disk space could cripple merge processes and overall performance.
+            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
+
+      - alert: RequestErrorsToAPI
+        expr: increase(vl_http_errors_total[5m]) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
+          description: "Requests to path {{ $labels.path }} are receiving errors.
+            Please verify if clients are sending correct requests."
+
+      - alert: RowsRejectedOnIngestion
+        expr: rate(vl_rows_dropped_total[5m]) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt"
+          description: "VictoriaLogs is rejecting to ingest rows on \"{{ $labels.instance }}\" due to the
+            following reason: \"{{ $labels.reason }}\""

deployment/docker/docker-compose-cluster.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.104.0
+    image: victoriametrics/vmagent:v1.105.0
     depends_on:
       - "vminsert"
     ports:
@@ -13,8 +13,8 @@ services:
       - vmagentdata:/vmagentdata
       - ./prometheus-cluster.yml:/etc/prometheus/prometheus.yml
     command:
-      - '--promscrape.config=/etc/prometheus/prometheus.yml'
-      - '--remoteWrite.url=http://vminsert:8480/insert/0/prometheus/'
+      - "--promscrape.config=/etc/prometheus/prometheus.yml"
+      - "--remoteWrite.url=http://vminsert:8480/insert/0/prometheus/"
     restart: always
 
   # Grafana instance configured with VictoriaMetrics as datasource
@@ -39,7 +39,7 @@ services:
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
     container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.104.0-cluster
+    image: victoriametrics/vmstorage:v1.105.0-cluster
     ports:
       - 8482
       - 8400
@@ -47,11 +47,11 @@ services:
     volumes:
       - strgdata-1:/storage
     command:
-      - '--storageDataPath=/storage'
+      - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
     container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.104.0-cluster
+    image: victoriametrics/vmstorage:v1.105.0-cluster
     ports:
       - 8482
       - 8400
@@ -59,20 +59,20 @@ services:
     volumes:
       - strgdata-2:/storage
     command:
-      - '--storageDataPath=/storage'
+      - "--storageDataPath=/storage"
     restart: always
 
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.104.0-cluster
+    image: victoriametrics/vminsert:v1.105.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
     command:
-      - '--storageNode=vmstorage-1:8400'
-      - '--storageNode=vmstorage-2:8400'
+      - "--storageNode=vmstorage-1:8400"
+      - "--storageNode=vmstorage-2:8400"
     ports:
       - 8480:8480
     restart: always
@@ -81,27 +81,27 @@ services:
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
     container_name: vmselect-1
-    image: victoriametrics/vmselect:v1.104.0-cluster
+    image: victoriametrics/vmselect:v1.105.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
     command:
-      - '--storageNode=vmstorage-1:8401'
-      - '--storageNode=vmstorage-2:8401'
-      - '--vmalert.proxyURL=http://vmalert:8880'
+      - "--storageNode=vmstorage-1:8401"
+      - "--storageNode=vmstorage-2:8401"
+      - "--vmalert.proxyURL=http://vmalert:8880"
     ports:
       - 8481
     restart: always
   vmselect-2:
     container_name: vmselect-2
-    image: victoriametrics/vmselect:v1.104.0-cluster
+    image: victoriametrics/vmselect:v1.105.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
     command:
-      - '--storageNode=vmstorage-1:8401'
-      - '--storageNode=vmstorage-2:8401'
-      - '--vmalert.proxyURL=http://vmalert:8880'
+      - "--storageNode=vmstorage-1:8401"
+      - "--storageNode=vmstorage-2:8401"
+      - "--vmalert.proxyURL=http://vmalert:8880"
     ports:
       - 8481
     restart: always
@@ -112,14 +112,14 @@ services:
   # It can be used as an authentication proxy.
   vmauth:
     container_name: vmauth
-    image: victoriametrics/vmauth:v1.104.0
+    image: victoriametrics/vmauth:v1.105.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
     volumes:
       - ./auth-cluster.yml:/etc/auth.yml
     command:
-      - '--auth.config=/etc/auth.yml'
+      - "--auth.config=/etc/auth.yml"
     ports:
       - 8427:8427
     restart: always
@@ -127,7 +127,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.104.0
+    image: victoriametrics/vmalert:v1.105.0
     depends_on:
       - "vmauth"
     ports:
@@ -138,13 +138,13 @@ services:
       - ./alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
       - ./alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
     command:
-      - '--datasource.url=http://vmauth:8427/select/0/prometheus'
-      - '--remoteRead.url=http://vmauth:8427/select/0/prometheus'
-      - '--remoteWrite.url=http://vminsert:8480/insert/0/prometheus'
-      - '--notifier.url=http://alertmanager:9093/'
-      - '--rule=/etc/alerts/*.yml'
+      - "--datasource.url=http://vmauth:8427/select/0/prometheus"
+      - "--remoteRead.url=http://vmauth:8427/select/0/prometheus"
+      - "--remoteWrite.url=http://vminsert:8480/insert/0/prometheus"
+      - "--notifier.url=http://alertmanager:9093/"
+      - "--rule=/etc/alerts/*.yml"
       # display source of alerts in grafana
-      - '-external.url=http://127.0.0.1:3000' #grafana outside container
+      - "-external.url=http://127.0.0.1:3000" #grafana outside container
       - '--external.alert.source=explore?orgId=1&left={"datasource":"VictoriaMetrics","queries":[{"expr":{{.Expr|jsonEscape|queryEscape}},"refId":"A"}],"range":{"from":"{{ .ActiveAt.UnixMilli }}","to":"now"}}'
     restart: always
 
@@ -156,7 +156,7 @@ services:
     volumes:
       - ./alertmanager.yml:/config/alertmanager.yml
     command:
-      - '--config.file=/config/alertmanager.yml'
+      - "--config.file=/config/alertmanager.yml"
     ports:
       - 9093:9093
     restart: always