apptest/tests: add test to verify sparse cache usage

Sparse cache is only used for "final merges" - merge of data of previous months, so tests verify that.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

Makefile

@@ -513,19 +513,19 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test: victoria-metrics vmagent vmalert vmauth
 	go test ./apptest/... -skip="^TestCluster.*"
@@ -567,7 +567,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.63.4
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

app/vlinsert/insertutils/common_params.go

@@ -199,8 +199,8 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 	lmp.bytesIngestedTotal.Add(n)
 
 	if len(fields) > *MaxFieldsPerLine {
-		rf := logstorage.RowFormatter(fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		line := logstorage.MarshalFieldsToJSON(nil, fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
 		rowsDroppedTotalTooManyFields.Inc()
 		return
 	}

app/vlinsert/insertutils/flags.go

@@ -8,8 +8,10 @@ import (
 
 var (
 	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
 
 	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
 )

app/vlogscli/json_prettifier.go

@@ -176,7 +176,7 @@ func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
 		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
 		return err
 	}
-	if len(fields) == 2 && fields[0].Name == "_time" || fields[1].Name == "_time" {
+	if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
 		// Write _time\tfieldValue as is
 		if fields[0].Name == "_time" {
 			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)

app/vlogscli/main.go

@@ -270,7 +270,7 @@ func printCommandsHelp(w io.Writer) {
 \h - show this help
 \s - singleline json output mode
 \m - multiline json output mode
-\c - compact output
+\c - compact output mode
 \logfmt - logfmt output mode
 \wrap_long_lines - toggles wrapping long lines
 \tail <query> - live tail <query> results

app/vlselect/logsql/logsql.go

@@ -688,13 +688,13 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	m := make(map[string]*statsSeries)
 	var mLock sync.Mutex
 
+	timestamp := q.GetTimestamp()
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
 		for i := range timestamps {
-			timestamp := q.GetTimestamp()
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if c.Name == "_time" {

app/vlselect/logsql/logsql_test.go

@@ -28,7 +28,7 @@ func TestParseExtraFilters_Success(t *testing.T) {
 	// LogsQL filter
 	f(`foobar`, `foobar`)
 	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
+	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
 }
 
 func TestParseExtraFilters_Failure(t *testing.T) {
@@ -77,7 +77,7 @@ func TestParseExtraStreamFilters_Success(t *testing.T) {
 	// LogsQL filter
 	f(`foobar`, `foobar`)
 	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
+	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
 }
 
 func TestParseExtraStreamFilters_Failure(t *testing.T) {

app/vlselect/vmui/asset-manifest.json

@@ -1,12 +1,12 @@
 {
   "files": {
-    "main.css": "./static/css/main.4aacd559.css",
-    "main.js": "./static/js/main.5ce54a05.js",
+    "main.css": "./static/css/main.02a1c6cb.css",
+    "main.js": "./static/js/main.55c8060b.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.4aacd559.css",
-    "static/js/main.5ce54a05.js"
+    "static/css/main.02a1c6cb.css",
+    "static/js/main.55c8060b.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.5ce54a05.js"></script><link href="./static/css/main.4aacd559.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.55c8060b.js"></script><link href="./static/css/main.02a1c6cb.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -8,8 +8,6 @@ tests:
     input_series:
       - series: 'up{job="vmagent2", instance="localhost:9090"}'
         values: "0+0x1440"
-      - series: "test_query"
-        values: "1x200"
 
     metricsql_expr_test:
       - expr: suquery_interval_test
@@ -34,8 +32,7 @@ tests:
       - eval_time: 0
         alertname: AlwaysFiring
         exp_alerts:
-          - exp_annotations:
-              queryValue: "1"
+          - {}
 
       - eval_time: 0
         alertname: InstanceDown

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -12,8 +12,6 @@ groups:
           dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
       - alert: AlwaysFiring
         expr: 1
-        annotations:
-          queryValue: "{{ query \"test_query\" | first | value }}"
       - alert: SameAlertNameWithDifferentGroup
         expr: absent(test)
         for: 1m

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -10,8 +10,6 @@ tests:
     input_series:
       - series: "test"
         values: "_x5 1x5 _ stale"
-      - series: "test_query"
-        values: "1x100"
 
     alert_rule_test:
       - eval_time: 1m
@@ -52,8 +50,6 @@ tests:
         values: "0+0x1440"
       - series: "test"
         values: "0+1x1440"
-      - series: "test_query"
-        values: "0+1x100"
 
     metricsql_expr_test:
       - expr: count(ALERTS) by (alertgroup, alertname, alertstate)
@@ -101,17 +97,6 @@ tests:
         exp_alerts:
           - exp_labels:
               cluster: prod
-            exp_annotations:
-              queryValue: "0"
-
-      - eval_time: 5m
-        groupname: group1
-        alertname: AlwaysFiring
-        exp_alerts:
-          - exp_labels:
-              cluster: prod
-            exp_annotations:
-              queryValue: "5"
 
       - eval_time: 0
         groupname: alerts

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -8,8 +8,6 @@ tests:
     input_series:
       - series: 'up{job="vmagent2", instance="localhost:9090"}'
         values: "0+0x1440"
-      - series: "test_query"
-        values: "0+1x200"
 
     metricsql_expr_test:
       - expr: suquery_interval_test
@@ -39,8 +37,6 @@ tests:
         exp_alerts:
           - exp_labels:
               cluster: prod
-            exp_annotations:
-              queryValue: "0"
 
       - eval_time: 0
         groupname: group1

app/vmalert-tool/unittest/unittest.go

@@ -64,18 +64,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	if err != nil {
 		logger.Fatalf("failed to parse external URL: %w", err)
 	}
-	labels := make(map[string]string)
-	for _, s := range externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		labels[s[:n]] = s[n+1:]
-	}
-	if err := templates.Init([]string{}, labels, *eu); err != nil {
+	if err := templates.Load([]string{}, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
 	storagePath = filepath.Join(os.TempDir(), testStoragePath)
@@ -95,7 +84,19 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	if len(testfiles) == 0 {
 		logger.Fatalf("no test file found")
 	}
-	_, err = notifier.Init(nil)
+
+	labels := make(map[string]string)
+	for _, s := range externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+	_, err = notifier.Init(nil, labels, externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init notifier: %v", err)
 	}

app/vmalert/config/config_test.go

@@ -9,13 +9,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())
@@ -78,7 +79,7 @@ groups:
 		for i, u := range urls {
 			urls[i] = srv.URL + u
 		}
-		_, err := Parse(urls, templates.ValidateTemplates, true)
+		_, err := Parse(urls, notifier.ValidateTemplates, true)
 		if err != nil && !expErr {
 			t.Fatalf("error parsing URLs %s", err)
 		}
@@ -94,7 +95,7 @@ groups:
 }
 
 func TestParse_Success(t *testing.T) {
-	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, templates.ValidateTemplates, true)
+	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
 	if err != nil {
 		t.Fatalf("error parsing files %s", err)
 	}
@@ -104,7 +105,7 @@ func TestParse_Failure(t *testing.T) {
 	f := func(paths []string, errStrExpected string) {
 		t.Helper()
 
-		_, err := Parse(paths, templates.ValidateTemplates, true)
+		_, err := Parse(paths, notifier.ValidateTemplates, true)
 		if err == nil {
 			t.Fatalf("expected to get error")
 		}
@@ -115,7 +116,7 @@ func TestParse_Failure(t *testing.T) {
 
 	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
 	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "failed to parse text")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
 	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
@@ -329,7 +330,7 @@ func TestGroupValidate_Success(t *testing.T) {
 
 		var validateTplFn ValidateTplFn
 		if validateAnnotations {
-			validateTplFn = templates.ValidateTemplates
+			validateTplFn = notifier.ValidateTemplates
 		}
 		err := group.Validate(validateTplFn, validateExpressions)
 		if err != nil {

app/vmalert/main.go

@@ -83,6 +83,7 @@ absolute path to all .tpl files in root.
 
 var (
 	alertURLGeneratorFn notifier.AlertURLGenerator
+	extURL              *url.URL
 )
 
 func main() {
@@ -98,29 +99,18 @@ func main() {
 	logger.Init()
 
 	var err error
-	extURL, err := getExternalURL(*externalURL)
+	extURL, err = getExternalURL(*externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init external.url %q: %s", *externalURL, err)
 	}
-	externalls := make(map[string]string)
-	for _, s := range *externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			logger.Fatalf("wrong format in `-external.label`, it must contain label as `Name=value`; got %q", s)
-		}
-		externalls[s[:n]] = s[n+1:]
-	}
 
-	err = templates.Init(*ruleTemplatesPath, externalls, *extURL)
+	err = templates.Load(*ruleTemplatesPath, *extURL)
 	if err != nil {
 		logger.Fatalf("failed to load template %q: %s", *ruleTemplatesPath, err)
 	}
 
 	if *dryRun {
-		groups, err := config.Parse(*rulePath, templates.ValidateTemplates, true)
+		groups, err := config.Parse(*rulePath, notifier.ValidateTemplates, true)
 		if err != nil {
 			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
 		}
@@ -137,7 +127,7 @@ func main() {
 
 	var validateTplFn config.ValidateTplFn
 	if *validateTemplates {
-		validateTplFn = templates.ValidateTemplates
+		validateTplFn = notifier.ValidateTemplates
 	}
 
 	if *replayFrom != "" {
@@ -166,7 +156,7 @@ func main() {
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
-	manager, err := newManager(ctx, externalls)
+	manager, err := newManager(ctx)
 	if err != nil {
 		logger.Fatalf("failed to init: %s", err)
 	}
@@ -213,13 +203,25 @@ var (
 	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
 )
 
-func newManager(ctx context.Context, externalls map[string]string) (*manager, error) {
+func newManager(ctx context.Context) (*manager, error) {
 	q, err := datasource.Init(nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
 
-	nts, err := notifier.Init(alertURLGeneratorFn)
+	labels := make(map[string]string)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
@@ -227,7 +229,7 @@ func newManager(ctx context.Context, externalls map[string]string) (*manager, er
 		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: q,
 		notifiers:      nts,
-		labels:         externalls,
+		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -291,36 +293,24 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		}, nil
 	}
 	if validateTemplate {
-		if err := templates.ValidateTemplates(map[string]string{
+		if err := notifier.ValidateTemplates(map[string]string{
 			"tpl": externalAlertSource,
 		}); err != nil {
-			return nil, fmt.Errorf("cannot parse `external.alert.source` %q: %w", externalAlertSource, err)
+			return nil, fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
 		}
 	}
-	var err error
-	tmpl := templates.GetCurrentTmpl()
-	tmpl, err = templates.ParseWithFixedHeader(externalAlertSource, tmpl)
-	if err != nil {
-		return nil, err
+	m := map[string]string{
+		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		// recreate template if it was changed during config reload
-		cm := templates.GetCurrentTmpl()
-		if tmpl.Name() != cm.Name() {
-			tmpl = cm
-			tmpl, err = templates.ParseWithFixedHeader(externalAlertSource, tmpl)
-			if err != nil {
-				logger.Errorf("cannot parse `external.alert.source` %q: %w", externalAlertSource, err)
-				return fmt.Sprintf("%s/%s", externalURL, externalAlertSource)
-			}
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
 		}
-		tplData := alert.ToTplData()
-		rr, err := templates.ExecuteWithTemplate(tplData, tmpl)
+		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
 		if err != nil {
-			logger.Errorf("can not template alert source: %v", err)
-			return fmt.Sprintf("%s/%s", externalURL, externalAlertSource)
+			logger.Errorf("cannot template alert source: %s", err)
 		}
-		return fmt.Sprintf("%s/%s", externalURL, rr)
+		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
 	}, nil
 }
 
@@ -344,7 +334,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 
 	var validateTplFn config.ValidateTplFn
 	if *validateTemplates {
-		validateTplFn = templates.ValidateTemplates
+		validateTplFn = notifier.ValidateTemplates
 	}
 
 	// init metrics for config state with positive values to improve alerting conditions
@@ -373,7 +363,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to reload notifier config: %s", err)
 			continue
 		}
-		err := templates.LoadTemplateFile(*ruleTemplatesPath)
+		err := templates.Load(*ruleTemplatesPath, *extURL)
 		if err != nil {
 			setConfigError(err)
 			logger.Errorf("failed to load new templates: %s", err)
@@ -386,6 +376,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
+			templates.Reload()
 			// set success to 1 since previous reload could have been unsuccessful
 			// do not update configTimestamp as config version remains old.
 			configSuccess.Set(1)
@@ -399,6 +390,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		templates.Reload()
 		groupsCfg = newGroupsCfg
 		setConfigSuccessAt(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)

app/vmalert/main_test.go

@@ -74,7 +74,10 @@ func TestGetAlertURLGenerator(t *testing.T) {
 
 func TestConfigReload(t *testing.T) {
 	originalRulePath := *rulePath
+	originalExternalURL := extURL
+	extURL = &url.URL{}
 	defer func() {
+		extURL = originalExternalURL
 		*rulePath = originalRulePath
 	}()
 

app/vmalert/manager.go

@@ -160,8 +160,8 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 			// it is important to call InterruptEval before the update, because cancel fn
 			// can be re-assigned during the update.
 			item.old.InterruptEval()
-			go func(old *rule.Group, new *rule.Group) {
-				old.UpdateWith(new)
+			go func(oldGroup *rule.Group, newGroup *rule.Group) {
+				oldGroup.UpdateWith(newGroup)
 				wg.Done()
 			}(item.old, item.new)
 		}

app/vmalert/manager_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())
@@ -72,7 +72,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
-				cfg, err := config.Parse([]string{paths[rnd]}, templates.ValidateTemplates, true)
+				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
 					continue
 				}
@@ -135,7 +135,7 @@ func TestManagerUpdate_Success(t *testing.T) {
 			t.Fatalf("failed to complete initial rules update: %s", err)
 		}
 
-		cfgUpdate, err := config.Parse([]string{updatePath}, templates.ValidateTemplates, true)
+		cfgUpdate, err := config.Parse([]string{updatePath}, notifier.ValidateTemplates, true)
 		if err == nil { // update can fail and that's expected
 			_ = m.update(ctx, cfgUpdate, false)
 		}
@@ -326,7 +326,7 @@ func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressio
 	t.Helper()
 	var validateTplFn config.ValidateTplFn
 	if validateAnnotations {
-		validateTplFn = templates.ValidateTemplates
+		validateTplFn = notifier.ValidateTemplates
 	}
 	cfg, err := config.Parse(path, validateTplFn, validateExpressions)
 	if err != nil {

app/vmalert/notifier/alert.go

@@ -1,9 +1,15 @@
 package notifier
 
 import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	textTpl "text/template"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -70,18 +76,118 @@ func (as AlertState) String() string {
 	return "inactive"
 }
 
-// ToTplData converts Alert to AlertTplData,
-// which only exposes necessary fields for template.
-func (a Alert) ToTplData() templates.AlertTplData {
-	return templates.AlertTplData{
+// AlertTplData is used to execute templating
+type AlertTplData struct {
+	Labels   map[string]string
+	Value    float64
+	Expr     string
+	AlertID  uint64
+	GroupID  uint64
+	ActiveAt time.Time
+	For      time.Duration
+}
+
+var tplHeaders = []string{
+	"{{ $value := .Value }}",
+	"{{ $labels := .Labels }}",
+	"{{ $expr := .Expr }}",
+	"{{ $externalLabels := .ExternalLabels }}",
+	"{{ $externalURL := .ExternalURL }}",
+	"{{ $alertID := .AlertID }}",
+	"{{ $groupID := .GroupID }}",
+	"{{ $activeAt := .ActiveAt }}",
+	"{{ $for := .For }}",
+}
+
+// ExecTemplate executes the Alert template for given
+// map of annotations.
+// Every alert could have a different datasource, so function
+// requires a queryFunction as an argument.
+func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
+	tplData := AlertTplData{
 		Value:    a.Value,
-		Labels:   a.Labels,
+		Labels:   labels,
 		Expr:     a.Expr,
 		AlertID:  a.ID,
 		GroupID:  a.GroupID,
 		ActiveAt: a.ActiveAt,
 		For:      a.For,
 	}
+	return ExecTemplate(q, annotations, tplData)
+}
+
+// ExecTemplate executes the given template for given annotations map.
+func ExecTemplate(q templates.QueryFn, annotations map[string]string, tplData AlertTplData) (map[string]string, error) {
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error cloning template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
+}
+
+// ValidateTemplates validate annotations for possible template error, uses empty data for template population
+func ValidateTemplates(annotations map[string]string) error {
+	tmpl, err := templates.GetWithFuncs(nil)
+	if err != nil {
+		return err
+	}
+	_, err = templateAnnotations(annotations, AlertTplData{
+		Labels: map[string]string{},
+		Value:  0,
+	}, tmpl, false)
+	return err
+}
+
+func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl *textTpl.Template, execute bool) (map[string]string, error) {
+	var builder strings.Builder
+	var buf bytes.Buffer
+	eg := new(utils.ErrGroup)
+	r := make(map[string]string, len(annotations))
+	tData := tplData{data, externalLabels, externalURL}
+	header := strings.Join(tplHeaders, "")
+	for key, text := range annotations {
+		// simple check to skip text without template
+		if !strings.Contains(text, "{{") || !strings.Contains(text, "}}") {
+			r[key] = text
+			continue
+		}
+
+		buf.Reset()
+		builder.Reset()
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
+		builder.WriteString(text)
+		// clone a new template for each parse to avoid collision
+		ctmpl, _ := tmpl.Clone()
+		ctmpl = ctmpl.Option("missingkey=zero")
+		if err := templateAnnotation(&buf, builder.String(), tData, ctmpl, execute); err != nil {
+			r[key] = text
+			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
+			continue
+		}
+		r[key] = buf.String()
+	}
+	return r, eg.Err()
+}
+
+type tplData struct {
+	AlertTplData
+	ExternalLabels map[string]string
+	ExternalURL    string
+}
+
+func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
+	tpl, err := tpl.Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation template: %w", err)
+	}
+	if !execute {
+		return nil
+	}
+	if err = tpl.Execute(dst, data); err != nil {
+		return fmt.Errorf("error evaluating annotation template: %w", err)
+	}
+	return nil
 }
 
 func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {

app/vmalert/notifier/alert_test.go

@@ -1,13 +1,207 @@
 package notifier
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
+func TestAlertExecTemplate(t *testing.T) {
+	extLabels := make(map[string]string)
+	const (
+		extCluster = "prod"
+		extDC      = "east"
+		extURL     = "https://foo.bar"
+	)
+	extLabels["cluster"] = extCluster
+	extLabels["dc"] = extDC
+	_, err := Init(nil, extLabels, extURL)
+	checkErr(t, err)
+
+	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
+		t.Helper()
+
+		if err := ValidateTemplates(annotations); err != nil {
+			t.Fatalf("cannot validate annotations: %s", err)
+		}
+
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return []datasource.Metric{
+				{
+					Labels: []prompbmarshal.Label{
+						{Name: "foo", Value: "bar"},
+						{Name: "baz", Value: "qux"},
+					},
+					Values:     []float64{1},
+					Timestamps: []int64{1},
+				},
+				{
+					Labels: []prompbmarshal.Label{
+						{Name: "foo", Value: "garply"},
+						{Name: "baz", Value: "fred"},
+					},
+					Values:     []float64{2},
+					Timestamps: []int64{1},
+				},
+			}, nil
+		}
+
+		tpl, err := alert.ExecTemplate(qFn, alert.Labels, annotations)
+		if err != nil {
+			t.Fatalf("cannot execute template: %s", err)
+		}
+		if len(tpl) != len(tplExpected) {
+			t.Fatalf("unexpected number of elements; got %d; want %d", len(tpl), len(tplExpected))
+		}
+		for k := range tplExpected {
+			got, exp := tpl[k], tplExpected[k]
+			if got != exp {
+				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
+			}
+		}
+	}
+
+	// empty-alert
+	f(&Alert{}, map[string]string{}, map[string]string{})
+
+	// no-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"summary":     "it's a test summary",
+		"description": "it's a test description",
+	}, map[string]string{
+		"summary":     "it's a test summary",
+		"description": "it's a test description",
+	})
+
+	// label-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+		For: 5 * time.Minute,
+	}, map[string]string{
+		"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
+	}, map[string]string{
+		"summary":     "Too high connection number for localhost for job staging",
+		"description": "It is 10000 connections for localhost for more than 5m0s",
+	})
+
+	// label template override
+	f(&Alert{
+		Value: 1e4,
+	}, map[string]string{
+		"summary":     `{{- define "default.template" -}} {{ printf "summary" }} {{- end -}} {{ template "default.template" . }}`,
+		"description": `{{ template "default.template" . }}`,
+		"value":       `{{$value }}`,
+	}, map[string]string{
+		"summary":     "summary",
+		"description": "",
+		"value":       "10000",
+	})
+
+	// expression-template
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}<0`,
+	}, map[string]string{
+		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
+		"exprEscapedPath":   "{{ $expr|pathEscape }}",
+		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
+		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
+		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
+	}, map[string]string{
+		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
+		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
+		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
+		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
+		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
+	})
+
+	// query
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}>0`,
+	}, map[string]string{
+		"summary": `{{ query "foo" | first | value }}`,
+		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+	}, map[string]string{
+		"summary": "1",
+		"desc":    "bar 1;garply 2;",
+	})
+
+	// external
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"url":         "{{ $externalURL }}",
+		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+	}, map[string]string{
+		"url":         extURL,
+		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+	})
+
+	// alert and group IDs
+	f(&Alert{
+		ID:      42,
+		GroupID: 24,
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID=42&groupID=24",
+	})
+
+	// ActiveAt time
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+	}, map[string]string{
+		"diagram": "![](http://example.com?render=1660941298",
+	})
+
+	// ActiveAt time is nil
+	f(&Alert{}, map[string]string{
+		"default_time": "{{$activeAt}}",
+	}, map[string]string{
+		"default_time": "0001-01-01 00:00:00 +0000 UTC",
+	})
+
+	// ActiveAt custom format
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+	}, map[string]string{
+		"fire_time": "2022/08/19 20:34:58",
+	})
+
+	// ActiveAt query range
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+	}, map[string]string{
+		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+	})
+}
+
 func TestAlert_toPromLabels(t *testing.T) {
 	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
 		t.Helper()

app/vmalert/notifier/init.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
@@ -72,6 +73,15 @@ func Reload() error {
 
 var staticNotifiersFn func() []Notifier
 
+var (
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
+
 // Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
 //   - configuration via flags (for backward compatibility). Is always static
@@ -79,7 +89,14 @@ var staticNotifiersFn func() []Notifier
 //   - configuration via file. Supports live reloads and service discovery.
 //
 // Init returns an error if both mods are used.
-func Init(gen AlertURLGenerator) (func() []Notifier, error) {
+func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
+	externalURL = extURL
+	externalLabels = extLabels
+	_, err := url.Parse(externalURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse external URL: %w", err)
+	}
+
 	if *blackHole {
 		if len(*addrs) > 0 || *configPath != "" {
 			return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
@@ -109,7 +126,6 @@ func Init(gen AlertURLGenerator) (func() []Notifier, error) {
 		return staticNotifiersFn, nil
 	}
 
-	var err error
 	cw, err = newWatcher(*configPath, gen)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init config watcher: %w", err)

app/vmalert/notifier/init_test.go

@@ -12,7 +12,7 @@ func TestInit(t *testing.T) {
 
 	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
 
-	fn, err := Init(nil)
+	fn, err := Init(nil, nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
@@ -52,7 +52,7 @@ func TestInitNegative(t *testing.T) {
 		*configPath = path
 		*addrs = flagutil.ArrayString{addr}
 		*blackHole = bh
-		if _, err := Init(nil); err == nil {
+		if _, err := Init(nil, nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
 		}
 	}
@@ -69,7 +69,7 @@ func TestBlackHole(t *testing.T) {
 
 	*blackHole = true
 
-	fn, err := Init(nil)
+	fn, err := Init(nil, nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}

app/vmalert/notifier/package_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())

app/vmalert/rule/alerting.go

@@ -7,7 +7,6 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	textTpl "text/template"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -48,12 +47,6 @@ type AlertingRule struct {
 	state *ruleState
 
 	metrics *alertingRuleMetrics
-
-	// set rootTemplateName with loaded global template name,
-	// so we can check if the template has changed when evaluating.
-	rootTemplateName     string
-	LabelTemplates       map[string]*textTpl.Template
-	AnnotationsTemplates map[string]*textTpl.Template
 }
 
 type alertingRuleMetrics struct {
@@ -149,47 +142,9 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			}
 			return seriesFetched
 		})
-
-	ar.initTemplate()
-
 	return ar
 }
 
-// initTemplate pre-creates templates that can be reused in execution.
-func (ar *AlertingRule) initTemplate() {
-	currentTmpl := templates.GetCurrentTmpl()
-	ar.rootTemplateName = currentTmpl.Name()
-	ar.LabelTemplates = make(map[string]*textTpl.Template, len(ar.Labels))
-	ar.AnnotationsTemplates = make(map[string]*textTpl.Template, len(ar.Annotations))
-	for k, v := range ar.Labels {
-		var err error
-		tmpl, _ := currentTmpl.Clone()
-		tmpl, err = templates.ParseWithFixedHeader(v, tmpl)
-		if err != nil {
-			// parse can fail in two cases:
-			// 1. the text contains `query` function, which is not supported during rule initialization.
-			// 2. the text itself is invalid.
-			// In both case, we skip the error here, and try it again during rule execution.
-			continue
-		}
-		ar.LabelTemplates[k] = tmpl
-
-	}
-	for k, v := range ar.Annotations {
-		var err error
-		tmpl, _ := currentTmpl.Clone()
-		tmpl, err = templates.ParseWithFixedHeader(v, tmpl)
-		if err != nil {
-			// parse can fail in two cases:
-			// 1. the text contains `query` function, which is not supported during rule initialization.
-			// 2. the text itself is invalid.
-			// In both case, we skip the error here, and try it again during rule execution.
-			continue
-		}
-		ar.AnnotationsTemplates[k] = tmpl
-	}
-}
-
 // close unregisters rule metrics
 func (ar *AlertingRule) close() {
 	ar.metrics.active.Unregister()
@@ -270,8 +225,6 @@ func (ar *AlertingRule) updateWith(r Rule) error {
 	ar.KeepFiringFor = nr.KeepFiringFor
 	ar.Labels = nr.Labels
 	ar.Annotations = nr.Annotations
-	ar.initTemplate()
-
 	ar.EvalInterval = nr.EvalInterval
 	ar.Debug = nr.Debug
 	ar.q = nr.q
@@ -326,32 +279,15 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		}
 		ls.processed[l.Name] = l.Value
 	}
-	extraLabels := make(map[string]string, len(ar.Labels))
-	// compare to annotation, label value can only use limited variables for now
-	labelTplData := templates.AlertTplData{
+
+	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
 		Expr:   ar.Expr,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
-	for k := range ar.Labels {
-		if ar.LabelTemplates[k] == nil {
-			// this label may contain `query` function, which requires creating new template with query function in each evaluation.
-			v, err := templates.ExecuteWithoutTemplate(qFn, ar.Labels[k], labelTplData)
-			if err != nil {
-				logger.Errorf("error templating label %q for rule %q: %w", ar.Labels[k], ar.Name, err)
-				v = ar.Labels[k]
-			}
-			extraLabels[k] = v
-			continue
-		}
-		v, err := templates.ExecuteWithTemplate(labelTplData, ar.LabelTemplates[k])
-		if err != nil {
-			logger.Errorf("error templating label %q for rule %q: %w", ar.Labels[k], ar.Name, err)
-			v = ar.Labels[k]
-		}
-		extraLabels[k] = v
-	}
-
 	for k, v := range extraLabels {
 		ls.add(k, v)
 	}
@@ -577,18 +513,12 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 }
 
 func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
-	// check if the rule template has changed during reload,
-	// if so, label&annotation templates must be re-created.
-	if ar.rootTemplateName != templates.GetCurrentTmpl().Name() {
-		ar.initTemplate()
-	}
-
 	ls, err := ar.toLabels(m, qFn)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
-	extraAnnotation := make(map[string]string, len(ar.Annotations))
-	annotationTplData := templates.AlertTplData{
+
+	tplData := notifier.AlertTplData{
 		Value:    m.Values[0],
 		Labels:   ls.origin,
 		Expr:     ar.Expr,
@@ -597,26 +527,11 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 		ActiveAt: ts,
 		For:      ar.For,
 	}
-	for k := range ar.Annotations {
-		if ar.AnnotationsTemplates[k] == nil {
-			// this label may contain `query` function, which requires creating new template with query function in each evaluation.
-			v, err := templates.ExecuteWithoutTemplate(qFn, ar.Annotations[k], annotationTplData)
-			if err != nil {
-				logger.Errorf("error templating annotation %q for rule %q: %w", ar.Annotations[k], ar.Name, err)
-				v = ar.Annotations[k]
-			}
-			extraAnnotation[k] = v
-			continue
-		}
-		v, err := templates.ExecuteWithTemplate(annotationTplData, ar.AnnotationsTemplates[k])
-		if err != nil {
-			logger.Errorf("error templating annotation %q for rule %q: %w", ar.Annotations[k], ar.Name, err)
-			v = ar.Annotations[k]
-		}
-		extraAnnotation[k] = v
+	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to template annotations: %w", err)
 	}
-
-	return ls, extraAnnotation, nil
+	return ls, as, nil
 }
 
 // toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts

app/vmalert/rule/alerting_test.go

@@ -1046,7 +1046,6 @@ func TestAlertingRule_Template(t *testing.T) {
 		}
 		fq.Add(metrics...)
 
-		rule.initTemplate()
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -1263,7 +1262,6 @@ func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, kee
 	}
 	rule.KeepFiringFor = keepFiringFor
 	rule.Annotations = annotation
-	rule.initTemplate()
 	return rule
 }
 

app/vmalert/rule/group.go

@@ -443,8 +443,8 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 }
 
 // UpdateWith inserts new group to updateCh
-func (g *Group) UpdateWith(new *Group) {
-	g.updateCh <- new
+func (g *Group) UpdateWith(newGroup *Group) {
+	g.updateCh <- newGroup
 }
 
 // DeepCopy returns a deep copy of group

app/vmalert/rule/group_test.go

@@ -27,7 +27,7 @@ func init() {
 }
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{}, url.URL{}); err != nil {
 		fmt.Println("failed to load template for test")
 		os.Exit(1)
 	}

app/vmalert/templates/init.go

@@ -1,553 +0,0 @@
-package templates
-
-import (
-	"errors"
-	"fmt"
-	htmlTpl "html/template"
-	"io"
-	"math"
-	"net"
-	"net/url"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	textTpl "text/template"
-	"time"
-
-	"github.com/bmatcuk/doublestar/v4"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-var (
-	tplMu      sync.RWMutex
-	masterTmpl *textTpl.Template
-
-	// externalLabels is a global variable for holding external labels configured via flags
-	// It is supposed to be initiated via Init function only.
-	externalLabels map[string]string
-	// externalURL is a global variable for holding external URL value configured via flag
-	// It is supposed to be initiated via Init function only.
-	externalURL url.URL
-)
-
-// Init initializes global externalLabels and externalURL variables, and load templates from pathPatterns.
-func Init(pathPatterns []string, extLabels map[string]string, extURL url.URL) error {
-	externalURL = extURL
-	externalLabels = extLabels
-	return LoadTemplateFile(pathPatterns)
-}
-
-// LoadTemplateFile loads templates from multiple globs specified in pathPatterns:
-// 1. if it's the first load, sets them directly to current template;
-// 2. if it's not the first load, only update masterTmpl when the contents change.
-func LoadTemplateFile(pathPatterns []string) error {
-	templateName := fmt.Sprintf("rule-template-%d", time.Now().UnixMilli())
-	// using Load timestamp as template root name,
-	// so we can check if this global template has been reloaded when use it elsewhere, like during alerting rules execution.
-	tmpl := newTemplate(templateName)
-	tmpl = tmpl.Funcs(funcsWithExternalURL(externalURL))
-
-	for _, tp := range pathPatterns {
-		p, err := doublestar.FilepathGlob(tp)
-		if err != nil {
-			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
-		}
-		if len(p) > 0 {
-			tmpl, err = tmpl.ParseFiles(p...)
-			if err != nil {
-				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
-			}
-		}
-	}
-	err := tmpl.Execute(io.Discard, nil)
-	if err != nil {
-		return fmt.Errorf("failed to test rule template: %w", err)
-	}
-
-	tplMu.Lock()
-	defer tplMu.Unlock()
-
-	if masterTmpl == nil {
-		masterTmpl = tmpl
-	} else {
-		// only update the masterTmpl when content has changed
-		if !isTemplatesTheSame(masterTmpl, tmpl) {
-			masterTmpl = tmpl
-		}
-	}
-	return nil
-}
-
-func newTemplate(name string) *textTpl.Template {
-	tmpl := textTpl.New(name).Funcs(templateFuncs())
-	return textTpl.Must(tmpl.Parse(""))
-}
-
-// isTemplatesTheSame returns true if the content of two templates are the same,
-// the root template name difference is ignored.
-func isTemplatesTheSame(t1, t2 *textTpl.Template) bool {
-	getRootString := func(t *textTpl.Template) string {
-		if t.Tree == nil || t.Tree.Root == nil {
-			return ""
-		}
-		return t.Tree.Root.String()
-	}
-	if getRootString(t1) != getRootString(t2) {
-		return false
-	}
-	t1Templates := make(map[string]string)
-	t2Templates := make(map[string]string)
-	for _, tmpl := range t1.Templates() {
-		// skip root template, since it's null and changes every time
-		if tmpl.Name() == t1.Name() {
-			continue
-		}
-		t1Templates[tmpl.Name()] = getRootString(tmpl)
-	}
-	for _, tmpl := range t2.Templates() {
-		if tmpl.Name() == t2.Name() {
-			continue
-		}
-		t2Templates[tmpl.Name()] = getRootString(tmpl)
-	}
-	if len(t1Templates) != len(t2Templates) {
-		return false
-	}
-	for k, v := range t1Templates {
-		if t2Templates[k] != v {
-			return false
-		}
-		delete(t2Templates, k)
-	}
-	return len(t2Templates) == 0
-}
-
-// funcsWithExternalURL returns a function map that depends on externalURL value
-func funcsWithExternalURL(externalURL url.URL) textTpl.FuncMap {
-	return textTpl.FuncMap{
-		"externalURL": func() string {
-			return externalURL.String()
-		},
-
-		"pathPrefix": func() string {
-			return externalURL.Path
-		},
-	}
-}
-
-// templateFuncs initiates template helper functions
-func templateFuncs() textTpl.FuncMap {
-	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
-	// and https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/template/template.go#L150
-	return textTpl.FuncMap{
-		/* Strings */
-
-		// title returns a copy of the string s with all Unicode letters
-		// that begin words mapped to their Unicode title case.
-		// alias for https://golang.org/pkg/strings/#Title
-		"title": strings.Title,
-
-		// toUpper returns s with all Unicode letters mapped to their upper case.
-		// alias for https://golang.org/pkg/strings/#ToUpper
-		"toUpper": strings.ToUpper,
-
-		// toLower returns s with all Unicode letters mapped to their lower case.
-		// alias for https://golang.org/pkg/strings/#ToLower
-		"toLower": strings.ToLower,
-
-		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
-		// This function is deprecated.
-		//
-		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
-		// these functions properly escape `\n` and `\r` chars according to their purpose.
-		"crlfEscape": func(q string) string {
-			q = strings.Replace(q, "\n", `\n`, -1)
-			return strings.Replace(q, "\r", `\r`, -1)
-		},
-
-		// quotesEscape escapes the string, so it can be safely put inside JSON string.
-		//
-		// See also jsonEscape.
-		"quotesEscape": quotesEscape,
-
-		// jsonEscape converts the string to properly encoded JSON string.
-		//
-		// See also quotesEscape.
-		"jsonEscape": jsonEscape,
-
-		// htmlEscape applies html-escaping to q, so it can be safely embedded as plaintext into html.
-		//
-		// See also safeHtml.
-		"htmlEscape": htmlEscape,
-
-		// stripPort splits string into host and port, then returns only host.
-		"stripPort": func(hostPort string) string {
-			host, _, err := net.SplitHostPort(hostPort)
-			if err != nil {
-				return hostPort
-			}
-			return host
-		},
-
-		// stripDomain removes the domain part of a FQDN. Leaves port untouched.
-		"stripDomain": func(hostPort string) string {
-			host, port, err := net.SplitHostPort(hostPort)
-			if err != nil {
-				host = hostPort
-			}
-			ip := net.ParseIP(host)
-			if ip != nil {
-				return hostPort
-			}
-			host = strings.Split(host, ".")[0]
-			if port != "" {
-				return net.JoinHostPort(host, port)
-			}
-			return host
-		},
-
-		// match reports whether the string s
-		// contains any match of the regular expression pattern.
-		// alias for https://golang.org/pkg/regexp/#MatchString
-		"match": regexp.MatchString,
-
-		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
-		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
-		// so for instance $1 represents the text of the first submatch.
-		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
-		"reReplaceAll": func(pattern, repl, text string) string {
-			re := regexp.MustCompile(pattern)
-			return re.ReplaceAllString(text, repl)
-		},
-
-		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
-		"parseDuration": func(s string) (float64, error) {
-			d, err := promutils.ParseDuration(s)
-			if err != nil {
-				return 0, err
-			}
-			return d.Seconds(), nil
-		},
-
-		// same with parseDuration but returns a time.Duration
-		"parseDurationTime": func(s string) (time.Duration, error) {
-			d, err := promutils.ParseDuration(s)
-			if err != nil {
-				return 0, err
-			}
-			return d, nil
-		},
-
-		/* Numbers */
-
-		// humanize converts given number to a human readable format
-		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			if math.Abs(v) >= 1 {
-				prefix := ""
-				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
-					if math.Abs(v) < 1000 {
-						break
-					}
-					prefix = p
-					v /= 1000
-				}
-				return fmt.Sprintf("%.4g%s", v, prefix), nil
-			}
-			prefix := ""
-			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
-				if math.Abs(v) >= 1 {
-					break
-				}
-				prefix = p
-				v *= 1000
-			}
-			return fmt.Sprintf("%.4g%s", v, prefix), nil
-		},
-
-		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			return formatutil.HumanizeBytes(v), nil
-		},
-
-		// humanizeDuration converts given seconds to a human-readable duration
-		"humanizeDuration": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			if v == 0 {
-				return fmt.Sprintf("%.4gs", v), nil
-			}
-			if math.Abs(v) >= 1 {
-				sign := ""
-				if v < 0 {
-					sign = "-"
-					v = -v
-				}
-				seconds := int64(v) % 60
-				minutes := (int64(v) / 60) % 60
-				hours := (int64(v) / 60 / 60) % 24
-				days := int64(v) / 60 / 60 / 24
-				// For days to minutes, we display seconds as an integer.
-				if days != 0 {
-					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
-				}
-				if hours != 0 {
-					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
-				}
-				if minutes != 0 {
-					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
-				}
-				// For seconds, we display 4 significant digits.
-				return fmt.Sprintf("%s%.4gs", sign, v), nil
-			}
-			prefix := ""
-			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
-				if math.Abs(v) >= 1 {
-					break
-				}
-				prefix = p
-				v *= 1000
-			}
-			return fmt.Sprintf("%.4g%ss", v, prefix), nil
-		},
-
-		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			return fmt.Sprintf("%.4g%%", v*100), nil
-		},
-
-		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			t := timeFromUnixTimestamp(v).Time().UTC()
-			return fmt.Sprint(t), nil
-		},
-
-		// toTime converts given timestamp to a time.Time.
-		"toTime": func(i any) (time.Time, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return time.Time{}, err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
-			}
-			t := timeFromUnixTimestamp(v).Time().UTC()
-			return t, nil
-		},
-
-		/* URLs */
-
-		// externalURL returns value of `external.url` flag
-		"externalURL": func() string {
-			// externalURL function supposed to be substituted at FuncsWithExteralURL().
-			// it is present here only for validation purposes, when there is no
-			// provided datasource.
-			//
-			// return non-empty slice to pass validation with chained functions in template
-			return ""
-		},
-
-		// pathPrefix returns a Path segment from the URL value in `external.url` flag
-		"pathPrefix": func() string {
-			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
-			// it is present here only for validation purposes, when there is no
-			// provided datasource.
-			//
-			// return non-empty slice to pass validation with chained functions in template
-			return ""
-		},
-
-		// pathEscape escapes the string so it can be safely placed inside a URL path segment.
-		//
-		// See also queryEscape.
-		"pathEscape": url.PathEscape,
-
-		// queryEscape escapes the string so it can be safely placed inside a query arg in URL.
-		//
-		// See also queryEscape.
-		"queryEscape": url.QueryEscape,
-
-		// first returns the first by order element from the given metrics list.
-		// usually used alongside with `query` template function.
-		"first": func(metrics []metric) (metric, error) {
-			if len(metrics) > 0 {
-				return metrics[0], nil
-			}
-			return metric{}, errors.New("first() called on vector with no elements")
-		},
-
-		// label returns the value of the given label name for the given metric.
-		// usually used alongside with `query` template function.
-		"label": func(label string, m metric) string {
-			return m.Labels[label]
-		},
-
-		// value returns the value of the given metric.
-		// usually used alongside with `query` template function.
-		"value": func(m metric) float64 {
-			return m.Value
-		},
-
-		// strvalue returns metric name.
-		"strvalue": func(m metric) string {
-			return m.Labels["__name__"]
-		},
-
-		// sortByLabel sorts the given metrics by provided label key
-		"sortByLabel": func(label string, metrics []metric) []metric {
-			sort.SliceStable(metrics, func(i, j int) bool {
-				return metrics[i].Labels[label] < metrics[j].Labels[label]
-			})
-			return metrics
-		},
-
-		/* Helpers */
-
-		// Converts a list of objects to a map with keys arg0, arg1 etc.
-		// This is intended to allow multiple arguments to be passed to templates.
-		"args": func(args ...any) map[string]any {
-			result := make(map[string]any)
-			for i, a := range args {
-				result[fmt.Sprintf("arg%d", i)] = a
-			}
-			return result
-		},
-
-		// safeHtml marks string as HTML not requiring auto-escaping.
-		//
-		// See also htmlEscape.
-		"safeHtml": func(text string) htmlTpl.HTML {
-			return htmlTpl.HTML(text)
-		},
-	}
-}
-
-// metric is private copy of datasource.Metric,
-// it is used for templating annotations,
-// Labels as map simplifies templates evaluation.
-type metric struct {
-	Labels    map[string]string
-	Timestamp int64
-	Value     float64
-}
-
-// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
-func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
-	mss := make([]metric, 0, len(ms))
-	for _, m := range ms {
-		labelsMap := make(map[string]string, len(m.Labels))
-		for _, labelValue := range m.Labels {
-			labelsMap[labelValue.Name] = labelValue.Value
-		}
-		mss = append(mss, metric{
-			Labels:    labelsMap,
-			Timestamp: m.Timestamps[0],
-			Value:     m.Values[0]})
-	}
-	return mss
-}
-
-// QueryFn is used to wrap a call to datasource into simple-to-use function
-// for templating functions.
-type QueryFn func(query string) ([]datasource.Metric, error)
-
-// FuncsWithQuery returns a function map that depends on metric data
-func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
-	return textTpl.FuncMap{
-		"query": func(q string) ([]metric, error) {
-			if query == nil {
-				return nil, fmt.Errorf("cannot execute query %q: query is not available in this context", q)
-			}
-
-			result, err := query(q)
-			if err != nil {
-				return nil, err
-			}
-			return datasourceMetricsToTemplateMetrics(result), nil
-		},
-	}
-}
-
-// Time is the number of milliseconds since the epoch
-// (1970-01-01 00:00 UTC) excluding leap seconds.
-type Time int64
-
-// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
-func timeFromUnixTimestamp(t float64) Time {
-	return Time(t * 1e3)
-}
-
-// The number of nanoseconds per minimum tick.
-const nanosPerTick = int64(minimumTick / time.Nanosecond)
-
-// MinimumTick is the minimum supported time resolution. This has to be
-// at least time.Second in order for the code below to work.
-const minimumTick = time.Millisecond
-
-// second is the Time duration equivalent to one second.
-const second = int64(time.Second / minimumTick)
-
-// Time returns the time.Time representation of t.
-func (t Time) Time() time.Time {
-	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
-}
-
-func toFloat64(v any) (float64, error) {
-	switch i := v.(type) {
-	case float64:
-		return i, nil
-	case float32:
-		return float64(i), nil
-	case int64:
-		return float64(i), nil
-	case int32:
-		return float64(i), nil
-	case int:
-		return float64(i), nil
-	case uint64:
-		return float64(i), nil
-	case uint32:
-		return float64(i), nil
-	case uint:
-		return float64(i), nil
-	case string:
-		return strconv.ParseFloat(i, 64)
-	default:
-		return 0, fmt.Errorf("unexpected value type %v", i)
-	}
-}

app/vmalert/templates/init_test.go

@@ -1,261 +0,0 @@
-package templates
-
-import (
-	"fmt"
-	"math"
-	"net/url"
-	"os"
-	"strings"
-	"testing"
-	textTpl "text/template"
-)
-
-func TestMain(m *testing.M) {
-	if err := Init([]string{}, nil, url.URL{}); err != nil {
-		fmt.Println("failed to load template for test")
-		os.Exit(1)
-	}
-	os.Exit(m.Run())
-}
-
-func TestTemplateFuncs_Match(t *testing.T) {
-	funcs := templateFuncs()
-	// check "match" func
-	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
-	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
-		t.Fatalf("expecting non-nil error on invalid regexp")
-	}
-	ok, err := matchFunc("abc", "def")
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	if ok {
-		t.Fatalf("unexpected match")
-	}
-	ok, err = matchFunc("a.+b", "acsdb")
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	if !ok {
-		t.Fatalf("unexpected mismatch")
-	}
-}
-
-func TestTemplateFuncs_Formatting(t *testing.T) {
-	f := func(funcName string, p any, resultExpected string) {
-		t.Helper()
-
-		funcs := templateFuncs()
-		v := funcs[funcName]
-		fLocal := v.(func(s any) (string, error))
-		result, err := fLocal(p)
-		if err != nil {
-			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
-		}
-	}
-
-	f("humanize1024", float64(0), "0")
-	f("humanize1024", math.Inf(0), "+Inf")
-	f("humanize1024", math.NaN(), "NaN")
-	f("humanize1024", float64(127087), "124.1ki")
-	f("humanize1024", float64(130137088), "124.1Mi")
-	f("humanize1024", float64(133260378112), "124.1Gi")
-	f("humanize1024", float64(136458627186688), "124.1Ti")
-	f("humanize1024", float64(139733634239168512), "124.1Pi")
-	f("humanize1024", float64(143087241460908556288), "124.1Ei")
-	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
-	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
-	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
-
-	f("humanize", float64(127087), "127.1k")
-	f("humanize", float64(136458627186688), "136.5T")
-
-	f("humanizeDuration", 1, "1s")
-	f("humanizeDuration", 0.2, "200ms")
-	f("humanizeDuration", 42000, "11h 40m 0s")
-	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
-
-	f("humanizePercentage", 1, "100%")
-	f("humanizePercentage", 0.8, "80%")
-	f("humanizePercentage", 0.015, "1.5%")
-
-	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
-}
-
-func TestTemplateFuncs_StringConversion(t *testing.T) {
-	f := func(funcName, s, resultExpected string) {
-		t.Helper()
-
-		funcs := templateFuncs()
-		v := funcs[funcName]
-		fLocal := v.(func(s string) string)
-		result := fLocal(s)
-		if result != resultExpected {
-			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
-		}
-	}
-
-	f("title", "foo bar", "Foo Bar")
-	f("toUpper", "foo", "FOO")
-	f("toLower", "FOO", "foo")
-	f("pathEscape", "foo/bar\n+baz", "foo%2Fbar%0A+baz")
-	f("queryEscape", "foo+bar\n+baz", "foo%2Bbar%0A%2Bbaz")
-	f("jsonEscape", `foo{bar="baz"}`+"\n + 1", `"foo{bar=\"baz\"}\n + 1"`)
-	f("quotesEscape", `foo{bar="baz"}`+"\n + 1", `foo{bar=\"baz\"}\n + 1`)
-	f("htmlEscape", "foo < 10\nabc", "foo &lt; 10\nabc")
-	f("crlfEscape", "foo\nbar\rx", `foo\nbar\rx`)
-	f("stripPort", "foo", "foo")
-	f("stripPort", "foo:1234", "foo")
-	f("stripDomain", "foo.bar.baz", "foo")
-	f("stripDomain", "foo.bar:123", "foo:123")
-}
-
-func TestTemplatesLoad_Success(t *testing.T) {
-	f := func(pathPatterns []string, expectedTmpl *textTpl.Template) {
-		t.Helper()
-
-		masterTmplOrig := masterTmpl
-		defer func() {
-			masterTmpl = masterTmplOrig
-		}()
-		masterTmpl = nil
-
-		if err := LoadTemplateFile(pathPatterns); err != nil {
-			t.Fatalf("cannot load templates: %s", err)
-		}
-		if !isTemplatesTheSame(masterTmpl, expectedTmpl) {
-			t.Fatalf("unexpected template\ngot\n%+v\nwant\n%+v", masterTmpl, expectedTmpl)
-		}
-	}
-
-	// non existing path
-	pathPatterns := []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
-	}
-	expectedTmpl := textTpl.Must(newTemplate("").Parse(""))
-	f(pathPatterns, expectedTmpl)
-
-	// existing path
-	pathPatterns = []string{
-		"templates/test/good0-*.tpl",
-	}
-	expectedTmpl = textTpl.Must(newTemplate("").Parse(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`))
-	f(pathPatterns, expectedTmpl)
-
-	// template update
-	pathPatterns = []string{
-		"templates/other/nested/good0-*.tpl",
-	}
-	expectedTmpl = textTpl.Must(newTemplate("").Parse(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`))
-	f(pathPatterns, expectedTmpl)
-}
-
-func TestTemplatesLoad_Failure(t *testing.T) {
-	f := func(pathPatterns []string, expectedErrStr string) {
-		t.Helper()
-
-		err := LoadTemplateFile(pathPatterns)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-
-		errStr := err.Error()
-		if !strings.Contains(errStr, expectedErrStr) {
-			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
-		}
-	}
-
-	// load template with syntax error
-	f([]string{
-		"templates/other/nested/bad0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}, "failed to parse template glob")
-}
-
-func TestTemplatesReload(t *testing.T) {
-	masterTmplOrig := masterTmpl
-	defer func() {
-		masterTmpl = masterTmplOrig
-	}()
-	masterTmpl = nil
-
-	// load with non existing path
-	pathPatterns := []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
-	}
-	if err := LoadTemplateFile(pathPatterns); err != nil {
-		t.Fatalf("cannot load templates: %s", err)
-	}
-	tpl1 := GetCurrentTmpl()
-
-	// reload with existing path
-	pathPatterns = []string{
-		"templates/test/good0-*.tpl",
-	}
-	if err := LoadTemplateFile(pathPatterns); err != nil {
-		t.Fatalf("cannot load templates: %s", err)
-	}
-	tpl2 := GetCurrentTmpl()
-	if isTemplatesTheSame(tpl1, tpl2) {
-		t.Fatalf("tpl1 should be different from tpl2")
-	}
-
-	// reload the same path
-	pathPatterns = []string{
-		"templates/test/good0-*.tpl",
-	}
-	if err := LoadTemplateFile(pathPatterns); err != nil {
-		t.Fatalf("cannot load templates: %s", err)
-	}
-	tpl3 := GetCurrentTmpl()
-	if !isTemplatesTheSame(tpl2, tpl3) || tpl2.Name() != tpl3.Name() {
-		t.Fatalf("tpl3 should be the same as tpl2")
-	}
-}
-
-func TestIsTemplateTheSame(t *testing.T) {
-	f := func(tmpl1, tmpl2 *textTpl.Template, isTheSame bool) {
-		t.Helper()
-
-		if isTemplatesTheSame(tmpl1, tmpl2) != isTheSame {
-			t.Fatalf("unexpected result for isTemplatesTheSame")
-		}
-	}
-
-	tmpl1 := textTpl.Must(newTemplate("t1").Parse("{{- define \"test\" -}}{{- end -}}"))
-	tmpl2 := textTpl.Must(newTemplate("t2").Parse("{{- define \"test\" -}}{{- end -}}"))
-	f(tmpl1, tmpl2, true)
-
-	tmpl1, _ = tmpl1.Parse("{{- define \"test2\" -}}{{- end -}}")
-	f(tmpl1, tmpl2, false)
-
-	tmpl2, _ = tmpl2.Parse("{{- define \"test3\" -}}{{- end -}}")
-	f(tmpl1, tmpl2, false)
-}

app/vmalert/templates/template.go

@@ -1,95 +1,554 @@
+// Copyright 2013 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package templates
 
 import (
-	"bytes"
+	"errors"
 	"fmt"
+	htmlTpl "html/template"
+	"io"
+	"math"
+	"net"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	textTpl "text/template"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/bmatcuk/doublestar/v4"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
-// supported variables are list in https://docs.victoriametrics.com/vmalert/#templating.
-const tplHeaders = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}{{ $externalLabels := .ExternalLabels }}{{ $externalURL := .ExternalURL }}{{ $alertID := .AlertID }}{{ $groupID := .GroupID }}{{ $activeAt := .ActiveAt }}{{ $for := .For }}`
+// go template execution fails when it's tree is empty
+const defaultTemplate = `{{- define "default.template" -}}{{- end -}}`
 
-// AlertTplData is used to execute templating
-type AlertTplData struct {
-	Labels   map[string]string
-	Value    float64
-	Expr     string
-	AlertID  uint64
-	GroupID  uint64
-	ActiveAt time.Time
-	For      time.Duration
+var tplMu sync.RWMutex
+
+type textTemplate struct {
+	current     *textTpl.Template
+	replacement *textTpl.Template
 }
 
-// ValidateTemplates validates the given annotations,
-// mock the `query` function during validation.
-func ValidateTemplates(annotations map[string]string) error {
-	// it's ok to reuse one template for multiple text validations.
-	tmpl := GetCurrentTmpl()
-	tmpl = tmpl.Funcs(FuncsWithQuery(nil))
-	for _, v := range annotations {
-		_, err := tmpl.Parse(tplHeaders + v)
+var masterTmpl textTemplate
+
+func newTemplate() *textTpl.Template {
+	tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs())
+	return textTpl.Must(tmpl.Parse(defaultTemplate))
+}
+
+// Load func loads templates from multiple globs specified in pathPatterns and either
+// sets them directly to current template if it's the first init;
+// or sets replacement templates and wait for Reload() to replace current template with replacement.
+func Load(pathPatterns []string, externalURL url.URL) error {
+	tmpl := newTemplate()
+	for _, tp := range pathPatterns {
+		p, err := doublestar.FilepathGlob(tp)
 		if err != nil {
-			return fmt.Errorf("failed to parse text %q into template: %w", v, err)
+			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
 		}
+		if len(p) > 0 {
+			tmpl, err = tmpl.ParseFiles(p...)
+			if err != nil {
+				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
+			}
+		}
+	}
+	if len(tmpl.Templates()) > 0 {
+		err := tmpl.Execute(io.Discard, nil)
+		if err != nil {
+			return fmt.Errorf("failed to execute template: %w", err)
+		}
+	}
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	tmpl = tmpl.Funcs(funcsWithExternalURL(externalURL))
+
+	if masterTmpl.current == nil {
+		masterTmpl.current = tmpl
+	} else {
+		masterTmpl.replacement = tmpl
 	}
 	return nil
 }
 
-// GetCurrentTmpl returns a copy of the current global template
-func GetCurrentTmpl() *textTpl.Template {
+// Reload func replaces current template with a replacement template
+// which was set by Load with override=false
+func Reload() {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.replacement != nil {
+		masterTmpl.current = masterTmpl.replacement
+		masterTmpl.replacement = nil
+	}
+}
+
+// metric is private copy of datasource.Metric,
+// it is used for templating annotations,
+// Labels as map simplifies templates evaluation.
+type metric struct {
+	Labels    map[string]string
+	Timestamp int64
+	Value     float64
+}
+
+// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
+func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
+	mss := make([]metric, 0, len(ms))
+	for _, m := range ms {
+		labelsMap := make(map[string]string, len(m.Labels))
+		for _, labelValue := range m.Labels {
+			labelsMap[labelValue.Name] = labelValue.Value
+		}
+		mss = append(mss, metric{
+			Labels:    labelsMap,
+			Timestamp: m.Timestamps[0],
+			Value:     m.Values[0]})
+	}
+	return mss
+}
+
+// QueryFn is used to wrap a call to datasource into simple-to-use function
+// for templating functions.
+type QueryFn func(query string) ([]datasource.Metric, error)
+
+// GetWithFuncs returns a copy of current template with additional FuncMap
+// provided with funcs argument
+func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
 	tplMu.RLock()
 	defer tplMu.RUnlock()
-	tmpl, err := masterTmpl.Clone()
+	tmpl, err := masterTmpl.current.Clone()
 	if err != nil {
-		logger.Panicf("failed to clone current rule template: %w", err)
+		return nil, err
 	}
-	return tmpl
+	// Clone() doesn't copy tpl Options, so we set them manually
+	tmpl = tmpl.Option("missingkey=zero")
+	return tmpl.Funcs(funcs), nil
 }
 
-type tplData struct {
-	AlertTplData
-	ExternalLabels map[string]string
-	ExternalURL    string
+// FuncsWithQuery returns a function map that depends on metric data
+func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"query": func(q string) ([]metric, error) {
+			if query == nil {
+				return nil, fmt.Errorf("cannot execute query %q: query is not available in this context", q)
+			}
+
+			result, err := query(q)
+			if err != nil {
+				return nil, err
+			}
+			return datasourceMetricsToTemplateMetrics(result), nil
+		},
+	}
 }
 
-// ParseWithFixedHeader parses the text with the fixed tplHeaders into the given template
-func ParseWithFixedHeader(text string, tpl *textTpl.Template) (*textTpl.Template, error) {
-	return tpl.Parse(tplHeaders + text)
+// funcsWithExternalURL returns a function map that depends on externalURL value
+func funcsWithExternalURL(externalURL url.URL) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+	}
 }
 
-// ExecuteWithoutTemplate retrieves the current global templates, parses the text and executes with the given data
-func ExecuteWithoutTemplate(q QueryFn, text string, data AlertTplData) (string, error) {
-	if !strings.Contains(text, "{{") || !strings.Contains(text, "}}") {
-		return text, nil
-	}
+// templateFuncs initiates template helper functions
+func templateFuncs() textTpl.FuncMap {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
+	// and https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/template/template.go#L150
+	return textTpl.FuncMap{
+		/* Strings */
 
-	var err error
-	tmpl := GetCurrentTmpl()
-	tmpl = tmpl.Funcs(FuncsWithQuery(q))
-	tmpl, err = tmpl.Parse(tplHeaders + text)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse text %q into template: %w", text, err)
+		// title returns a copy of the string s with all Unicode letters
+		// that begin words mapped to their Unicode title case.
+		// alias for https://golang.org/pkg/strings/#Title
+		"title": strings.Title,
 
+		// toUpper returns s with all Unicode letters mapped to their upper case.
+		// alias for https://golang.org/pkg/strings/#ToUpper
+		"toUpper": strings.ToUpper,
+
+		// toLower returns s with all Unicode letters mapped to their lower case.
+		// alias for https://golang.org/pkg/strings/#ToLower
+		"toLower": strings.ToLower,
+
+		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
+		// This function is deprecated.
+		//
+		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
+		// these functions properly escape `\n` and `\r` chars according to their purpose.
+		"crlfEscape": func(q string) string {
+			q = strings.Replace(q, "\n", `\n`, -1)
+			return strings.Replace(q, "\r", `\r`, -1)
+		},
+
+		// quotesEscape escapes the string, so it can be safely put inside JSON string.
+		//
+		// See also jsonEscape.
+		"quotesEscape": quotesEscape,
+
+		// jsonEscape converts the string to properly encoded JSON string.
+		//
+		// See also quotesEscape.
+		"jsonEscape": jsonEscape,
+
+		// htmlEscape applies html-escaping to q, so it can be safely embedded as plaintext into html.
+		//
+		// See also safeHtml.
+		"htmlEscape": htmlEscape,
+
+		// stripPort splits string into host and port, then returns only host.
+		"stripPort": func(hostPort string) string {
+			host, _, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				return hostPort
+			}
+			return host
+		},
+
+		// stripDomain removes the domain part of a FQDN. Leaves port untouched.
+		"stripDomain": func(hostPort string) string {
+			host, port, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				host = hostPort
+			}
+			ip := net.ParseIP(host)
+			if ip != nil {
+				return hostPort
+			}
+			host = strings.Split(host, ".")[0]
+			if port != "" {
+				return net.JoinHostPort(host, port)
+			}
+			return host
+		},
+
+		// match reports whether the string s
+		// contains any match of the regular expression pattern.
+		// alias for https://golang.org/pkg/regexp/#MatchString
+		"match": regexp.MatchString,
+
+		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
+		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
+		// so for instance $1 represents the text of the first submatch.
+		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
+		"reReplaceAll": func(pattern, repl, text string) string {
+			re := regexp.MustCompile(pattern)
+			return re.ReplaceAllString(text, repl)
+		},
+
+		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
+		"parseDuration": func(s string) (float64, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d.Seconds(), nil
+		},
+
+		// same with parseDuration but returns a time.Duration
+		"parseDurationTime": func(s string) (time.Duration, error) {
+			d, err := promutils.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d, nil
+		},
+
+		/* Numbers */
+
+		// humanize converts given number to a human readable format
+		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
+		"humanize": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			if math.Abs(v) >= 1 {
+				prefix := ""
+				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
+					if math.Abs(v) < 1000 {
+						break
+					}
+					prefix = p
+					v /= 1000
+				}
+				return fmt.Sprintf("%.4g%s", v, prefix), nil
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
+		},
+
+		// humanize1024 converts given number to a human readable format with 1024 as base
+		"humanize1024": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			return formatutil.HumanizeBytes(v), nil
+		},
+
+		// humanizeDuration converts given seconds to a human-readable duration
+		"humanizeDuration": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			if v == 0 {
+				return fmt.Sprintf("%.4gs", v), nil
+			}
+			if math.Abs(v) >= 1 {
+				sign := ""
+				if v < 0 {
+					sign = "-"
+					v = -v
+				}
+				seconds := int64(v) % 60
+				minutes := (int64(v) / 60) % 60
+				hours := (int64(v) / 60 / 60) % 24
+				days := int64(v) / 60 / 60 / 24
+				// For days to minutes, we display seconds as an integer.
+				if days != 0 {
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
+				}
+				if hours != 0 {
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
+				}
+				if minutes != 0 {
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
+				}
+				// For seconds, we display 4 significant digits.
+				return fmt.Sprintf("%s%.4gs", sign, v), nil
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%ss", v, prefix), nil
+		},
+
+		// humanizePercentage converts given ratio value to a fraction of 100
+		"humanizePercentage": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("%.4g%%", v*100), nil
+		},
+
+		// humanizeTimestamp converts given timestamp to a human readable time equivalent
+		"humanizeTimestamp": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return fmt.Sprint(t), nil
+		},
+
+		// toTime converts given timestamp to a time.Time.
+		"toTime": func(i any) (time.Time, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return time.Time{}, err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return t, nil
+		},
+
+		/* URLs */
+
+		// externalURL returns value of `external.url` flag
+		"externalURL": func() string {
+			// externalURL function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
+		},
+
+		// pathPrefix returns a Path segment from the URL value in `external.url` flag
+		"pathPrefix": func() string {
+			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
+		},
+
+		// pathEscape escapes the string so it can be safely placed inside a URL path segment.
+		//
+		// See also queryEscape.
+		"pathEscape": url.PathEscape,
+
+		// queryEscape escapes the string so it can be safely placed inside a query arg in URL.
+		//
+		// See also queryEscape.
+		"queryEscape": url.QueryEscape,
+
+		// query executes the MetricsQL/PromQL query against
+		// configured `datasource.url` address.
+		// For example, {{ query "foo" | first | value }} will
+		// execute "/api/v1/query?query=foo" request and will return
+		// the first value in response.
+		"query": func(_ string) ([]metric, error) {
+			// query function supposed to be substituted at FuncsWithQuery().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			// see issue #989 for details
+			return []metric{{}}, nil
+		},
+
+		// first returns the first by order element from the given metrics list.
+		// usually used alongside with `query` template function.
+		"first": func(metrics []metric) (metric, error) {
+			if len(metrics) > 0 {
+				return metrics[0], nil
+			}
+			return metric{}, errors.New("first() called on vector with no elements")
+		},
+
+		// label returns the value of the given label name for the given metric.
+		// usually used alongside with `query` template function.
+		"label": func(label string, m metric) string {
+			return m.Labels[label]
+		},
+
+		// value returns the value of the given metric.
+		// usually used alongside with `query` template function.
+		"value": func(m metric) float64 {
+			return m.Value
+		},
+
+		// strvalue returns metric name.
+		"strvalue": func(m metric) string {
+			return m.Labels["__name__"]
+		},
+
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
+		/* Helpers */
+
+		// Converts a list of objects to a map with keys arg0, arg1 etc.
+		// This is intended to allow multiple arguments to be passed to templates.
+		"args": func(args ...any) map[string]any {
+			result := make(map[string]any)
+			for i, a := range args {
+				result[fmt.Sprintf("arg%d", i)] = a
+			}
+			return result
+		},
+
+		// safeHtml marks string as HTML not requiring auto-escaping.
+		//
+		// See also htmlEscape.
+		"safeHtml": func(text string) htmlTpl.HTML {
+			return htmlTpl.HTML(text)
+		},
 	}
-	return ExecuteWithTemplate(data, tmpl)
 }
 
-// ExecuteWithTemplate executes with the given template and data
-func ExecuteWithTemplate(data AlertTplData, tpl *textTpl.Template) (string, error) {
-	fullData := tplData{
-		data,
-		externalLabels,
-		externalURL.String(),
-	}
-	var buf bytes.Buffer
-	// returns the zero value for the map type's element
-	tpl.Option("missingkey=zero")
-	if err := tpl.Execute(&buf, fullData); err != nil {
-		return "", fmt.Errorf("failed to execute template: %w", err)
-	}
-	return buf.String(), nil
+// Time is the number of milliseconds since the epoch
+// (1970-01-01 00:00 UTC) excluding leap seconds.
+type Time int64
+
+// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
+func timeFromUnixTimestamp(t float64) Time {
+	return Time(t * 1e3)
+}
+
+// The number of nanoseconds per minimum tick.
+const nanosPerTick = int64(minimumTick / time.Nanosecond)
+
+// MinimumTick is the minimum supported time resolution. This has to be
+// at least time.Second in order for the code below to work.
+const minimumTick = time.Millisecond
+
+// second is the Time duration equivalent to one second.
+const second = int64(time.Second / minimumTick)
+
+// Time returns the time.Time representation of t.
+func (t Time) Time() time.Time {
+	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
+}
+
+func toFloat64(v any) (float64, error) {
+	switch i := v.(type) {
+	case float64:
+		return i, nil
+	case float32:
+		return float64(i), nil
+	case int64:
+		return float64(i), nil
+	case int32:
+		return float64(i), nil
+	case int:
+		return float64(i), nil
+	case uint64:
+		return float64(i), nil
+	case uint32:
+		return float64(i), nil
+	case uint:
+		return float64(i), nil
+	case string:
+		return strconv.ParseFloat(i, 64)
+	default:
+		return 0, fmt.Errorf("unexpected value type %v", i)
+	}
 }

app/vmalert/templates/template_test.go

@@ -1,226 +1,239 @@
 package templates
 
 import (
-	"fmt"
+	"math"
 	"net/url"
+	"strings"
 	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	textTpl "text/template"
 )
 
-func TestValidateTemplates(t *testing.T) {
-	f := func(annotations map[string]string, isValid bool) {
+func TestTemplateFuncs_StringConversion(t *testing.T) {
+	f := func(funcName, s, resultExpected string) {
 		t.Helper()
 
-		err := ValidateTemplates(annotations)
-		if (err == nil) != isValid {
-			t.Fatalf("failed to validate template, got %t; want %t", (err == nil), isValid)
+		funcs := templateFuncs()
+		v := funcs[funcName]
+		fLocal := v.(func(s string) string)
+		result := fLocal(s)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
 		}
 	}
 
-	// empty
-	f(map[string]string{}, true)
-
-	// wrong text
-	f(map[string]string{
-		"summary": "{{",
-	}, false)
-
-	// valid
-	f(map[string]string{
-		"value":   "{{$value}}",
-		"summary": "it's a test summary",
-	}, true)
-
-	// invalid variable
-	f(map[string]string{
-		"value":   "{{$invalidValue}}",
-		"summary": "it's a test summary",
-	}, false)
+	f("title", "foo bar", "Foo Bar")
+	f("toUpper", "foo", "FOO")
+	f("toLower", "FOO", "foo")
+	f("pathEscape", "foo/bar\n+baz", "foo%2Fbar%0A+baz")
+	f("queryEscape", "foo+bar\n+baz", "foo%2Bbar%0A%2Bbaz")
+	f("jsonEscape", `foo{bar="baz"}`+"\n + 1", `"foo{bar=\"baz\"}\n + 1"`)
+	f("quotesEscape", `foo{bar="baz"}`+"\n + 1", `foo{bar=\"baz\"}\n + 1`)
+	f("htmlEscape", "foo < 10\nabc", "foo &lt; 10\nabc")
+	f("crlfEscape", "foo\nbar\rx", `foo\nbar\rx`)
+	f("stripPort", "foo", "foo")
+	f("stripPort", "foo:1234", "foo")
+	f("stripDomain", "foo.bar.baz", "foo")
+	f("stripDomain", "foo.bar:123", "foo:123")
 }
 
-func TestExecuteWithoutTemplate(t *testing.T) {
-	extLabels := make(map[string]string)
-	const (
-		extCluster = "prod"
-		extDC      = "east"
-		extURL     = "https://foo.bar"
-	)
-	url, _ := url.Parse(extURL)
-	extLabels["cluster"] = extCluster
-	extLabels["dc"] = extDC
-
-	err := Init(nil, extLabels, *url)
+func TestTemplateFuncs_Match(t *testing.T) {
+	funcs := templateFuncs()
+	// check "match" func
+	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
+	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
+		t.Fatalf("expecting non-nil error on invalid regexp")
+	}
+	ok, err := matchFunc("abc", "def")
 	if err != nil {
-		t.Fatalf("cannot init templates: %s", err)
+		t.Fatalf("unexpected error")
 	}
+	if ok {
+		t.Fatalf("unexpected match")
+	}
+	ok, err = matchFunc("a.+b", "acsdb")
+	if err != nil {
+		t.Fatalf("unexpected error")
+	}
+	if !ok {
+		t.Fatalf("unexpected mismatch")
+	}
+}
 
-	f := func(data AlertTplData, annotations, expResults map[string]string) {
+func TestTemplateFuncs_Formatting(t *testing.T) {
+	f := func(funcName string, p any, resultExpected string) {
 		t.Helper()
 
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return []datasource.Metric{
-				{
-					Labels: []prompbmarshal.Label{
-						{Name: "foo", Value: "bar"},
-						{Name: "baz", Value: "qux"},
-					},
-					Values:     []float64{1},
-					Timestamps: []int64{1},
-				},
-				{
-					Labels: []prompbmarshal.Label{
-						{Name: "foo", Value: "garply"},
-						{Name: "baz", Value: "fred"},
-					},
-					Values:     []float64{2},
-					Timestamps: []int64{1},
-				},
-			}, nil
+		funcs := templateFuncs()
+		v := funcs[funcName]
+		fLocal := v.(func(s any) (string, error))
+		result, err := fLocal(p)
+		if err != nil {
+			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
 		}
-
-		for k := range annotations {
-			v, err := ExecuteWithoutTemplate(qFn, annotations[k], data)
-			if err != nil {
-				t.Fatalf("cannot execute template: %s", err)
-			}
-			if v != expResults[k] {
-				t.Fatalf("unexpected result; got %s; want %s", v, expResults[k])
-			}
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
 		}
-
 	}
 
-	// empty-alert
-	f(AlertTplData{}, map[string]string{}, map[string]string{})
+	f("humanize1024", float64(0), "0")
+	f("humanize1024", math.Inf(0), "+Inf")
+	f("humanize1024", math.NaN(), "NaN")
+	f("humanize1024", float64(127087), "124.1ki")
+	f("humanize1024", float64(130137088), "124.1Mi")
+	f("humanize1024", float64(133260378112), "124.1Gi")
+	f("humanize1024", float64(136458627186688), "124.1Ti")
+	f("humanize1024", float64(139733634239168512), "124.1Pi")
+	f("humanize1024", float64(143087241460908556288), "124.1Ei")
+	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
+	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
+	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
 
-	// no-template
-	f(AlertTplData{
-		Value: 1e4,
-		Labels: map[string]string{
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"summary":     "it's a test summary",
-		"description": "it's a test description",
-	}, map[string]string{
-		"summary":     "it's a test summary",
-		"description": "it's a test description",
-	})
+	f("humanize", float64(127087), "127.1k")
+	f("humanize", float64(136458627186688), "136.5T")
 
-	// label-template
-	f(AlertTplData{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-		For: 5 * time.Minute,
-	}, map[string]string{
-		"summary":            "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
-		"description":        "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
-		"non-existing-label": "{{$labels.nonexisting}}",
-	}, map[string]string{
-		"summary":            "Too high connection number for localhost for job staging",
-		"description":        "It is 10000 connections for localhost for more than 5m0s",
-		"non-existing-label": "",
-	})
+	f("humanizeDuration", 1, "1s")
+	f("humanizeDuration", 0.2, "200ms")
+	f("humanizeDuration", 42000, "11h 40m 0s")
+	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
 
-	// label template override
-	f(AlertTplData{
-		Value: 1e4,
-	}, map[string]string{
-		"summary":     `{{- define "default.template" -}} {{ printf "summary" }} {{- end -}} {{ template "default.template" . }}`,
-		"description": `{{- define "default.template" -}} {{ printf "description" }} {{- end -}} {{ template "default.template" . }}`,
-		"value":       `{{$value }}`,
-	}, map[string]string{
-		"summary":     "summary",
-		"description": "description",
-		"value":       "10000",
-	})
+	f("humanizePercentage", 1, "100%")
+	f("humanizePercentage", 0.8, "80%")
+	f("humanizePercentage", 0.015, "1.5%")
 
-	// expression-template
-	f(AlertTplData{
-		Expr: `vm_rows{"label"="bar"}<0`,
-	}, map[string]string{
-		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
-		"exprEscapedPath":   "{{ $expr|pathEscape }}",
-		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
-		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
-		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
-	}, map[string]string{
-		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
-		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
-		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
-		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
-		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
-	})
-
-	// query function
-	f(AlertTplData{
-		Expr: `vm_rows{"label"="bar"}>0`,
-	}, map[string]string{
-		"summary": `{{ query "foo" | first | value }}`,
-		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
-	}, map[string]string{
-		"summary": "1",
-		"desc":    "bar 1;garply 2;",
-	})
-
-	// external
-	f(AlertTplData{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"url":         "{{ $externalURL }}",
-		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
-	}, map[string]string{
-		"url":         extURL,
-		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
-		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
-	})
-
-	// alert, group IDs & ActiveAt time
-	f(AlertTplData{
-		AlertID:  42,
-		GroupID:  24,
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"url":     "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
-		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
-	}, map[string]string{
-		"url":     "/api/v1/alert?alertID=42&groupID=24",
-		"diagram": "![](http://example.com?render=1660941298",
-	})
-
-	// ActiveAt time is nil
-	f(AlertTplData{}, map[string]string{
-		"default_time": "{{$activeAt}}",
-	}, map[string]string{
-		"default_time": "0001-01-01 00:00:00 +0000 UTC",
-	})
-
-	// ActiveAt custom format
-	f(AlertTplData{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
-	}, map[string]string{
-		"fire_time": "2022/08/19 20:34:58",
-	})
-
-	// ActiveAt query range
-	f(AlertTplData{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
-	}, map[string]string{
-		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
-	})
+	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
+}
+
+func mkTemplate(current, replacement any) textTemplate {
+	tmpl := textTemplate{}
+	if current != nil {
+		switch val := current.(type) {
+		case string:
+			tmpl.current = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	if replacement != nil {
+		switch val := replacement.(type) {
+		case string:
+			tmpl.replacement = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	return tmpl
+}
+
+func equalTemplates(tmpls ...*textTpl.Template) bool {
+	var cmp *textTpl.Template
+	for i, tmpl := range tmpls {
+		if i == 0 {
+			cmp = tmpl
+		} else {
+			if cmp == nil || tmpl == nil {
+				if cmp != tmpl {
+					return false
+				}
+				continue
+			}
+			if len(tmpl.Templates()) != len(cmp.Templates()) {
+				return false
+			}
+			for _, t := range tmpl.Templates() {
+				tp := cmp.Lookup(t.Name())
+				if tp == nil {
+					return false
+				}
+				if tp.Root.String() != t.Root.String() {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+func TestTemplatesLoad_Failure(t *testing.T) {
+	f := func(pathPatterns []string, expectedErrStr string) {
+		t.Helper()
+
+		err := Load(pathPatterns, url.URL{})
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		errStr := err.Error()
+		if !strings.Contains(errStr, expectedErrStr) {
+			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
+		}
+	}
+
+	// load template with syntax error
+	f([]string{
+		"templates/other/nested/bad0-*.tpl",
+		"templates/test/good0-*.tpl",
+	}, "failed to parse template glob")
+}
+
+func TestTemplatesLoad_Success(t *testing.T) {
+	f := func(pathPatterns []string, expectedTmpl textTemplate) {
+		t.Helper()
+
+		masterTmplOrig := masterTmpl
+		defer func() {
+			masterTmpl = masterTmplOrig
+		}()
+
+		if err := Load(pathPatterns, url.URL{}); err != nil {
+			t.Fatalf("cannot load templates: %s", err)
+		}
+		Reload()
+
+		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
+			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
+		}
+		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
+			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
+		}
+	}
+
+	// non existing path
+	pathPatterns := []string{
+		"templates/non-existing/good-*.tpl",
+		"templates/absent/good-*.tpl",
+	}
+	expectedTmpl := mkTemplate(``, nil)
+	f(pathPatterns, expectedTmpl)
+
+	// existing path
+	pathPatterns = []string{
+		"templates/test/good0-*.tpl",
+	}
+	expectedTmpl = mkTemplate(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.2" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`, nil)
+	f(pathPatterns, expectedTmpl)
+
+	// existing path defined template override
+	pathPatterns = []string{
+		"templates/other/nested/good0-*.tpl",
+	}
+	expectedTmpl = mkTemplate(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.1" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`, nil)
+	f(pathPatterns, expectedTmpl)
 }

app/vmauth/main.go

@@ -222,8 +222,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		isDefault = true
 	}
 
-	rtb := getReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
-	defer putReadTrackingBody(rtb)
+	rtb := newReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
 	r.Body = rtb
 
 	maxAttempts := up.getBackendsCount()
@@ -559,22 +558,11 @@ type readTrackingBody struct {
 	bufComplete bool
 }
 
-func (rtb *readTrackingBody) reset() {
-	rtb.maxBodySize = 0
-	rtb.r = nil
-	rtb.buf = rtb.buf[:0]
-	rtb.readBuf = nil
-	rtb.cannotRetry = false
-	rtb.bufComplete = false
-}
-
-func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
-	v := readTrackingBodyPool.Get()
-	if v == nil {
-		v = &readTrackingBody{}
-	}
-	rtb := v.(*readTrackingBody)
-
+func newReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
+	// do not use sync.Pool there
+	// since http.RoundTrip may still use request body after return
+	// See this issue for details https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
+	rtb := &readTrackingBody{}
 	if maxBodySize < 0 {
 		maxBodySize = 0
 	}
@@ -597,13 +585,6 @@ func (r *zeroReader) Close() error {
 	return nil
 }
 
-func putReadTrackingBody(rtb *readTrackingBody) {
-	rtb.reset()
-	readTrackingBodyPool.Put(rtb)
-}
-
-var readTrackingBodyPool sync.Pool
-
 // Read implements io.Reader interface.
 func (rtb *readTrackingBody) Read(p []byte) (int, error) {
 	if len(rtb.readBuf) > 0 {

app/vmauth/main_test.go

@@ -195,7 +195,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=401
-The provided authKey doesn't match -reloadAuthKey`
+Expected to receive non-empty authKey when -reloadAuthKey is set`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	if err := reloadAuthKey.Set(origAuthKey); err != nil {
 		t.Fatalf("unexpected error: %s", err)
@@ -545,8 +545,7 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")
@@ -581,8 +580,7 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 		t.Helper()
 
 		// Check the case with partial read
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		for i := 0; i < len(s); i++ {
 			buf := make([]byte, i)
@@ -631,8 +629,7 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")
@@ -681,8 +678,7 @@ func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")

app/vmctl/prometheus/prometheus.go

@@ -40,15 +40,15 @@ type filter struct {
 	labelValue string
 }
 
-func (f filter) inRange(min, max int64) bool {
+func (f filter) inRange(minV, maxV int64) bool {
 	fmin, fmax := f.min, f.max
-	if min == 0 {
-		fmin = min
+	if minV == 0 {
+		fmin = minV
 	}
 	if fmax == 0 {
-		fmax = max
+		fmax = maxV
 	}
-	return min <= fmax && fmin <= max
+	return minV <= fmax && fmin <= maxV
 }
 
 // NewClient creates and validates new Client
@@ -59,13 +59,13 @@ func NewClient(cfg Config) (*Client, error) {
 		return nil, fmt.Errorf("failed to open snapshot %q: %s", cfg.Snapshot, err)
 	}
 	c := &Client{DBReadOnly: db}
-	min, max, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
+	minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse time in filter: %s", err)
 	}
 	c.filter = filter{
-		min:        min,
-		max:        max,
+		min:        minTime,
+		max:        maxTime,
 		label:      cfg.Filter.Label,
 		labelValue: cfg.Filter.LabelValue,
 	}

app/vmselect/graphite/aggr.go

@@ -98,13 +98,13 @@ func aggrMin(values []float64) float64 {
 	if pos < 0 {
 		return nan
 	}
-	min := values[pos]
+	minV := values[pos]
 	for _, v := range values[pos+1:] {
-		if !math.IsNaN(v) && v < min {
-			min = v
+		if !math.IsNaN(v) && v < minV {
+			minV = v
 		}
 	}
-	return min
+	return minV
 }
 
 func aggrMax(values []float64) float64 {
@@ -112,13 +112,13 @@ func aggrMax(values []float64) float64 {
 	if pos < 0 {
 		return nan
 	}
-	max := values[pos]
+	maxV := values[pos]
 	for _, v := range values[pos+1:] {
-		if !math.IsNaN(v) && v > max {
-			max = v
+		if !math.IsNaN(v) && v > maxV {
+			maxV = v
 		}
 	}
-	return max
+	return maxV
 }
 
 func aggrDiff(values []float64) float64 {
@@ -177,12 +177,12 @@ func aggrCount(values []float64) float64 {
 }
 
 func aggrRange(values []float64) float64 {
-	min := aggrMin(values)
-	if math.IsNaN(min) {
+	minV := aggrMin(values)
+	if math.IsNaN(minV) {
 		return nan
 	}
-	max := aggrMax(values)
-	return max - min
+	maxV := aggrMax(values)
+	return maxV - minV
 }
 
 func aggrMultiply(values []float64) float64 {

app/vmselect/graphite/transform.go

@@ -2594,17 +2594,17 @@ func transformMinMax(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, e
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		min := aggrMin(values)
-		if math.IsNaN(min) {
-			min = 0
+		minV := aggrMin(values)
+		if math.IsNaN(minV) {
+			minV = 0
 		}
-		max := aggrMax(values)
-		if math.IsNaN(max) {
-			max = 0
+		maxV := aggrMax(values)
+		if math.IsNaN(maxV) {
+			maxV = 0
 		}
-		vRange := max - min
+		vRange := maxV - minV
 		for i, v := range values {
-			v = (v - min) / vRange
+			v = (v - minV) / vRange
 			if math.IsInf(v, 0) {
 				v = 0
 			}
@@ -2975,9 +2975,9 @@ func transformRemoveAbovePercentile(ec *evalConfig, fe *graphiteql.FuncExpr) (ne
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		max := aggrFunc(values)
+		maxV := aggrFunc(values)
 		for i, v := range values {
-			if v > max {
+			if v > maxV {
 				values[i] = nan
 			}
 		}
@@ -3035,9 +3035,9 @@ func transformRemoveBelowPercentile(ec *evalConfig, fe *graphiteql.FuncExpr) (ne
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		min := aggrFunc(values)
+		minV := aggrFunc(values)
 		for i, v := range values {
-			if v < min {
+			if v < minV {
 				values[i] = nan
 			}
 		}
@@ -4514,11 +4514,11 @@ func transformOffsetToZero(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesF
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		min := aggrMin(values)
+		minV := aggrMin(values)
 		for i, v := range values {
-			values[i] = v - min
+			values[i] = v - minV
 		}
-		s.Tags["offsetToZero"] = fmt.Sprintf("%g", min)
+		s.Tags["offsetToZero"] = fmt.Sprintf("%g", minV)
 		s.Name = fmt.Sprintf("offsetToZero(%s)", s.Name)
 		s.expr = fe
 		s.pathExpression = s.Name
@@ -4567,29 +4567,29 @@ func transformPerSecond(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc
 	return f, nil
 }
 
-func nonNegativeDelta(curr, prev, max, min float64) (float64, float64) {
-	if !math.IsNaN(max) && curr > max {
+func nonNegativeDelta(currV, prevV, maxV, minV float64) (float64, float64) {
+	if !math.IsNaN(maxV) && currV > maxV {
 		return nan, nan
 	}
-	if !math.IsNaN(min) && curr < min {
+	if !math.IsNaN(minV) && currV < minV {
 		return nan, nan
 	}
-	if math.IsNaN(curr) || math.IsNaN(prev) {
-		return nan, curr
+	if math.IsNaN(currV) || math.IsNaN(prevV) {
+		return nan, currV
 	}
-	if curr >= prev {
-		return curr - prev, curr
+	if currV >= prevV {
+		return currV - prevV, currV
 	}
-	if !math.IsNaN(max) {
-		if math.IsNaN(min) {
-			min = float64(0)
+	if !math.IsNaN(maxV) {
+		if math.IsNaN(minV) {
+			minV = float64(0)
 		}
-		return max + 1 + curr - prev - min, curr
+		return maxV + 1 + currV - prevV - minV, currV
 	}
-	if !math.IsNaN(min) {
-		return curr - min, curr
+	if !math.IsNaN(minV) {
+		return currV - minV, currV
 	}
-	return nan, curr
+	return nan, currV
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.threshold
@@ -4941,8 +4941,8 @@ func transformSortByMinima(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesF
 	}
 	// Filter out series with all the values smaller than 0
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
-		max := aggrMax(s.Values)
-		if math.IsNaN(max) || max <= 0 {
+		maxV := aggrMax(s.Values)
+		if math.IsNaN(maxV) || maxV <= 0 {
 			return nil, nil
 		}
 		return s, nil

app/vmselect/main.go

@@ -29,13 +29,13 @@ import (
 )
 
 var (
-	deleteAuthKey         = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries. It overrides -httpAuth.*")
+	deleteAuthKey         = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries. It could be passed via authKey query arg. It overrides -httpAuth.*")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It overrides -httpAuth.*")
+	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It could be passed via authKey query arg. It overrides -httpAuth.*")
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
 	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules")

app/vmselect/prometheus/prometheus.go

@@ -481,6 +481,8 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
 	if err != nil {
 		return err
 	}
+	cp.deadline = searchutils.GetDeadlineForDelete(r, startTime)
+
 	if !cp.IsDefaultTimeRange() {
 		return fmt.Errorf("start=%d and end=%d args aren't supported. Remove these args from the query in order to delete all the matching metrics", cp.start, cp.end)
 	}

app/vmselect/promql/aggr.go

@@ -295,13 +295,13 @@ func aggrFuncMin(tss []*timeseries) []*timeseries {
 	}
 	dst := tss[0]
 	for i := range dst.Values {
-		min := dst.Values[i]
+		minV := dst.Values[i]
 		for _, ts := range tss {
-			if math.IsNaN(min) || ts.Values[i] < min {
-				min = ts.Values[i]
+			if math.IsNaN(minV) || ts.Values[i] < minV {
+				minV = ts.Values[i]
 			}
 		}
-		dst.Values[i] = min
+		dst.Values[i] = minV
 	}
 	return tss[:1]
 }
@@ -313,13 +313,13 @@ func aggrFuncMax(tss []*timeseries) []*timeseries {
 	}
 	dst := tss[0]
 	for i := range dst.Values {
-		max := dst.Values[i]
+		maxV := dst.Values[i]
 		for _, ts := range tss {
-			if math.IsNaN(max) || ts.Values[i] > max {
-				max = ts.Values[i]
+			if math.IsNaN(maxV) || ts.Values[i] > maxV {
+				maxV = ts.Values[i]
 			}
 		}
-		dst.Values[i] = max
+		dst.Values[i] = maxV
 	}
 	return tss[:1]
 }
@@ -793,7 +793,7 @@ func fillNaNsAtIdx(idx int, k float64, tss []*timeseries) {
 	}
 }
 
-func getIntK(k float64, max int) int {
+func getIntK(k float64, maxV int) int {
 	if math.IsNaN(k) {
 		return 0
 	}
@@ -801,38 +801,38 @@ func getIntK(k float64, max int) int {
 	if kn < 0 {
 		return 0
 	}
-	if kn > max {
-		return max
+	if kn > maxV {
+		return maxV
 	}
 	return kn
 }
 
 func minValue(values []float64) float64 {
-	min := nan
-	for len(values) > 0 && math.IsNaN(min) {
-		min = values[0]
+	minV := nan
+	for len(values) > 0 && math.IsNaN(minV) {
+		minV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if !math.IsNaN(v) && v < min {
-			min = v
+		if !math.IsNaN(v) && v < minV {
+			minV = v
 		}
 	}
-	return min
+	return minV
 }
 
 func maxValue(values []float64) float64 {
-	max := nan
-	for len(values) > 0 && math.IsNaN(max) {
-		max = values[0]
+	maxV := nan
+	for len(values) > 0 && math.IsNaN(maxV) {
+		maxV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if !math.IsNaN(v) && v > max {
-			max = v
+		if !math.IsNaN(v) && v > maxV {
+			maxV = v
 		}
 	}
-	return max
+	return maxV
 }
 
 func avgValue(values []float64) float64 {

app/vmselect/promql/rollup.go

@@ -374,8 +374,8 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 	preFunc := func(_ []float64, _ []int64) {}
 	funcName = strings.ToLower(funcName)
 	if rollupFuncsRemoveCounterResets[funcName] {
-		preFunc = func(values []float64, _ []int64) {
-			removeCounterResets(values)
+		preFunc = func(values []float64, timestamps []int64) {
+			removeCounterResets(values, timestamps, lookbackDelta)
 		}
 	}
 	samplesScannedPerCall := rollupFuncsSamplesScannedPerCall[funcName]
@@ -486,8 +486,8 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 		for _, aggrFuncName := range aggrFuncNames {
 			if rollupFuncsRemoveCounterResets[aggrFuncName] {
 				// There is no need to save the previous preFunc, since it is either empty or the same.
-				preFunc = func(values []float64, _ []int64) {
-					removeCounterResets(values)
+				preFunc = func(values []float64, timestamps []int64) {
+					removeCounterResets(values, timestamps, lookbackDelta)
 				}
 			}
 			rf := rollupAggrFuncs[aggrFuncName]
@@ -521,7 +521,7 @@ type rollupFuncArg struct {
 	timestamps []int64
 
 	// Real value preceding values.
-	// Is populated if preceding value is within the staleness interval.
+	// Is populated if preceding value is within the -search.maxStalenessInterval (rc.LookbackDelta).
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -768,7 +768,13 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		rfa.realPrevValue = nan
 		if i > 0 {
 			prevValue, prevTimestamp := values[i-1], timestamps[i-1]
-			if (tEnd - prevTimestamp) < maxPrevInterval {
+			// set realPrevValue if rc.LookbackDelta == 0
+			// or if distance between datapoint in prev interval and beginning of this interval
+			// doesn't exceed LookbackDelta.
+			// https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1381
+			// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/894
+			// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8045
+			if rc.LookbackDelta == 0 || (tStart-prevTimestamp) < rc.LookbackDelta {
 				rfa.realPrevValue = prevValue
 			}
 		}
@@ -894,7 +900,7 @@ func getMaxPrevInterval(scrapeInterval int64) int64 {
 	return scrapeInterval + scrapeInterval/8
 }
 
-func removeCounterResets(values []float64) {
+func removeCounterResets(values []float64, timestamps []int64, maxStalenessInterval int64) {
 	// There is no need in handling NaNs here, since they are impossible
 	// on values from vmstorage.
 	if len(values) == 0 {
@@ -913,6 +919,16 @@ func removeCounterResets(values []float64) {
 				correction += prevValue
 			}
 		}
+		if i > 0 && maxStalenessInterval > 0 {
+			gap := timestamps[i] - timestamps[i-1]
+			if gap > maxStalenessInterval {
+				// reset correction if gap between samples exceeds staleness interval
+				// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072
+				correction = 0
+				prevValue = v
+				continue
+			}
+		}
 		prevValue = v
 		values[i] = v + correction
 		// Check again, there could be precision error in float operations,
@@ -1685,9 +1701,9 @@ func rollupRateOverSum(rfa *rollupFuncArg) float64 {
 }
 
 func rollupRange(rfa *rollupFuncArg) float64 {
-	max := rollupMax(rfa)
-	min := rollupMin(rfa)
-	return max - min
+	maxV := rollupMax(rfa)
+	minV := rollupMin(rfa)
+	return maxV - minV
 }
 
 func rollupSum2(rfa *rollupFuncArg) float64 {
@@ -2195,38 +2211,38 @@ func rollupClose(rfa *rollupFuncArg) float64 {
 
 func rollupHigh(rfa *rollupFuncArg) float64 {
 	values := getCandlestickValues(rfa)
-	max := getFirstValueForCandlestick(rfa)
-	if math.IsNaN(max) {
+	maxV := getFirstValueForCandlestick(rfa)
+	if math.IsNaN(maxV) {
 		if len(values) == 0 {
 			return nan
 		}
-		max = values[0]
+		maxV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if v > max {
-			max = v
+		if v > maxV {
+			maxV = v
 		}
 	}
-	return max
+	return maxV
 }
 
 func rollupLow(rfa *rollupFuncArg) float64 {
 	values := getCandlestickValues(rfa)
-	min := getFirstValueForCandlestick(rfa)
-	if math.IsNaN(min) {
+	minV := getFirstValueForCandlestick(rfa)
+	if math.IsNaN(minV) {
 		if len(values) == 0 {
 			return nan
 		}
-		min = values[0]
+		minV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if v < min {
-			min = v
+		if v < minV {
+			minV = v
 		}
 	}
-	return min
+	return minV
 }
 
 func rollupModeOverTime(rfa *rollupFuncArg) float64 {

app/vmselect/promql/rollup_test.go

@@ -117,31 +117,49 @@ func TestRollupIderivDuplicateTimestamps(t *testing.T) {
 }
 
 func TestRemoveCounterResets(t *testing.T) {
-	removeCounterResets(nil)
+	removeCounterResets(nil, nil, 0)
 
 	values := append([]float64{}, testValues...)
-	removeCounterResets(values)
+	timestamps := append([]int64{}, testTimestamps...)
+	removeCounterResets(values, timestamps, 0)
 	valuesExpected := []float64{123, 157, 167, 188, 221, 255, 320, 332, 364, 396, 398, 398}
 	testRowsEqual(t, values, testTimestamps, valuesExpected, testTimestamps)
 
 	// removeCounterResets doesn't expect negative values, so it doesn't work properly with them.
 	values = []float64{-100, -200, -300, -400}
-	removeCounterResets(values)
-	valuesExpected = []float64{-100, -100, -100, -100}
 	timestampsExpected := []int64{0, 1, 2, 3}
+	removeCounterResets(values, timestampsExpected, 0)
+	valuesExpected = []float64{-100, -100, -100, -100}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
 
 	// verify how partial counter reset is handled.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2787
 	values = []float64{100, 95, 120, 119, 139, 50}
-	removeCounterResets(values)
-	valuesExpected = []float64{100, 100, 125, 125, 145, 195}
 	timestampsExpected = []int64{0, 1, 2, 3, 4, 5}
+	removeCounterResets(values, timestampsExpected, 0)
+	valuesExpected = []float64{100, 100, 125, 125, 145, 195}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
 
+	// verify that staleness interval is respected during resets
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072
+	values = []float64{10, 12, 14, 4, 6, 8, 6, 8, 4, 6}
+	timestamps = []int64{10, 20, 30, 60, 70, 80, 90, 100, 120, 130}
+	valuesExpected = []float64{10, 12, 14, 4, 6, 8, 14, 16, 4, 6}
+	removeCounterResets(values, timestamps, 10)
+	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
+
+	// verify that staleness is respected if there was no counter reset
+	// but correction was made previously
+	values = []float64{10, 12, 2, 4}
+	timestamps = []int64{10, 20, 30, 60}
+	valuesExpected = []float64{10, 12, 14, 4}
+	removeCounterResets(values, timestamps, 10)
+	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
+
 	// verify results always increase monotonically with possible float operations precision error
 	values = []float64{34.094223, 2.7518, 2.140669, 0.044878, 1.887095, 2.546569, 2.490149, 0.045, 0.035684, 0.062454, 0.058296}
-	removeCounterResets(values)
+	timestampsExpected = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+	removeCounterResets(values, timestampsExpected, 0)
 	var prev float64
 	for i, v := range values {
 		if v < prev {
@@ -166,7 +184,7 @@ func TestDeltaValues(t *testing.T) {
 
 	// remove counter resets
 	values = append([]float64{}, testValues...)
-	removeCounterResets(values)
+	removeCounterResets(values, testTimestamps, 0)
 	deltaValues(values)
 	valuesExpected = []float64{34, 10, 21, 33, 34, 65, 12, 32, 32, 2, 0, 0}
 	testRowsEqual(t, values, testTimestamps, valuesExpected, testTimestamps)
@@ -188,7 +206,7 @@ func TestDerivValues(t *testing.T) {
 
 	// remove counter resets
 	values = append([]float64{}, testValues...)
-	removeCounterResets(values)
+	removeCounterResets(values, testTimestamps, 0)
 	derivValues(values, testTimestamps)
 	valuesExpected = []float64{3400, 1111.111111111111, 1750, 2538.4615384615386, 3090.909090909091, 3611.1111111111113,
 		6000, 1882.3529411764705, 1777.7777777777778, 400, 0, 0}
@@ -219,7 +237,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
 	rfa.timestamps = append(rfa.timestamps, testTimestamps...)
 	rfa.window = rfa.timestamps[len(rfa.timestamps)-1] - rfa.timestamps[0]
 	if rollupFuncsRemoveCounterResets[funcName] {
-		removeCounterResets(rfa.values)
+		removeCounterResets(rfa.values, rfa.timestamps, 0)
 	}
 	for i := 0; i < 5; i++ {
 		v := rf(&rfa)
@@ -1590,17 +1608,60 @@ func TestRollupDelta(t *testing.T) {
 	f(100, nan, nan, nil, 0)
 }
 
-func TestRollupIncreaseWithStaleness(t *testing.T) {
+func TestRollupDeltaWithStaleness(t *testing.T) {
 	// there is a gap between samples in the dataset below
 	timestamps := []int64{0, 15000, 30000, 70000}
 	values := []float64{1, 1, 1, 1}
 
-	t.Run("step > gap", func(t *testing.T) {
+	// if step > gap, then delta will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:               rollupDelta,
 			Start:              0,
 			End:                70000,
-			Step:               35000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupDelta,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta==0 then delta will always respect value before gap
+	// as LookbackDelta=0 ignores staleness
+	t.Run("step<gap;LookbackDelta=0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupDelta,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			LookbackDelta:      0,
 			Window:             0,
 			MaxPointsPerSeries: 1e4,
 		}
@@ -1609,12 +1670,14 @@ func TestRollupIncreaseWithStaleness(t *testing.T) {
 		if samplesScanned != 8 {
 			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
-		valuesExpected := []float64{1, 0, 0}
-		timestampsExpected := []int64{0, 35e3, 70e3}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 
-	t.Run("step < gap", func(t *testing.T) {
+	// if step < gap and LookbackDelta>0 then delta will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:               rollupDelta,
 			Start:              0,
@@ -1622,6 +1685,7 @@ func TestRollupIncreaseWithStaleness(t *testing.T) {
 			Step:               10000,
 			Window:             0,
 			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
 		}
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
@@ -1656,3 +1720,116 @@ func TestRollupIncreaseWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
+
+func TestRollupIncreasePureWithStaleness(t *testing.T) {
+	// there is a gap between samples in the dataset below
+	timestamps := []int64{0, 15000, 30000, 70000}
+	values := []float64{1, 1, 1, 1}
+
+	// if step > gap, then delta will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta==0 then delta will always respect value before gap
+	// as LookbackDelta=0 ignores staleness
+	t.Run("step<gap;LookbackDelta=0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			LookbackDelta:      0,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta>0 then delta will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// there is a staleness marker between samples in the dataset below
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{1, 1, 1, decimal.StaleNaN, 1}
+
+	t.Run("staleness marker", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                40000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, nan, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}

app/vmselect/searchutils/searchutils.go

@@ -15,6 +15,7 @@ import (
 
 var (
 	maxExportDuration        = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
+	maxDeleteDuration        = flag.Duration("search.maxDeleteDuration", time.Minute*5, "The maximum duration for /api/v1/admin/tsdb/delete_series call")
 	maxQueryDuration         = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden to a smaller value on a per-query basis via 'timeout' query arg")
 	maxStatusRequestDuration = flag.Duration("search.maxStatusRequestDuration", time.Minute*5, "The maximum duration for /api/v1/status/* requests")
 	maxLabelsAPIDuration     = flag.Duration("search.maxLabelsAPIDuration", time.Second*5, "The maximum duration for /api/v1/labels, /api/v1/label/.../values and /api/v1/series requests. "+
@@ -58,6 +59,12 @@ func GetDeadlineForLabelsAPI(r *http.Request, startTime time.Time) Deadline {
 	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxLabelsAPIDuration")
 }
 
+// GetDeadlineForDelete returns deadline for the given request to /api/v1/admin/tsdb/delete_series.
+func GetDeadlineForDelete(r *http.Request, startTime time.Time) Deadline {
+	dMax := maxDeleteDuration.Milliseconds()
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxDeleteDuration")
+}
+
 func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) Deadline {
 	d, err := httputils.GetDuration(r, "timeout", 0)
 	if err != nil {

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.63479b72.css",
-    "main.js": "./static/js/main.256ee243.js",
+    "main.css": "./static/css/main.af583aad.css",
+    "main.js": "./static/js/main.1413b18d.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.63479b72.css",
-    "static/js/main.256ee243.js"
+    "static/css/main.af583aad.css",
+    "static/js/main.1413b18d.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.256ee243.js"></script><link href="./static/css/main.63479b72.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.1413b18d.js"></script><link href="./static/css/main.af583aad.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.23.4 AS build-web-stage
+FROM golang:1.23.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.21.0
+FROM alpine:3.21.2
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/src/api/types.ts

@@ -1,3 +1,6 @@
+import uPlot from "uplot";
+import { ReactNode } from "react";
+
 export interface MetricBase {
   group: number;
   metric: {
@@ -6,13 +9,13 @@ export interface MetricBase {
 }
 
 export interface MetricResult extends MetricBase {
-  values: [number, string][]
+  values: [number, string][];
 }
 
 
 export interface InstantMetricResult extends MetricBase {
-  value?: [number, string]
-  values?: [number, string][]
+  value?: [number, string];
+  values?: [number, string][];
 }
 
 export interface ExportMetricResult extends MetricBase {
@@ -44,9 +47,23 @@ export interface LogHits {
   timestamps: string[];
   values: number[];
   total?: number;
-  fields: {
-    [key: string]: string;
-  };
+  fields: { [key: string]: string; };
+  _isOther: boolean;
+}
+
+export interface LegendLogHits {
+  label: string;
+  total: number;
+  totalHits: number;
+  isOther: boolean;
+  fields: { [key: string]: string; };
+  stroke?: uPlot.Series.Stroke;
+}
+
+export interface LegendLogHitsMenu {
+  title: string;
+  icon?: ReactNode;
+  handler?: () => void;
 }
 
 export interface ReportMetaData {

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsChart.tsx

@@ -1,22 +1,23 @@
-import React, { FC, useMemo, useRef, useState } from "preact/compat";
+import React, { FC, useCallback, useMemo, useRef, useState } from "preact/compat";
 import "./style.scss";
 import "uplot/dist/uPlot.min.css";
 import useElementSize from "../../../hooks/useElementSize";
 import uPlot, { AlignedData } from "uplot";
 import { useEffect } from "react";
-import useBarHitsOptions from "./hooks/useBarHitsOptions";
+import useBarHitsOptions, { getLabelFromLogHit } from "./hooks/useBarHitsOptions";
 import BarHitsTooltip from "./BarHitsTooltip/BarHitsTooltip";
 import { TimeParams } from "../../../types";
 import usePlotScale from "../../../hooks/uplot/usePlotScale";
 import useReadyChart from "../../../hooks/uplot/useReadyChart";
 import useZoomChart from "../../../hooks/uplot/useZoomChart";
 import classNames from "classnames";
-import { LogHits } from "../../../api/types";
+import { LegendLogHits, LogHits } from "../../../api/types";
 import { addSeries, delSeries, setBand } from "../../../utils/uplot";
 import { GraphOptions, GRAPH_STYLES } from "./types";
 import BarHitsOptions from "./BarHitsOptions/BarHitsOptions";
 import stack from "../../../utils/uplot/stack";
 import BarHitsLegend from "./BarHitsLegend/BarHitsLegend";
+import { calculateTotalHits, sortLogHits } from "../../../utils/logs";
 
 interface Props {
   logHits: LogHits[];
@@ -57,6 +58,29 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
     graphOptions
   });
 
+  const prepareLegend = useCallback((hits: LogHits[], totalHits: number): LegendLogHits[] => {
+    return hits.map((hit) => {
+      const label = getLabelFromLogHit(hit);
+
+      const legendItem: LegendLogHits = {
+        label,
+        isOther: hit._isOther,
+        fields: hit.fields,
+        total: hit.total || 0,
+        totalHits,
+        stroke: series.find((s) => s.label === label)?.stroke,
+      };
+
+      return legendItem;
+    }).sort(sortLogHits("total"));
+  }, [series]);
+
+
+  const legendDetails: LegendLogHits[] = useMemo(() => {
+    const totalHits = calculateTotalHits(logHits);
+    return prepareLegend(logHits, totalHits);
+  }, [logHits, prepareLegend]);
+
   useEffect(() => {
     if (!uPlotInst) return;
     delSeries(uPlotInst);
@@ -121,6 +145,7 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
         <BarHitsLegend
           uPlotInst={uPlotInst}
           onApplyFilter={onApplyFilter}
+          legendDetails={legendDetails}
         />
       )}
     </div>

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/BarHitsLegend.tsx

@@ -1,83 +1,53 @@
-import React, { FC, useCallback, useEffect, useState } from "preact/compat";
+import React, { FC, useEffect, useState } from "preact/compat";
 import uPlot, { Series } from "uplot";
 import "./style.scss";
 import "../../Line/Legend/style.scss";
-import classNames from "classnames";
-import { MouseEvent } from "react";
-import { isMacOs } from "../../../../utils/detect-device";
-import Tooltip from "../../../Main/Tooltip/Tooltip";
-import { getStreamPairs } from "../../../../utils/logs";
+import BarHitsLegendItem from "./BarHitsLegendItem";
+import { LegendLogHits } from "../../../../api/types";
 
 interface Props {
   uPlotInst: uPlot;
+  legendDetails: LegendLogHits[];
   onApplyFilter: (value: string) => void;
 }
 
-const BarHitsLegend: FC<Props> = ({ uPlotInst, onApplyFilter }) => {
+const BarHitsLegend: FC<Props> = ({ uPlotInst, legendDetails, onApplyFilter }) => {
   const [series, setSeries] = useState<Series[]>([]);
-  const [pairs, setPairs] = useState<string[][]>([]);
+  const totalHits = legendDetails[0]?.totalHits || 0;
 
-  const updateSeries = useCallback(() => {
-    const series = uPlotInst.series.filter(s => s.scale !== "x");
-    setSeries(series);
-    setPairs(series.map(s => getStreamPairs(s.label || "")));
+  const getSeries = () => {
+    return uPlotInst.series.filter(s => s.scale !== "x");
+  };
+
+  const handleRedrawGraph = () => {
+    uPlotInst.redraw();
+    setSeries(getSeries());
+  };
+
+  useEffect(() => {
+    setSeries(getSeries());
   }, [uPlotInst]);
 
-  const handleClickByValue = (value: string) => (e: MouseEvent<HTMLDivElement>) => {
-    const metaKey = e.metaKey || e.ctrlKey;
-    if (!metaKey) return;
-    onApplyFilter(`{${value}}` || "");
-    updateSeries();
-    uPlotInst.redraw();
-  };
-
-  const handleClickByStream = (target: Series) => (e: MouseEvent<HTMLDivElement>) => {
-    const metaKey = e.metaKey || e.ctrlKey;
-    if (metaKey) return;
-    target.show = !target.show;
-    updateSeries();
-    uPlotInst.redraw();
-  };
-
-  useEffect(updateSeries, [uPlotInst]);
-
   return (
     <div className="vm-bar-hits-legend">
-      {series.map((s, i) => (
-        <Tooltip
-          key={s.label}
-          title={(
-            <ul className="vm-bar-hits-legend-info">
-              <li>Click to {s.show ? "hide" : "show"} the _stream.</li>
-              <li>{isMacOs() ? "Cmd" : "Ctrl"} + Click to filter by the _stream.</li>
-            </ul>
-          )}
-        >
-          <div
-            className={classNames({
-              "vm-bar-hits-legend-item": true,
-              "vm-bar-hits-legend-item_hide": !s.show,
-            })}
-            onClick={handleClickByStream(s)}
-          >
-            <div
-              className="vm-bar-hits-legend-item__marker"
-              style={{ backgroundColor: `${(s?.stroke as () => string)?.()}` }}
-            />
-            <div className="vm-bar-hits-legend-item-pairs">
-              {pairs[i].map(value => (
-                <span
-                  className="vm-bar-hits-legend-item-pairs__value"
-                  key={value}
-                  onClick={handleClickByValue(value)}
-                >
-                  {value}
-                </span>
-              ))}
-            </div>
-          </div>
-        </Tooltip>
+      {legendDetails.map((legend) => (
+        <BarHitsLegendItem
+          key={legend.label}
+          legend={legend}
+          series={series}
+          onRedrawGraph={handleRedrawGraph}
+          onApplyFilter={onApplyFilter}
+        />
       ))}
+      <div className="vm-bar-hits-legend-info">
+        <div>
+          Total hits: <b>{totalHits.toLocaleString("en-US")}</b>
+        </div>
+        <div>
+          <code>L-Click</code> toggles visibility.&nbsp;
+          <code>R-Click</code> opens menu.
+        </div>
+      </div>
     </div>
   );
 };

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/BarHitsLegendItem.tsx

@@ -0,0 +1,92 @@
+import React, { FC, useMemo, useRef, useState } from "preact/compat";
+import classNames from "classnames";
+import { Series } from "uplot";
+import { MouseEvent } from "react";
+import { LegendLogHits } from "../../../../api/types";
+import { getStreamPairs } from "../../../../utils/logs";
+import { formatNumberShort } from "../../../../utils/math";
+import Popper from "../../../Main/Popper/Popper";
+import useBoolean from "../../../../hooks/useBoolean";
+import LegendHitsMenu from "../LegendHitsMenu/LegendHitsMenu";
+
+interface Props {
+  legend: LegendLogHits;
+  series: Series[];
+  onRedrawGraph: () => void;
+  onApplyFilter: (value: string) => void;
+}
+
+const BarHitsLegendItem: FC<Props> = ({ legend, series, onRedrawGraph, onApplyFilter }) => {
+  const {
+    value: openContextMenu,
+    setTrue: handleOpenContextMenu,
+    setFalse: handleCloseContextMenu,
+  } = useBoolean(false);
+
+  const legendRef = useRef<HTMLDivElement>(null);
+  const [clickPosition, setClickPosition] = useState<{ top: number; left: number } | null>(null);
+
+  const targetSeries = useMemo(() => series.find(s => s.label === legend.label), [series]);
+
+  const fields = useMemo(() => getStreamPairs(legend.label), [legend.label]);
+
+  const label = fields.join(", ");
+  const totalShortFormatted = formatNumberShort(legend.total);
+
+  const handleClickByStream = (e: MouseEvent<HTMLDivElement>) => {
+    if (!targetSeries) return;
+
+    if (e.metaKey || e.ctrlKey) {
+      targetSeries.show = !targetSeries.show;
+    } else {
+      const isOnlyTargetVisible = series.every(s => s === targetSeries || !s.show);
+      series.forEach(s => {
+        s.show = isOnlyTargetVisible || (s === targetSeries);
+      });
+    }
+
+    onRedrawGraph();
+  };
+
+  const handleContextMenu = (e: MouseEvent<HTMLDivElement>) => {
+    e.preventDefault();
+    setClickPosition({ top: e.clientY, left: e.clientX });
+    handleOpenContextMenu();
+  };
+
+  return (
+    <div
+      ref={legendRef}
+      className={classNames({
+        "vm-bar-hits-legend-item": true,
+        "vm-bar-hits-legend-item_other": legend.isOther,
+        "vm-bar-hits-legend-item_hide": !targetSeries?.show,
+      })}
+      onClick={handleClickByStream}
+      onContextMenu={handleContextMenu}
+    >
+      <div
+        className="vm-bar-hits-legend-item__marker"
+        style={{ backgroundColor: `${legend.stroke}` }}
+      />
+      <div className="vm-bar-hits-legend-item__label">{label}</div>
+      <span className="vm-bar-hits-legend-item__total">({totalShortFormatted})</span>
+      <Popper
+        placement="fixed"
+        open={openContextMenu}
+        buttonRef={legendRef}
+        placementPosition={clickPosition}
+        onClose={handleCloseContextMenu}
+      >
+        <LegendHitsMenu
+          legend={legend}
+          fields={fields}
+          onApplyFilter={onApplyFilter}
+          onClose={handleCloseContextMenu}
+        />
+      </Popper>
+    </div>
+  );
+};
+
+export default BarHitsLegendItem;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/style.scss

@@ -3,16 +3,16 @@
 .vm-bar-hits-legend {
   display: flex;
   flex-wrap: wrap;
-  gap: $padding-small;
   padding: 0 $padding-small $padding-small;
+  color: $color-text;
 
   &-item {
-    display: grid;
-    grid-template-columns: auto 1fr;
+    max-width: 50%;
+    display: flex;
     align-items: center;
     gap: $padding-small;
-    font-size: 12px;
-    padding: 0 $padding-small;
+    font-size: $font-size-small;
+    padding: $padding-small $padding-global;
     border-radius: $border-radius-small;
     cursor: pointer;
     transition: 0.2s;
@@ -27,34 +27,44 @@
     }
 
     &__marker {
-      width: 14px;
+      min-width: 14px;
+      max-width: 14px;
       height: 14px;
       border: $color-background-block;
     }
 
-    &-pairs {
-      display: flex;
-      gap: $padding-small;
+    &__label {
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
 
-      &__value {
-        padding: $padding-small 0;
-
-        &:hover {
-          text-decoration: underline;
-        }
-
-        &:after {
-          content: ",";
-        }
-
-        &:last-child:after {
-          content: "";
-        }
-      }
+    &__total {
+      color: $color-text-secondary;
+      font-style: italic;
+      grid-column: 2;
     }
   }
 
   &-info {
-    list-style-position: inside;
+    flex-grow: 1;
+    width: 100%;
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding-top: $padding-small;
+    color: $color-text-secondary;
+    font-size: $font-size-small;
+
+    code {
+      display: inline-block;
+      padding: calc($padding-small / 2) $padding-small;
+      font-size: $font-size-small;
+      text-align: center;
+      background-color: $color-background-body;
+      background-repeat: repeat-x;
+      border: $border-divider;
+      border-radius: 4px;
+    }
   }
 }

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/BarHitsOptions.tsx

@@ -5,7 +5,6 @@ import "./style.scss";
 import useStateSearchParams from "../../../../hooks/useStateSearchParams";
 import { useSearchParams } from "react-router-dom";
 import Button from "../../../Main/Button/Button";
-import classNames from "classnames";
 import { SettingsIcon, VisibilityIcon, VisibilityOffIcon } from "../../../Main/Icons";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import Popper from "../../../Main/Popper/Popper";
@@ -24,27 +23,20 @@ const BarHitsOptions: FC<Props> = ({ onChange }) => {
     setFalse: handleCloseOptions,
   } = useBoolean(false);
 
-  const [graphStyle, setGraphStyle] = useStateSearchParams(GRAPH_STYLES.LINE_STEPPED, "graph");
   const [stacked, setStacked] = useStateSearchParams(false, "stacked");
-  const [fill, setFill] = useStateSearchParams(false, "fill");
+  const [fill, setFill] = useStateSearchParams("true", "fill");
   const [hideChart, setHideChart] = useStateSearchParams(false, "hide_chart");
 
   const options: GraphOptions = useMemo(() => ({
-    graphStyle,
+    graphStyle: GRAPH_STYLES.BAR,
     stacked,
-    fill,
+    fill: fill === "true",
     hideChart,
-  }), [graphStyle, stacked, fill, hideChart]);
-
-  const handleChangeGraphStyle = (val: string) => () => {
-    setGraphStyle(val as GRAPH_STYLES);
-    searchParams.set("graph", val);
-    setSearchParams(searchParams);
-  };
+  }), [stacked, fill, hideChart]);
 
   const handleChangeFill = (val: boolean) => {
-    setFill(val);
-    val ? searchParams.set("fill", "true") : searchParams.delete("fill");
+    setFill(`${val}`);
+    searchParams.set("fill", `${val}`);
     setSearchParams(searchParams);
   };
 
@@ -97,21 +89,6 @@ const BarHitsOptions: FC<Props> = ({ onChange }) => {
         title={"Graph settings"}
       >
         <div className="vm-bar-hits-options-settings">
-          <div className="vm-bar-hits-options-settings-item vm-bar-hits-options-settings-item_list">
-            <p className="vm-bar-hits-options-settings-item__title">Graph style:</p>
-            {Object.values(GRAPH_STYLES).map(style => (
-              <div
-                key={style}
-                className={classNames({
-                  "vm-list-item": true,
-                  "vm-list-item_active": graphStyle === style,
-                })}
-                onClick={handleChangeGraphStyle(style)}
-              >
-                {style}
-              </div>
-            ))}
-          </div>
           <div className="vm-bar-hits-options-settings-item">
             <Switch
               label={"Stacked"}
@@ -122,7 +99,7 @@ const BarHitsOptions: FC<Props> = ({ onChange }) => {
           <div className="vm-bar-hits-options-settings-item">
             <Switch
               label={"Fill"}
-              value={fill}
+              value={fill === "true"}
               onChange={handleChangeFill}
             />
           </div>

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/style.scss

@@ -11,12 +11,12 @@
   &-settings {
     display: grid;
     align-items: flex-start;
-    gap: $padding-global;
     min-width: 200px;
+    gap: $padding-global;
+    padding-bottom: $padding-global;
 
     &-item {
-      border-bottom: $border-divider;
-      padding: 0 $padding-global $padding-global;
+      padding: 0 $padding-global;
 
       &_list {
         padding: 0;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/BarHitsTooltip.tsx

@@ -5,6 +5,7 @@ import { DATE_TIME_FORMAT } from "../../../../constants/date";
 import classNames from "classnames";
 import "./style.scss";
 import "../../ChartTooltip/style.scss";
+import { sortLogHits } from "../../../../utils/logs";
 
 interface Props {
   data: AlignedData;
@@ -26,7 +27,7 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
     const tooltipItems = values.map((value, i) => {
       const targetSeries = series[i + 1];
       const stroke = (targetSeries?.stroke as () => string)?.();
-      const label = targetSeries?.label || "other";
+      const label = targetSeries?.label;
       const show = targetSeries?.show;
       return {
         label,
@@ -34,7 +35,7 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
         value,
         show
       };
-    }).filter(item => item.value > 0 && item.show).sort((a, b) => b.value - a.value);
+    }).filter(item => item.value > 0 && item.show).sort(sortLogHits("value"));
 
     const point = {
       top: tooltipItems[0] ? uPlotInst?.valToPos?.(tooltipItems[0].value, "y") || 0 : 0,
@@ -104,16 +105,19 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
               className="vm-chart-tooltip-data__marker"
               style={{ background: item.stroke }}
             />
-            <p>
-              {item.label}: <b>{item.value}</b>
+            <p className="vm-bar-hits-tooltip-item">
+              <span className="vm-bar-hits-tooltip-item__label">{item.label}</span>
+              <span>{item.value.toLocaleString("en-US")}</span>
             </p>
           </div>
         ))}
       </div>
       {tooltipData.values.length > 1 && (
         <div className="vm-chart-tooltip-data">
-          <p>
-              Total records: <b>{tooltipData.total}</b>
+          <span/>
+          <p className="vm-bar-hits-tooltip-item">
+            <span className="vm-bar-hits-tooltip-item__label">Total</span>
+            <span>{tooltipData.total.toLocaleString("en-US")}</span>
           </p>
         </div>
       )}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/style.scss

@@ -9,4 +9,19 @@
     opacity: 1;
     pointer-events: auto;
   }
+
+  &-item {
+    display: grid;
+    grid-template-columns: 1fr auto;
+    align-items: center;
+    gap: $padding-global;
+    max-width: 100%;
+
+    &__label {
+      display: inline-block;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+    }
+  }
 }

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenu.tsx

@@ -0,0 +1,50 @@
+import React, { FC } from "preact/compat";
+import "./style.scss";
+import { LegendLogHits } from "../../../../api/types";
+import LegendHitsMenuStats from "./LegendHitsMenuStats";
+import LegendHitsMenuBase from "./LegendHitsMenuBase";
+import LegendHitsMenuRow from "./LegendHitsMenuRow";
+import LegendHitsMenuFields from "./LegendHitsMenuFields";
+import { LOGS_LIMIT_HITS } from "../../../../constants/logs";
+
+const otherDescription = `aggregated results for fields not in the top ${LOGS_LIMIT_HITS}`;
+
+interface Props {
+  legend: LegendLogHits;
+  fields: string[];
+  onApplyFilter: (value: string) => void;
+  onClose: () => void;
+}
+
+const LegendHitsMenu: FC<Props> = ({ legend, fields, onApplyFilter, onClose }) => {
+  return (
+    <div className="vm-legend-hits-menu">
+      <div className="vm-legend-hits-menu-section">
+        <LegendHitsMenuRow
+          className="vm-legend-hits-menu-row_info"
+          title={legend.isOther ? otherDescription : legend.label}
+        />
+      </div>
+
+      {!legend.isOther && (
+        <LegendHitsMenuBase
+          legend={legend}
+          onApplyFilter={onApplyFilter}
+          onClose={onClose}
+        />
+      )}
+
+      {!legend.isOther && (
+        <LegendHitsMenuFields
+          fields={fields}
+          onApplyFilter={onApplyFilter}
+          onClose={onClose}
+        />
+      )}
+
+      <LegendHitsMenuStats legend={legend}/>
+    </div>
+  );
+};
+
+export default LegendHitsMenu;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuBase.tsx

@@ -0,0 +1,64 @@
+import React, { FC } from "preact/compat";
+import LegendHitsMenuRow from "./LegendHitsMenuRow";
+import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
+import { CopyIcon, FilterIcon, FilterOffIcon } from "../../../Main/Icons";
+import { LegendLogHits, LegendLogHitsMenu } from "../../../../api/types";
+import { LOGS_GROUP_BY } from "../../../../constants/logs";
+
+interface Props {
+  legend: LegendLogHits;
+  onApplyFilter: (value: string) => void;
+  onClose: () => void;
+}
+
+const LegendHitsMenuBase: FC<Props> = ({ legend, onApplyFilter, onClose }) => {
+  const copyToClipboard = useCopyToClipboard();
+
+  const handleAddStreamToFilter = () => {
+    onApplyFilter(`${LOGS_GROUP_BY}: ${legend.label}`);
+    onClose();
+  };
+
+  const handleExcludeStreamToFilter = () => {
+    onApplyFilter(`(NOT ${LOGS_GROUP_BY}: ${legend.label})`);
+    onClose();
+  };
+
+  const handlerCopyLabel = async () => {
+    await copyToClipboard(legend.label, `${legend.label} has been copied`);
+    onClose();
+  };
+
+  const options: LegendLogHitsMenu[] = [
+    {
+      title: `Copy ${LOGS_GROUP_BY} name`,
+      icon: <CopyIcon/>,
+      handler: handlerCopyLabel,
+    },
+    {
+      title: `Add ${LOGS_GROUP_BY} to filter`,
+      icon: <FilterIcon/>,
+      handler: handleAddStreamToFilter,
+    },
+    {
+      title: `Exclude ${LOGS_GROUP_BY} to filter`,
+      icon: <FilterOffIcon/>,
+      handler: handleExcludeStreamToFilter,
+    }
+  ];
+
+  return (
+    <div className="vm-legend-hits-menu-section">
+      {options.map(({ icon, title, handler }) => (
+        <LegendHitsMenuRow
+          key={title}
+          iconStart={icon}
+          title={title}
+          handler={handler}
+        />
+      ))}
+    </div>
+  );
+};
+
+export default LegendHitsMenuBase;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuFields.tsx

@@ -0,0 +1,74 @@
+import React, { FC, useMemo } from "preact/compat";
+import LegendHitsMenuRow from "./LegendHitsMenuRow";
+import { CopyIcon, FilterIcon, FilterOffIcon } from "../../../Main/Icons";
+import { convertToFieldFilter } from "../../../../utils/logs";
+import { LegendLogHitsMenu } from "../../../../api/types";
+import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
+
+interface Props {
+  fields: string[];
+  onApplyFilter: (value: string) => void;
+  onClose: () => void;
+}
+
+const LegendHitsMenuFields: FC<Props> = ({ fields, onApplyFilter, onClose }) => {
+  const copyToClipboard = useCopyToClipboard();
+
+  const handleCopy = (field: string) => async () => {
+    await copyToClipboard(field, `${field} has been copied`);
+    onClose();
+  };
+
+  const handleAddToFilter = (field: string) => () => {
+    onApplyFilter(field);
+    onClose();
+  };
+
+  const handleExcludeToFilter = (field: string) => () => {
+    onApplyFilter(`-${field}`);
+    onClose();
+  };
+
+  const generateFieldMenu = (field: string): LegendLogHitsMenu[] => {
+    return [
+      {
+        title: "Copy",
+        icon: <CopyIcon/>,
+        handler: handleCopy(field),
+      },
+      {
+        title: "Add to filter",
+        icon: <FilterIcon/>,
+        handler: handleAddToFilter(field),
+      },
+      {
+        title: "Exclude to filter",
+        icon: <FilterOffIcon/>,
+        handler: handleExcludeToFilter(field),
+      }
+    ];
+  };
+
+  const fieldsWithMenu: LegendLogHitsMenu[] = useMemo(() => {
+    return fields.map(field => {
+      const title = convertToFieldFilter(field);
+      return {
+        title,
+        submenu: generateFieldMenu(title),
+      };
+    });
+  }, [fields]);
+
+  return (
+    <div className="vm-legend-hits-menu-section">
+      {fieldsWithMenu?.map((field) => (
+        <LegendHitsMenuRow
+          key={field.title}
+          {...field}
+        />
+      ))}
+    </div>
+  );
+};
+
+export default LegendHitsMenuFields;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuRow.tsx

@@ -0,0 +1,116 @@
+import React, { FC, useRef, useState } from "preact/compat";
+import classNames from "classnames";
+import { ReactNode, useEffect } from "react";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import { LegendLogHitsMenu } from "../../../../api/types";
+import { ArrowDropDownIcon } from "../../../Main/Icons";
+import useClickOutside from "../../../../hooks/useClickOutside";
+
+interface Props {
+  title: string | ReactNode;
+  handler?: () => void;
+  iconStart?: ReactNode;
+  iconEnd?: ReactNode;
+  className?: string;
+  submenu?: LegendLogHitsMenu[];
+}
+
+const LegendHitsMenuRow: FC<Props> = ({ title, handler, iconStart, iconEnd, className, submenu }) => {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const titleRef = useRef<HTMLDivElement>(null);
+  const submenuRef = useRef<HTMLDivElement>(null);
+
+  const [isOverflownTitle, setIsOverflownTitle] = useState(false);
+
+  const [openSubmenu, setOpenSubmenu] = useState(false);
+  const [posSubmenuLeft, setPosSubmenuLeft] = useState(false);
+  const hasSubmenu = !!submenu?.length;
+
+  const handleToggleContextMenu = () => {
+    setOpenSubmenu(prev => !prev);
+  };
+
+  const handleCloseContextMenu = () => {
+    setOpenSubmenu(false);
+  };
+
+  const handleClick = () => {
+    handler && handler();
+    hasSubmenu && handleToggleContextMenu();
+  };
+
+
+  useEffect(() => {
+    if (!titleRef.current) return;
+    setIsOverflownTitle(titleRef.current.scrollWidth > titleRef.current.clientWidth);
+  }, [title, titleRef]);
+
+  useEffect(() => {
+    requestAnimationFrame(() => {
+      if (!openSubmenu || !submenuRef.current) {
+        setPosSubmenuLeft(false);
+        return;
+      }
+
+      const { left, width } = submenuRef.current.getBoundingClientRect();
+      setPosSubmenuLeft(left + width > window.innerWidth);
+    });
+  }, [submenuRef, openSubmenu]);
+
+  useClickOutside(containerRef, handleCloseContextMenu);
+
+  const titleContent = (
+    <div
+      ref={titleRef}
+      className="vm-legend-hits-menu-row__title"
+    >
+      {title}
+    </div>
+  );
+
+  return (
+    <div
+      ref={containerRef}
+      className={classNames({
+        "vm-legend-hits-menu-row": true,
+        "vm-legend-hits-menu-row_interactive": !!handler || hasSubmenu,
+        [`${className}`]: className
+      })}
+      onClick={handleClick}
+    >
+      {iconStart && <div className="vm-legend-hits-menu-row__icon">{iconStart}</div>}
+      {isOverflownTitle ? (<Tooltip title={title}>{titleContent}</Tooltip>) : titleContent}
+      {iconEnd && !hasSubmenu && <div className="vm-legend-hits-menu-row__icon">{iconEnd}</div>}
+
+      {hasSubmenu && (
+        <div className="vm-legend-hits-menu-row__icon vm-legend-hits-menu-row__icon_drop">
+          <ArrowDropDownIcon/>
+        </div>
+      )}
+
+      {openSubmenu && submenu && (
+        <div
+          ref={submenuRef}
+          className={classNames({
+            "vm-legend-hits-menu": true,
+            "vm-legend-hits-menu_submenu": true,
+            "vm-legend-hits-menu_submenu_left": posSubmenuLeft
+          })}
+        >
+          <div className="vm-legend-hits-menu-section">
+            {submenu.map(({ icon, title, handler }) => (
+              <LegendHitsMenuRow
+                key={title}
+                iconStart={icon}
+                title={title}
+                handler={handler}
+              />
+            ))}
+          </div>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default LegendHitsMenuRow;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuStats.tsx

@@ -0,0 +1,23 @@
+import React, { FC } from "preact/compat";
+import { LegendLogHits } from "../../../../api/types";
+
+interface Props {
+  legend: LegendLogHits;
+}
+
+const LegendHitsMenuStats: FC<Props> = ({ legend }) => {
+  const totalFormatted = legend.total.toLocaleString("en-US");
+  const percentage = Math.round((legend.total / legend.totalHits) * 100);
+
+  return (
+    <div className="vm-legend-hits-menu-section">
+      <div className="vm-legend-hits-menu-row">
+        <div className="vm-legend-hits-menu-row__title">
+          Total: {totalFormatted} ({percentage}%)
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default LegendHitsMenuStats;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/style.scss

@@ -0,0 +1,178 @@
+@use "src/styles/variables" as *;
+
+.vm-legend-hits-menu {
+  min-width: 160px;
+  z-index: 1;
+
+  &_submenu {
+    position: absolute;
+    top: calc(-1 * $padding-small);
+    background-color: $color-background-block;
+    left: calc(100% + ($padding-small / 2));
+    box-shadow: $box-shadow-popper;
+    border-radius: $border-radius-small;
+    animation: vm-submenu-show 150ms cubic-bezier(0.280, 0.840, 0.2, 1);
+    transform-origin: top left;
+
+    &_left {
+      left: auto;
+      right: calc(100% + ($padding-small / 2));
+      transform-origin: top right;
+    }
+  }
+
+  &-section {
+    border-bottom: $border-divider;
+
+    &:last-child {
+      border-bottom: none;
+    }
+  }
+
+  &-row {
+    position: relative;
+    display: flex;
+    gap: $padding-small;
+    align-items: center;
+    justify-content: flex-start;
+    padding: 0 $padding-global;
+    transition: background-color 0.3s;
+    color: $color-text;
+
+    &_interactive {
+      cursor: pointer;
+
+      &:hover {
+        background-color: rgba(0, 0, 0, 0.05);
+      }
+    }
+
+    &_info {
+      font-size: $font-size-small;
+      font-weight: 500;
+      padding-block: $padding-small;
+    }
+
+    &_info &__icon {
+      color: $color-info;
+    }
+
+    &__icon {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      width: 14px;
+      height: 14px;
+
+      &_drop {
+        transform: rotate(-90deg);
+      }
+    }
+
+    &__title {
+      flex-grow: 1;
+      padding: $padding-global 0;
+      position: relative;
+      max-width: 400px;
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+  }
+
+  &-other-list {
+    width: 80vw;
+    height: 80vh;
+    overflow: auto;
+
+    &__search {
+      position: sticky;
+      top: 0;
+      padding: $padding-small 0;
+      background-color: $color-background-block;
+      border-bottom: $border-divider;
+      z-index: 2;
+    }
+
+    &-row {
+      border-bottom: $border-divider;
+
+      &_header {
+        border-bottom: none;
+        position: sticky;
+        top: 65px;
+        background-color: $color-background-block;
+        z-index: 1;
+        width: 100%;
+
+        &:after {
+          content: '';
+          position: absolute;
+          bottom: 0;
+          left: 0;
+          width: 100%;
+          height: 1px;
+          border-bottom: $border-divider;
+        }
+      }
+    }
+
+    &-cell {
+      padding: calc($padding-small / 2) 0;
+      text-align: left;
+
+      &_header {
+        padding: $padding-small;
+        font-weight: 500;
+      }
+
+      &_number {
+        padding: $padding-small;
+        text-align: right;
+        font-variant-numeric: tabular-nums;
+      }
+
+      &_fields {
+        width: 100%;
+      }
+    }
+
+    &-fields {
+      display: flex;
+      flex-wrap: wrap;
+      align-items: center;
+      justify-content: flex-start;
+
+      &__field {
+        padding: calc($padding-small / 2) $padding-small;
+        border-radius: $border-radius-small;
+        transition: background-color 0.3s;
+
+        &:hover {
+          background-color: $color-hover-black;
+        }
+
+        &:not(:last-child) {
+          &:after {
+            content: ',';
+          }
+        }
+      }
+    }
+
+    &-actions {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    }
+  }
+}
+
+@keyframes vm-submenu-show {
+  0% {
+    opacity: 0;
+  }
+  100% {
+    opacity: 1;
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/hooks/useBarHitsOptions.ts

@@ -36,6 +36,14 @@ interface UseGetBarHitsOptionsArgs {
   graphOptions: GraphOptions;
 }
 
+export const OTHER_HITS_LABEL = "other";
+
+export const getLabelFromLogHit = (logHit: LogHits) => {
+  if (logHit?._isOther) return OTHER_HITS_LABEL;
+  const fields = Object.values(logHit?.fields || {});
+  return fields.map((value) => value || "\"\"").join(", ");
+};
+
 const useBarHitsOptions = ({
   data,
   logHits,
@@ -59,12 +67,12 @@ const useBarHitsOptions = ({
     let colorN = 0;
     return data.map((_d, i) => {
       if (i === 0) return {}; // 0 index is xAxis(timestamps)
-      const fields = Object.values(logHits?.[i - 1]?.fields || {});
-      const label = fields.map((value) => value || "\"\"").join(", ");
-      const color = getCssVariable(label ? seriesColors[colorN] : "color-log-hits-bar-0");
-      if (label) colorN++;
+      const target = logHits?.[i - 1];
+      const label = getLabelFromLogHit(target);
+      const color = getCssVariable(target?._isOther ? "color-log-hits-bar-0" : seriesColors[colorN]);
+      if (!target?._isOther) colorN++;
       return {
-        label: label || "other",
+        label,
         width: strokeWidth[graphOptions.graphStyle],
         spanGaps: true,
         stroke: color,

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -32,6 +32,11 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     max-width: calc(100vw/3);
   }
 
+  &_hits &-data {
+    display: grid;
+    grid-template-columns: $font-size 1fr;
+  }
+
   &_sticky {
     pointer-events: auto;
     z-index: 99;
@@ -90,6 +95,8 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     }
 
     &__marker {
+      min-width: $font-size;
+      max-width: $font-size;
       width: $font-size;
       height: $font-size;
       border: 1px solid rgba($color-white, 0.5);

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx

@@ -124,7 +124,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
   };
 
   useEffect(() => {
-    setOpenAutocomplete(!!AutocompleteEl);
+    setOpenAutocomplete(!!AutocompleteEl && autocompleteQuick);
   }, [autocompleteQuick]);
 
   useEffect(() => {

app/vmui/packages/vmui/src/components/LogsConfigurators/GroupLogsConfigurators/GroupLogsConfigurators.tsx

@@ -0,0 +1,246 @@
+import React, { FC, useMemo, useState } from "preact/compat";
+import useBoolean from "../../../hooks/useBoolean";
+import { RestartIcon, SettingsIcon } from "../../Main/Icons";
+import Button from "../../Main/Button/Button";
+import Modal from "../../Main/Modal/Modal";
+import Tooltip from "../../Main/Tooltip/Tooltip";
+import { Logs } from "../../../api/types";
+import Select from "../../Main/Select/Select";
+import { useSearchParams } from "react-router-dom";
+import "./style.scss";
+import Switch from "../../Main/Switch/Switch";
+import TextField from "../../Main/TextField/TextField";
+import dayjs from "dayjs";
+import Hyperlink from "../../Main/Hyperlink/Hyperlink";
+import {
+  LOGS_DISPLAY_FIELDS,
+  LOGS_GROUP_BY,
+  LOGS_DATE_FORMAT,
+  LOGS_URL_PARAMS,
+  WITHOUT_GROUPING
+} from "../../../constants/logs";
+
+const {
+  GROUP_BY,
+  NO_WRAP_LINES,
+  COMPACT_GROUP_HEADER,
+  DISPLAY_FIELDS,
+  DATE_FORMAT
+} = LOGS_URL_PARAMS;
+
+const title = "Group view settings";
+
+interface Props {
+  logs: Logs[];
+}
+
+const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
+  const [searchParams, setSearchParams] = useSearchParams();
+
+  const groupBy = searchParams.get(GROUP_BY) || LOGS_GROUP_BY;
+  const noWrapLines = searchParams.get(NO_WRAP_LINES) === "true";
+  const compactGroupHeader = searchParams.get(COMPACT_GROUP_HEADER) === "true";
+  const displayFieldsString = searchParams.get(DISPLAY_FIELDS) || "";
+  const displayFields = displayFieldsString ? displayFieldsString.split(",") : [];
+
+  const [dateFormat, setDateFormat] = useState(searchParams.get(DATE_FORMAT) || LOGS_DATE_FORMAT);
+  const [errorFormat, setErrorFormat] = useState("");
+
+  const isGroupChanged = groupBy !== LOGS_GROUP_BY;
+  const isDisplayFieldsChanged = displayFields.length > 0;
+  const isTimeChanged = searchParams.get(DATE_FORMAT) !== LOGS_DATE_FORMAT;
+  const hasChanges = [
+    isGroupChanged,
+    isDisplayFieldsChanged,
+    noWrapLines,
+    compactGroupHeader,
+    isTimeChanged
+  ].some(Boolean);
+
+  const logsKeys = useMemo(() => {
+    const excludeKeys = ["_msg", "_time"];
+    const uniqKeys = Array.from(new Set(logs.map(l => Object.keys(l)).flat()));
+    return uniqKeys.filter(k => !excludeKeys.includes(k));
+  }, [logs]);
+
+  const {
+    value: openModal,
+    toggle: toggleOpen,
+    setFalse: handleClose,
+  } = useBoolean(false);
+
+  const handleSelectGroupBy = (key: string) => {
+    searchParams.set(GROUP_BY, key);
+    setSearchParams(searchParams);
+  };
+
+  const handleSelectDisplayField = (value: string) => {
+    const prev = displayFields;
+    const newDisplayFields = prev.includes(value) ? prev.filter(v => v !== value) : [...prev, value];
+    searchParams.set(DISPLAY_FIELDS, newDisplayFields.join(","));
+    setSearchParams(searchParams);
+  };
+
+  const handleResetDisplayFields = () => {
+    searchParams.delete(DISPLAY_FIELDS);
+    setSearchParams(searchParams);
+  };
+
+  const toggleWrapLines = () => {
+    searchParams.set(NO_WRAP_LINES, String(!noWrapLines));
+    setSearchParams(searchParams);
+  };
+
+  const toggleCompactGroupHeader = () => {
+    searchParams.set(COMPACT_GROUP_HEADER, String(!compactGroupHeader));
+    setSearchParams(searchParams);
+  };
+
+  const handleChangeDateFormat = (format: string) => {
+    const date = new Date();
+    if (!dayjs(date, format, true).isValid()) {
+      setErrorFormat("Invalid date format");
+    }
+    setDateFormat(format);
+  };
+
+  const handleSaveAndClose = () => {
+    searchParams.set(DATE_FORMAT, dateFormat);
+    setSearchParams(searchParams);
+    handleClose();
+  };
+
+  const tooltipContent = () => {
+    if (!hasChanges) return title;
+    return (
+      <div className="vm-group-logs-configurator__tooltip">
+        <p>{title}</p>
+        <hr/>
+        <ul>
+          {isGroupChanged && <li>Group by <code>{`"${groupBy}"`}</code></li>}
+          {isDisplayFieldsChanged && <li>Display fields: {displayFields.length || 1}</li>}
+          {noWrapLines && <li>Single-line text is enabled</li>}
+          {compactGroupHeader && <li>Compact group header is enabled</li>}
+          {isTimeChanged && <li>Date format: <code>{dateFormat}</code></li>}
+        </ul>
+      </div>
+    );
+  };
+
+  return (
+    <>
+      <div className="vm-group-logs-configurator-button">
+        <Tooltip title={tooltipContent()}>
+          <Button
+            variant="text"
+            startIcon={<SettingsIcon/>}
+            onClick={toggleOpen}
+            ariaLabel={title}
+          />
+        </Tooltip>
+        {hasChanges && <span className="vm-group-logs-configurator-button__marker"/>}
+      </div>
+      {openModal && (
+        <Modal
+          title={title}
+          onClose={handleSaveAndClose}
+        >
+          <div className="vm-group-logs-configurator">
+            <div className="vm-group-logs-configurator-item">
+              <Select
+                value={groupBy}
+                list={[WITHOUT_GROUPING, ...logsKeys]}
+                label="Group by field"
+                placeholder="Group by field"
+                onChange={handleSelectGroupBy}
+                searchable
+              />
+              <Tooltip title={"Reset grouping"}>
+                <Button
+                  variant="text"
+                  color="primary"
+                  startIcon={<RestartIcon/>}
+                  onClick={() => handleSelectGroupBy(LOGS_GROUP_BY)}
+                />
+              </Tooltip>
+              <span className="vm-group-logs-configurator-item__info">
+                Select a field to group logs by (default: <code>{LOGS_GROUP_BY}</code>).
+              </span>
+            </div>
+
+            <div className="vm-group-logs-configurator-item">
+              <Select
+                value={displayFields}
+                list={logsKeys}
+                label="Display fields"
+                placeholder="Display fields"
+                onChange={handleSelectDisplayField}
+                searchable
+              />
+              <Tooltip title={"Clear fields"}>
+                <Button
+                  variant="text"
+                  color="primary"
+                  startIcon={<RestartIcon/>}
+                  onClick={handleResetDisplayFields}
+                />
+              </Tooltip>
+              <span className="vm-group-logs-configurator-item__info">
+                Select fields to display instead of the message (default: <code>{LOGS_DISPLAY_FIELDS}</code>).
+              </span>
+            </div>
+
+            <div className="vm-group-logs-configurator-item">
+              <TextField
+                autofocus
+                label="Date format"
+                value={dateFormat}
+                onChange={handleChangeDateFormat}
+                error={errorFormat}
+              />
+              <Tooltip title={"Reset format"}>
+                <Button
+                  variant="text"
+                  color="primary"
+                  startIcon={<RestartIcon/>}
+                  onClick={() => setDateFormat(LOGS_DATE_FORMAT)}
+                />
+              </Tooltip>
+              <span className="vm-group-logs-configurator-item__info vm-group-logs-configurator-item__info_input">
+                Set the date format (e.g., <code>YYYY-MM-DD HH:mm:ss</code>).
+                Learn more in <Hyperlink
+                  href="https://day.js.org/docs/en/display/format"
+                >this documentation</Hyperlink>. <br/>
+                Your current date format: <code>{dayjs().format(dateFormat || LOGS_DATE_FORMAT)}</code>
+              </span>
+            </div>
+
+            <div className="vm-group-logs-configurator-item">
+              <Switch
+                value={noWrapLines}
+                onChange={toggleWrapLines}
+                label="Single-line message"
+              />
+              <span className="vm-group-logs-configurator-item__info">
+                Displays message in a single line and truncates it with an ellipsis if it exceeds the available space
+              </span>
+            </div>
+
+            <div className="vm-group-logs-configurator-item">
+              <Switch
+                value={compactGroupHeader}
+                onChange={toggleCompactGroupHeader}
+                label="Compact group header"
+              />
+              <span className="vm-group-logs-configurator-item__info">
+                Shows group headers in one line with a &quot;+N more&quot; badge for extra fields.
+              </span>
+            </div>
+          </div>
+        </Modal>
+      )}
+    </>
+  );
+};
+
+export default GroupLogsConfigurators;

app/vmui/packages/vmui/src/components/LogsConfigurators/GroupLogsConfigurators/style.scss

@@ -0,0 +1,48 @@
+@use "src/styles/variables" as *;
+
+.vm-group-logs-configurator {
+  display: grid;
+  gap: calc($padding-large * 2);
+  padding: $padding-global 0;
+  width: 600px;
+
+  &-item {
+    display: grid;
+    grid-template-columns: 1fr 31px;
+    align-items: center;
+    justify-content: stretch;
+    gap: 0 $padding-small;
+
+    &__info {
+      margin-top: $padding-small;
+      grid-column: 1/span 2;
+      font-size: $font-size-small;
+      color: $color-text-secondary;
+      line-height: 130%;
+
+      &_input {
+        margin-top: 0;
+      }
+    }
+  }
+
+  &-button {
+    position: relative;
+
+    &__marker {
+      position: absolute;
+      top: 6px;
+      left: 6px;
+      width: 5px;
+      height: 5px;
+      border-radius: 50%;
+      background-color: $color-secondary;
+    }
+  }
+
+  &__tooltip {
+    ul {
+      list-style-position: inside;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Main/Accordion/Accordion.tsx

@@ -30,6 +30,10 @@ const Accordion: FC<AccordionProps> = ({
     onChange && onChange(isOpen);
   }, [isOpen]);
 
+  useEffect(() => {
+    setIsOpen(defaultExpanded);
+  }, [defaultExpanded]);
+
   return (
     <>
       <header

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -581,3 +581,45 @@ export const CommentIcon = () => (
     ></path>
   </svg>
 );
+
+export const FilterIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M4.25 5.61C6.27 8.2 10 13 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-6s3.72-4.8 5.74-7.39c.51-.66.04-1.61-.79-1.61H5.04c-.83 0-1.3.95-.79 1.61"
+    ></path>
+  </svg>
+);
+
+export const FilterOffIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M19.79 5.61C20.3 4.95 19.83 4 19 4H6.83l7.97 7.97zM2.81 2.81 1.39 4.22 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-2.17l5.78 5.78 1.41-1.41z"
+    ></path>
+  </svg>
+);
+
+export const OpenNewIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M19 19H5V5h7V3H5c-1.11 0-2 .9-2 2v14c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2v-7h-2zM14 3v2h3.59l-9.83 9.83 1.41 1.41L19 6.41V10h2V3z"
+    ></path>
+  </svg>
+);
+
+export const ModalIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path d="M19 4H5c-1.11 0-2 .9-2 2v12c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2V6c0-1.1-.89-2-2-2m0 14H5V8h14z"></path>
+  </svg>
+);

app/vmui/packages/vmui/src/components/Main/Modal/Modal.tsx

@@ -67,11 +67,11 @@ const Modal: FC<ModalProps> = ({
       })}
       onMouseDown={onClose}
     >
-      <div className="vm-modal-content">
-        <div
-          className="vm-modal-content-header"
-          onMouseDown={handleMouseDown}
-        >
+      <div
+        className="vm-modal-content"
+        onMouseDown={handleMouseDown}
+      >
+        <div className="vm-modal-content-header">
           {title && (
             <div className="vm-modal-content-header__title">
               {title}
@@ -91,7 +91,6 @@ const Modal: FC<ModalProps> = ({
         {/* tabIndex to fix Ctrl-A */}
         <div
           className="vm-modal-content-body"
-          onMouseDown={handleMouseDown}
           tabIndex={0}
         >
           {children}

app/vmui/packages/vmui/src/components/Main/Popper/Popper.tsx

@@ -15,9 +15,10 @@ interface PopperProps {
   open: boolean
   onClose: () => void
   buttonRef: React.RefObject<HTMLElement>
-  placement?: "bottom-right" | "bottom-left" | "top-left" | "top-right"
+  placement?: "bottom-right" | "bottom-left" | "top-left" | "top-right" | "fixed"
+  placementPosition?: { top: number, left: number } | null
   animation?: string
-  offset?: {top: number, left: number}
+  offset?: { top: number, left: number }
   clickOutside?: boolean,
   fullWidth?: boolean
   title?: string
@@ -29,6 +30,7 @@ const Popper: FC<PopperProps> = ({
   children,
   buttonRef,
   placement = "bottom-left",
+  placementPosition,
   open = false,
   onClose,
   offset = { top: 6, left: 0 },
@@ -92,13 +94,18 @@ const Popper: FC<PopperProps> = ({
     if (needAlignRight) position.left = buttonPos.right - popperSize.width;
     if (needAlignTop) position.top = buttonPos.top - popperSize.height - offsetTop;
 
-    const { innerWidth, innerHeight } = window;
-    const margin = 20;
+    if (placement === "fixed" && placementPosition) {
+      position.top = Math.max(placementPosition.top + offset.top, 0);
+      position.left = Math.max(placementPosition.left + offset.left, 0);
+      return position;
+    }
 
-    const isOverflowBottom = (position.top + popperSize.height + margin) > innerHeight;
-    const isOverflowTop = (position.top - margin) < 0;
-    const isOverflowRight = (position.left + popperSize.width + margin) > innerWidth;
-    const isOverflowLeft = (position.left - margin) < 0;
+    const { innerWidth, innerHeight } = window;
+
+    const isOverflowBottom = (position.top + popperSize.height) > innerHeight;
+    const isOverflowTop = (position.top) < 0;
+    const isOverflowRight = (position.left + popperSize.width) > innerWidth;
+    const isOverflowLeft = (position.left) < 0;
 
     if (isOverflowBottom) position.top = buttonPos.top - popperSize.height - offsetTop;
     if (isOverflowTop) position.top = buttonPos.height + buttonPos.top + offsetTop;
@@ -106,11 +113,11 @@ const Popper: FC<PopperProps> = ({
     if (isOverflowLeft) position.left = buttonPos.left + offsetLeft;
 
     if (fullWidth) position.width = `${buttonPos.width}px`;
-    if (position.top < 0) position.top = 20;
-    if (position.left < 0) position.left = 20;
+    if (position.top < 0) position.top = 0;
+    if (position.left < 0) position.left = 0;
 
     return position;
-  },[buttonRef, placement, isOpen, children, fullWidth]);
+  }, [buttonRef, placement, isOpen, children, fullWidth]);
 
   const handleClickClose = (e: ReactMouseEvent<HTMLButtonElement, MouseEvent>) => {
     e.stopPropagation();
@@ -131,10 +138,10 @@ const Popper: FC<PopperProps> = ({
     if (!popperRef.current || !isOpen || (isMobile && !disabledFullScreen)) return;
     const { right, width } = popperRef.current.getBoundingClientRect();
     if (right > window.innerWidth) {
-      const left = window.innerWidth - 20 - width;
-      popperRef.current.style.left = left < window.innerWidth ? "0" : `${left}px`;
+      const left = window.innerWidth - width;
+      popperRef.current.style.left = `${left}px`;
     }
-  }, [isOpen, popperRef]);
+  }, [isOpen, popperRef, placementPosition]);
 
   const handlePopstate = useCallback(() => {
     if (isOpen && isMobile && !disabledFullScreen) {

app/vmui/packages/vmui/src/components/Main/Popper/style.scss

@@ -11,7 +11,7 @@
   border-radius: $border-radius-small;
 
   &_open {
-    z-index: 101;
+    z-index: 100;
     opacity: 1;
     transform-origin: top center;
     animation: vm-slider 150ms cubic-bezier(0.280, 0.840, 0.420, 1.1);

app/vmui/packages/vmui/src/components/Main/Select/style.scss

@@ -33,9 +33,9 @@
         align-items: center;
         justify-content: center;
         background-color: $color-hover-black;
-        padding: 2px 2px 2px 6px;
+        padding: 2px 2px 2px $padding-small;
         border-radius: $border-radius-small;
-        font-size: $font-size;
+        font-size: $font-size-small;
         line-height: $font-size;
         max-width: 100%;
 

app/vmui/packages/vmui/src/components/Table/TableSettings/TableSettings.tsx

@@ -11,7 +11,7 @@ import useBoolean from "../../../hooks/useBoolean";
 import TextField from "../../Main/TextField/TextField";
 import { KeyboardEvent, useState } from "react";
 import Modal from "../../Main/Modal/Modal";
-import { getFromStorage, removeFromStorage, saveToStorage } from "../../../utils/storage";
+import { useSearchParams } from "react-router-dom";
 
 const title = "Table settings";
 
@@ -30,6 +30,8 @@ const TableSettings: FC<TableSettingsProps> = ({
   onChangeColumns,
   toggleTableCompact
 }) => {
+  const [searchParams, setSearchParams] = useSearchParams();
+
   const buttonRef = useRef<HTMLDivElement>(null);
 
   const {
@@ -38,11 +40,6 @@ const TableSettings: FC<TableSettingsProps> = ({
     setFalse: handleClose,
   } = useBoolean(false);
 
-  const {
-    value: saveColumns,
-    toggle: toggleSaveColumns,
-  } = useBoolean(Boolean(getFromStorage("TABLE_COLUMNS")));
-
   const [searchColumn, setSearchColumn] = useState("");
   const [indexFocusItem, setIndexFocusItem] = useState(-1);
 
@@ -60,15 +57,34 @@ const TableSettings: FC<TableSettingsProps> = ({
     return filteredColumns.every(col => selectedColumns.includes(col));
   }, [selectedColumns, filteredColumns]);
 
+  const handleChangeDisplayColumns = (displayColumns: string[]) => {
+    onChangeColumns(displayColumns);
+
+    const updatedParams = new URLSearchParams(searchParams.toString());
+    const isAllCheck = displayColumns.length === columns.length;
+
+    if (isAllCheck) {
+      updatedParams.delete("columns");
+    } else {
+      updatedParams.set("columns", displayColumns.map(encodeURIComponent).join(","));
+    }
+
+    setSearchParams(updatedParams);
+  };
+
   const handleChange = (key: string) => {
-    onChangeColumns(selectedColumns.includes(key) ? selectedColumns.filter(col => col !== key) : [...selectedColumns, key]);
+    const displayColumns = selectedColumns.includes(key)
+      ? selectedColumns.filter(col => col !== key)
+      : [...selectedColumns, key];
+
+    handleChangeDisplayColumns(displayColumns);
   };
 
   const toggleAllColumns = () => {
     if (isAllChecked) {
-      onChangeColumns(selectedColumns.filter(col => !filteredColumns.includes(col)));
+      handleChangeDisplayColumns(selectedColumns.filter(col => !filteredColumns.includes(col)));
     } else {
-      onChangeColumns(filteredColumns);
+      handleChangeDisplayColumns(filteredColumns);
     }
   };
 
@@ -95,22 +111,16 @@ const TableSettings: FC<TableSettingsProps> = ({
   };
 
   useEffect(() => {
-    if (arrayEquals(columns, selectedColumns) || saveColumns) return;
+    if (arrayEquals(columns, selectedColumns) || searchParams.has("columns")) return;
     onChangeColumns(columns);
   }, [columns]);
 
   useEffect(() => {
-    if (!saveColumns) {
-      removeFromStorage(["TABLE_COLUMNS"]);
-    } else if (selectedColumns.length) {
-      saveToStorage("TABLE_COLUMNS", selectedColumns.join(","));
-    }
-  }, [saveColumns, selectedColumns]);
-
-  useEffect(() => {
-    const saveColumns = getFromStorage("TABLE_COLUMNS") as string;
-    if (!saveColumns) return;
-    onChangeColumns(saveColumns.split(","));
+    const hasColumns = searchParams.has("columns");
+    if (!hasColumns) return;
+    const columnsParam = searchParams.get("columns") || "";
+    const columnsArray = columnsParam.split(",").map(decodeURIComponent).filter(Boolean);
+    onChangeColumns(columnsArray);
   }, []);
 
   return (
@@ -183,19 +193,6 @@ const TableSettings: FC<TableSettingsProps> = ({
                   </div>
                 ))}
               </div>
-              <div className="vm-table-settings-modal-preserve">
-                <Checkbox
-                  checked={saveColumns}
-                  onChange={toggleSaveColumns}
-                  label={"Preserve column settings"}
-                  disabled={tableCompact}
-                  color={"primary"}
-                />
-                <p className="vm-table-settings-modal-preserve__info">
-                  This label indicates that when the checkbox is activated,
-                  the current column configurations will not be reset.
-                </p>
-              </div>
             </div>
           </div>
           <div className="vm-table-settings-modal-section">

app/vmui/packages/vmui/src/components/Table/TableSettings/style.scss

@@ -3,6 +3,7 @@
 .vm-table-settings {
   &-modal {
     .vm-modal-content-body {
+      min-width: clamp(300px, 600px, 90vw);
       padding: 0;
     }
 
@@ -83,16 +84,5 @@
         }
       }
     }
-
-    &-preserve {
-      padding: $padding-global;
-
-      &__info {
-        padding-top: $padding-small;
-        font-size: $font-size-small;
-        color: $color-text-secondary;
-        line-height: 130%;
-      }
-    }
   }
 }

app/vmui/packages/vmui/src/constants/logs.ts

@@ -1,2 +1,22 @@
+import { DATE_TIME_FORMAT } from "./date";
+
 export const LOGS_ENTRIES_LIMIT = 50;
 export const LOGS_BARS_VIEW = 100;
+export const LOGS_LIMIT_HITS = 5;
+
+// "Ungrouped" is a string that is used as a value for the "groupBy" parameter.
+export const WITHOUT_GROUPING = "Ungrouped";
+
+// Default values for the logs configurators.
+export const LOGS_GROUP_BY = "_stream";
+export const LOGS_DISPLAY_FIELDS = "_msg";
+export const LOGS_DATE_FORMAT = `${DATE_TIME_FORMAT}.SSS`;
+
+// URL parameters for the logs page.
+export const LOGS_URL_PARAMS = {
+  GROUP_BY: "groupBy",
+  DISPLAY_FIELDS: "displayFields",
+  NO_WRAP_LINES: "noWrapLines",
+  COMPACT_GROUP_HEADER: "compactGroupHeader",
+  DATE_FORMAT: "dateFormat",
+};

app/vmui/packages/vmui/src/hooks/useClickOutside.ts

@@ -20,7 +20,7 @@ const useClickOutside = <T extends HTMLElement = HTMLElement>(
     handler(event); // Call the handler only if the click is outside of the element passed.
   }, [ref, handler]);
 
-  useEventListener("mousedown", listener);
+  useEventListener("mouseup", listener);
   useEventListener("touchstart", listener);
 };
 

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -69,7 +69,7 @@ const ExploreLogs: FC = () => {
   };
 
   const handleApplyFilter = (val: string) => {
-    setQuery(prev => `_stream: ${val === "other" ? "{}" : val} AND (${prev})`);
+    setQuery(prev => `${val} AND (${prev})`);
     setIsUpdatingQuery(true);
   };
 

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx

@@ -1,24 +1,19 @@
-import React, { FC, useCallback, useEffect, useMemo, useRef } from "preact/compat";
-import { MouseEvent, useState } from "react";
+import React, { FC, useCallback, useEffect, useMemo } from "preact/compat";
+import { useState } from "react";
 import "./style.scss";
 import { Logs } from "../../../api/types";
 import Accordion from "../../../components/Main/Accordion/Accordion";
 import { groupByMultipleKeys } from "../../../utils/array";
 import Tooltip from "../../../components/Main/Tooltip/Tooltip";
-import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 import GroupLogsItem from "./GroupLogsItem";
-import { useAppState } from "../../../state/common/StateContext";
-import classNames from "classnames";
 import Button from "../../../components/Main/Button/Button";
-import { CollapseIcon, ExpandIcon, StorageIcon } from "../../../components/Main/Icons";
-import Popper from "../../../components/Main/Popper/Popper";
-import TextField from "../../../components/Main/TextField/TextField";
-import useBoolean from "../../../hooks/useBoolean";
-import useStateSearchParams from "../../../hooks/useStateSearchParams";
+import { CollapseIcon, ExpandIcon } from "../../../components/Main/Icons";
 import { useSearchParams } from "react-router-dom";
 import { getStreamPairs } from "../../../utils/logs";
-
-const WITHOUT_GROUPING = "No Grouping";
+import GroupLogsConfigurators
+  from "../../../components/LogsConfigurators/GroupLogsConfigurators/GroupLogsConfigurators";
+import GroupLogsHeader from "./GroupLogsHeader";
+import { LOGS_DISPLAY_FIELDS, LOGS_GROUP_BY, LOGS_URL_PARAMS, WITHOUT_GROUPING } from "../../../constants/logs";
 
 interface Props {
   logs: Logs[];
@@ -26,73 +21,31 @@ interface Props {
 }
 
 const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
-  const { isDarkTheme } = useAppState();
-  const copyToClipboard = useCopyToClipboard();
-  const [searchParams, setSearchParams] = useSearchParams();
+  const [searchParams] = useSearchParams();
 
   const [expandGroups, setExpandGroups] = useState<boolean[]>([]);
-  const [groupBy, setGroupBy] = useStateSearchParams("_stream", "groupBy");
-  const [copied, setCopied] = useState<string | null>(null);
-  const [searchKey, setSearchKey] = useState("");
-  const optionsButtonRef = useRef<HTMLDivElement>(null);
 
-  const {
-    value: openOptions,
-    toggle: toggleOpenOptions,
-    setFalse: handleCloseOptions,
-  } = useBoolean(false);
+  const groupBy = searchParams.get(LOGS_URL_PARAMS.GROUP_BY) || LOGS_GROUP_BY;
+  const displayFieldsString = searchParams.get(LOGS_URL_PARAMS.DISPLAY_FIELDS) || LOGS_DISPLAY_FIELDS;
+  const displayFields = displayFieldsString.split(",");
 
   const expandAll = useMemo(() => expandGroups.every(Boolean), [expandGroups]);
 
-  const logsKeys = useMemo(() => {
-    const excludeKeys = ["_msg", "_time"];
-    const uniqKeys = Array.from(new Set(logs.map(l => Object.keys(l)).flat()));
-    return [WITHOUT_GROUPING, ...uniqKeys.filter(k => !excludeKeys.includes(k))];
-  }, [logs]);
-
-  const filteredLogsKeys = useMemo(() => {
-    if (!searchKey) return logsKeys;
-    try {
-      const regexp = new RegExp(searchKey, "i");
-      return logsKeys.filter(item => regexp.test(item))
-        .sort((a, b) => (a.match(regexp)?.index || 0) - (b.match(regexp)?.index || 0));
-    } catch (e) {
-      return [];
-    }
-  }, [logsKeys, searchKey]);
-
   const groupData = useMemo(() => {
     return groupByMultipleKeys(logs, [groupBy]).map((item) => {
       const streamValue = item.values[0]?.[groupBy] || "";
       const pairs = getStreamPairs(streamValue);
       // values sorting by time
-      const values = item.values.sort((a,b) => new Date(b._time).getTime() - new Date(a._time).getTime());
+      const values = item.values.sort((a, b) => new Date(b._time).getTime() - new Date(a._time).getTime());
       return {
         keys: item.keys,
         keysString: item.keys.join(""),
         values,
         pairs,
       };
-    }).sort((a, b) => a.keysString.localeCompare(b.keysString)); // groups sorting
+    }).sort((a, b) => b.values.length - a.values.length); // groups sorting
   }, [logs, groupBy]);
 
-  const handleClickByPair = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
-    e.stopPropagation();
-    const isKeyValue = /(.+)?=(".+")/.test(value);
-    const copyValue = isKeyValue ? `${value.replace(/=/, ": ")}` : `${groupBy}: "${value}"`;
-    const isCopied = await copyToClipboard(copyValue);
-    if (isCopied) {
-      setCopied(value);
-    }
-  };
-
-  const handleSelectGroupBy = (key: string) => () => {
-    setGroupBy(key);
-    searchParams.set("groupBy", key);
-    setSearchParams(searchParams);
-    handleCloseOptions();
-  };
-
   const handleToggleExpandAll = useCallback(() => {
     setExpandGroups(new Array(groupData.length).fill(!expandAll));
   }, [expandAll, groupData.length]);
@@ -105,11 +58,6 @@ const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
     });
   }, []);
 
-  useEffect(() => {
-    if (copied === null) return;
-    const timeout = setTimeout(() => setCopied(null), 2000);
-    return () => clearTimeout(timeout);
-  }, [copied]);
 
   useEffect(() => {
     setExpandGroups(new Array(groupData.length).fill(true));
@@ -124,38 +72,16 @@ const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
             key={item.keysString}
           >
             <Accordion
-              key={String(expandGroups[i])}
               defaultExpanded={expandGroups[i]}
               onChange={handleChangeExpand(i)}
-              title={groupBy !== WITHOUT_GROUPING && (
-                <div className="vm-group-logs-section-keys">
-                  <span className="vm-group-logs-section-keys__title">Group by <code>{groupBy}</code>:</span>
-                  {item.pairs.map((pair) => (
-                    <Tooltip
-                      title={copied === pair ? "Copied" : "Copy to clipboard"}
-                      key={`${item.keysString}_${pair}`}
-                      placement={"top-center"}
-                    >
-                      <div
-                        className={classNames({
-                          "vm-group-logs-section-keys__pair": true,
-                          "vm-group-logs-section-keys__pair_dark": isDarkTheme
-                        })}
-                        onClick={handleClickByPair(pair)}
-                      >
-                        {pair}
-                      </div>
-                    </Tooltip>
-                  ))}
-                  <span className="vm-group-logs-section-keys__count">{item.values.length} entries</span>
-                </div>
-              )}
+              title={groupBy !== WITHOUT_GROUPING && <GroupLogsHeader group={item}/>}
             >
               <div className="vm-group-logs-section-rows">
                 {item.values.map((value) => (
                   <GroupLogsItem
                     key={`${value._msg}${value._time}`}
                     log={value}
+                    displayFields={displayFields}
                   />
                 ))}
               </div>
@@ -175,47 +101,7 @@ const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
               ariaLabel={expandAll ? "Collapse All" : "Expand All"}
             />
           </Tooltip>
-          <Tooltip title={"Group by"}>
-            <div ref={optionsButtonRef}>
-              <Button
-                variant="text"
-                startIcon={<StorageIcon/>}
-                onClick={toggleOpenOptions}
-                ariaLabel={"Group by"}
-              />
-            </div>
-          </Tooltip>
-          {
-            <Popper
-              open={openOptions}
-              placement="bottom-right"
-              onClose={handleCloseOptions}
-              buttonRef={optionsButtonRef}
-            >
-              <div className="vm-list vm-group-logs-header-keys">
-                <div className="vm-group-logs-header-keys__search">
-                  <TextField
-                    label="Search key"
-                    value={searchKey}
-                    onChange={setSearchKey}
-                    type="search"
-                  />
-                </div>
-                {filteredLogsKeys.map(id => (
-                  <div
-                    className={classNames({
-                      "vm-list-item": true,
-                      "vm-list-item_active": id === groupBy
-                    })}
-                    key={id}
-                    onClick={handleSelectGroupBy(id)}
-                  >
-                    {id}
-                  </div>
-                ))}
-              </div>
-            </Popper>
-          }
+          <GroupLogsConfigurators logs={logs}/>
         </div>
       ), settingsRef.current)}
     </>

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsFieldRow.tsx

@@ -1,8 +1,10 @@
 import React, { FC, memo, useCallback, useEffect, useState } from "preact/compat";
 import Tooltip from "../../../components/Main/Tooltip/Tooltip";
 import Button from "../../../components/Main/Button/Button";
-import { CopyIcon } from "../../../components/Main/Icons";
+import { CopyIcon, StorageIcon, VisibilityIcon } from "../../../components/Main/Icons";
 import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
+import { useSearchParams } from "react-router-dom";
+import { LOGS_GROUP_BY, LOGS_URL_PARAMS } from "../../../constants/logs";
 
 interface Props {
   field: string;
@@ -11,8 +13,17 @@ interface Props {
 
 const GroupLogsFieldRow: FC<Props> = ({ field, value }) => {
   const copyToClipboard = useCopyToClipboard();
+  const [searchParams, setSearchParams] = useSearchParams();
+
   const [copied, setCopied] = useState<boolean>(false);
 
+  const groupBy = searchParams.get(LOGS_URL_PARAMS.GROUP_BY) || LOGS_GROUP_BY;
+  const displayFieldsString = searchParams.get(LOGS_URL_PARAMS.DISPLAY_FIELDS) || "";
+  const displayFields = displayFieldsString ? displayFieldsString.split(",") : [];
+
+  const isSelectedField = displayFields.includes(field);
+  const isGroupByField = groupBy === field;
+
   const handleCopy = useCallback(async () => {
     if (copied) return;
     try {
@@ -23,6 +34,18 @@ const GroupLogsFieldRow: FC<Props> = ({ field, value }) => {
     }
   }, [copied, copyToClipboard]);
 
+  const handleSelectDisplayField = () => {
+    const prev = displayFields;
+    const newDisplayFields = prev.includes(field) ? prev.filter(v => v !== field) : [...prev, field];
+    searchParams.set(LOGS_URL_PARAMS.DISPLAY_FIELDS, newDisplayFields.join(","));
+    setSearchParams(searchParams);
+  };
+
+  const handleSelectGroupBy = () => {
+    isGroupByField ? searchParams.delete(LOGS_URL_PARAMS.GROUP_BY) : searchParams.set(LOGS_URL_PARAMS.GROUP_BY, field);
+    setSearchParams(searchParams);
+  };
+
   useEffect(() => {
     if (copied === null) return;
     const timeout = setTimeout(() => setCopied(false), 2000);
@@ -35,6 +58,7 @@ const GroupLogsFieldRow: FC<Props> = ({ field, value }) => {
         <div className="vm-group-logs-row-fields-item-controls__wrapper">
           <Tooltip title={copied ? "Copied" : "Copy to clipboard"}>
             <Button
+              className="vm-group-logs-row-fields-item-controls__button"
               variant="text"
               color="gray"
               size="small"
@@ -43,6 +67,34 @@ const GroupLogsFieldRow: FC<Props> = ({ field, value }) => {
               ariaLabel="copy to clipboard"
             />
           </Tooltip>
+          <Tooltip
+            key={`${field}_${isSelectedField}_${isGroupByField}`}
+            title={isSelectedField ? "Hide this field" : "Show this field instead of the message"}
+          >
+            <Button
+              className="vm-group-logs-row-fields-item-controls__button"
+              variant="text"
+              color={isSelectedField ? "secondary" : "gray"}
+              size="small"
+              startIcon={isSelectedField ? <VisibilityIcon/> : <VisibilityIcon/>}
+              onClick={handleSelectDisplayField}
+              ariaLabel="copy to clipboard"
+            />
+          </Tooltip>
+          <Tooltip
+            key={`${field}_${isSelectedField}_${isGroupByField}`}
+            title={isGroupByField ? "Ungroup this field" : "Group by this field"}
+          >
+            <Button
+              className="vm-group-logs-row-fields-item-controls__button"
+              variant="text"
+              color={isGroupByField ? "secondary" : "gray"}
+              size="small"
+              startIcon={<StorageIcon/>}
+              onClick={handleSelectGroupBy}
+              ariaLabel="copy to clipboard"
+            />
+          </Tooltip>
         </div>
       </td>
       <td className="vm-group-logs-row-fields-item__key">{field}</td>

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsHeader.tsx

@@ -0,0 +1,127 @@
+import React, { FC, useCallback, useEffect, useRef } from "preact/compat";
+import classNames from "classnames";
+import { useSearchParams } from "react-router-dom";
+import { MouseEvent, useState } from "react";
+import { useAppState } from "../../../state/common/StateContext";
+import { Logs } from "../../../api/types";
+import useEventListener from "../../../hooks/useEventListener";
+import Popper from "../../../components/Main/Popper/Popper";
+import useBoolean from "../../../hooks/useBoolean";
+import GroupLogsHeaderItem from "./GroupLogsHeaderItem";
+import { LOGS_GROUP_BY, LOGS_URL_PARAMS } from "../../../constants/logs";
+
+interface Props {
+  group: {
+    keys: string[]
+    keysString: string
+    values: Logs[]
+    pairs: string[]
+  };
+}
+
+const GroupLogsHeader: FC<Props> = ({ group }) => {
+  const { isDarkTheme } = useAppState();
+  const [searchParams] = useSearchParams();
+
+  const containerRef = useRef<HTMLDivElement>(null);
+  const moreRef = useRef<HTMLDivElement>(null);
+
+  const {
+    value: openMore,
+    toggle: handleToggleMore,
+    setFalse: handleCloseMore,
+  } = useBoolean(false);
+
+  const [hideParisCount, setHideParisCount] = useState<number>(0);
+
+  const groupBy = searchParams.get(LOGS_URL_PARAMS.GROUP_BY) || LOGS_GROUP_BY;
+  const compactGroupHeader = searchParams.get(LOGS_URL_PARAMS.COMPACT_GROUP_HEADER) === "true";
+
+  const pairs = group.pairs;
+  const hideAboveIndex = pairs.length - hideParisCount - 1;
+
+  const handleClickMore = (e: MouseEvent<HTMLDivElement>) => {
+    e.stopPropagation();
+    handleToggleMore();
+  };
+
+  const calcVisiblePairsCount = useCallback(() => {
+    if (!compactGroupHeader || !containerRef.current) {
+      setHideParisCount(0);
+      return;
+    }
+
+    const container = containerRef.current;
+    const containerSize = container.getBoundingClientRect();
+    const selector = ".vm-group-logs-section-keys__pair:not(.vm-group-logs-section-keys__pair_more)";
+    const children = Array.from(container.querySelectorAll(selector));
+    let count = 0;
+
+    for (const child of children) {
+      const { right } = (child as HTMLElement).getBoundingClientRect();
+
+      if ((right + 220) > containerSize.width) {
+        count++;
+      }
+    }
+
+    setHideParisCount(count);
+  }, [compactGroupHeader, containerRef]);
+
+  useEffect(calcVisiblePairsCount, [group.pairs, compactGroupHeader, containerRef]);
+
+  useEventListener("resize", calcVisiblePairsCount);
+
+  return (
+    <div
+      className={classNames({
+        "vm-group-logs-section-keys": true,
+        "vm-group-logs-section-keys_compact": compactGroupHeader,
+      })}
+      ref={containerRef}
+    >
+      <span className="vm-group-logs-section-keys__title">Group by <code>{groupBy}</code>:</span>
+      {pairs.map((pair, i) => (
+        <GroupLogsHeaderItem
+          key={`${group.keysString}_${pair}`}
+          pair={pair}
+          isHide={hideParisCount ? i > hideAboveIndex : false}
+        />
+      ))}
+      {hideParisCount > 0 && (
+        <>
+          <div
+            className={classNames({
+              "vm-group-logs-section-keys__pair": true,
+              "vm-group-logs-section-keys__pair_more": true,
+              "vm-group-logs-section-keys__pair_dark": isDarkTheme
+            })}
+            ref={moreRef}
+            onClick={handleClickMore}
+          >
+            +{hideParisCount} more
+          </div>
+          <Popper
+            open={openMore}
+            buttonRef={moreRef}
+            placement="bottom-left"
+            onClose={handleCloseMore}
+          >
+            <div className="vm-group-logs-section-keys vm-group-logs-section-keys_popper">
+              {pairs.slice(hideAboveIndex + 1).map((pair) => (
+                <GroupLogsHeaderItem
+                  key={`${group.keysString}_${pair}`}
+                  pair={pair}
+                />
+              ))}
+            </div>
+          </Popper>
+        </>
+      )}
+      <span className="vm-group-logs-section-keys__count">{group.values.length} entries</span>
+    </div>
+  )
+  ;
+};
+
+export default GroupLogsHeader;

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsHeaderItem.tsx

@@ -0,0 +1,59 @@
+import React, { FC, useEffect } from "preact/compat";
+import { useAppState } from "../../../state/common/StateContext";
+import Tooltip from "../../../components/Main/Tooltip/Tooltip";
+import classNames from "classnames";
+import { MouseEvent, useState } from "react";
+import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
+import { useSearchParams } from "react-router-dom";
+import { LOGS_GROUP_BY, LOGS_URL_PARAMS } from "../../../constants/logs";
+import { convertToFieldFilter } from "../../../utils/logs";
+
+interface Props {
+  pair: string;
+  isHide?: boolean;
+}
+
+const GroupLogsHeaderItem: FC<Props> = ({ pair, isHide }) => {
+  const { isDarkTheme } = useAppState();
+  const copyToClipboard = useCopyToClipboard();
+  const [searchParams] = useSearchParams();
+
+  const [copied, setCopied] = useState<string | null>(null);
+
+  const groupBy = searchParams.get(LOGS_URL_PARAMS.GROUP_BY) || LOGS_GROUP_BY;
+
+  const handleClickByPair = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
+    e.stopPropagation();
+    const copyValue = convertToFieldFilter(value, groupBy);
+    const isCopied = await copyToClipboard(copyValue);
+    if (isCopied) {
+      setCopied(value);
+    }
+  };
+
+  useEffect(() => {
+    if (copied === null) return;
+    const timeout = setTimeout(() => setCopied(null), 2000);
+    return () => clearTimeout(timeout);
+  }, [copied]);
+
+  return (
+    <Tooltip
+      title={copied === pair ? "Copied" : "Copy to clipboard"}
+      placement={"top-center"}
+    >
+      <div
+        className={classNames({
+          "vm-group-logs-section-keys__pair": true,
+          "vm-group-logs-section-keys__pair_hide": isHide,
+          "vm-group-logs-section-keys__pair_dark": isDarkTheme
+        })}
+        onClick={handleClickByPair(pair)}
+      >
+        {pair}
+      </div>
+    </Tooltip>
+  );
+};
+
+export default GroupLogsHeaderItem;

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx

@@ -6,28 +6,34 @@ import { ArrowDownIcon } from "../../../components/Main/Icons";
 import classNames from "classnames";
 import { useLogsState } from "../../../state/logsPanel/LogsStateContext";
 import dayjs from "dayjs";
-import { DATE_TIME_FORMAT } from "../../../constants/date";
 import { useTimeState } from "../../../state/time/TimeStateContext";
 import GroupLogsFieldRow from "./GroupLogsFieldRow";
 import { marked } from "marked";
+import { useSearchParams } from "react-router-dom";
+import { LOGS_DATE_FORMAT, LOGS_URL_PARAMS } from "../../../constants/logs";
 
 interface Props {
   log: Logs;
+  displayFields?: string[];
 }
 
-const GroupLogsItem: FC<Props> = ({ log }) => {
+const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"] }) => {
   const {
     value: isOpenFields,
     toggle: toggleOpenFields,
   } = useBoolean(false);
 
+  const [searchParams] = useSearchParams();
   const { markdownParsing } = useLogsState();
   const { timezone } = useTimeState();
 
+  const noWrapLines = searchParams.get(LOGS_URL_PARAMS.NO_WRAP_LINES) === "true";
+  const dateFormat = searchParams.get(LOGS_URL_PARAMS.DATE_FORMAT) || LOGS_DATE_FORMAT;
+
   const formattedTime = useMemo(() => {
     if (!log._time) return "";
-    return dayjs(log._time).tz().format(`${DATE_TIME_FORMAT}.SSS`);
-  }, [log._time, timezone]);
+    return dayjs(log._time).tz().format(dateFormat);
+  }, [log._time, timezone, dateFormat]);
 
   const formattedMarkdown = useMemo(() => {
     if (!markdownParsing || !log._msg) return "";
@@ -38,6 +44,14 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
   const hasFields = fields.length > 0;
 
   const displayMessage = useMemo(() => {
+    if (displayFields.length) {
+      return displayFields.filter(field => log[field]).map((field, i) => (
+        <span
+          className="vm-group-logs-row-content__sub-msg"
+          key={field + i}
+        >{log[field]}</span>
+      ));
+    }
     if (log._msg) return log._msg;
     if (!hasFields) return;
     const dataObject = fields.reduce<{ [key: string]: string }>((obj, [key, value]) => {
@@ -45,7 +59,7 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
       return obj;
     }, {});
     return JSON.stringify(dataObject);
-  }, [log, fields, hasFields]);
+  }, [log, fields, hasFields, displayFields]);
 
   return (
     <div className="vm-group-logs-row">
@@ -76,7 +90,8 @@ const GroupLogsItem: FC<Props> = ({ log }) => {
           className={classNames({
             "vm-group-logs-row-content__msg": true,
             "vm-group-logs-row-content__msg_empty-msg": !log._msg,
-            "vm-group-logs-row-content__msg_missing": !displayMessage
+            "vm-group-logs-row-content__msg_missing": !displayMessage,
+            "vm-group-logs-row-content__msg_single-line": noWrapLines,
           })}
           dangerouslySetInnerHTML={(markdownParsing && formattedMarkdown) ? { __html: formattedMarkdown } : undefined}
         >

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss

@@ -1,5 +1,7 @@
 @use "src/styles/variables" as *;
 
+$font-size-logs: var(--font-size-logs, $font-size-small);
+
 .vm-group-logs {
   margin-top: calc(-1 * $padding-medium);
 
@@ -19,22 +21,44 @@
   }
 
   &-section {
+    border-bottom: $border-divider;
+
     &-keys {
+      position: relative;
       display: flex;
       align-items: center;
       flex-wrap: wrap;
       gap: $padding-small;
-      border-bottom: $border-divider;
-      padding: $padding-small 0;
+      padding: $padding-small 120px $padding-small 0;
+      font-size: $font-size-logs;
+
+      &_compact {
+        flex-wrap: nowrap;
+        overflow: hidden;
+      }
+
+      &_popper {
+        display: flex;
+        flex-wrap: nowrap;
+        flex-direction: column;
+        align-items: flex-start;
+        justify-content: flex-start;
+        padding: $padding-global;
+        max-height: 400px;
+        overflow: auto;
+      }
 
       &__title {
         font-weight: bold;
+        white-space: nowrap;
 
         code {
           font-family: monospace;
+
           &:before {
             content: "\"";
           }
+
           &:after {
             content: "\"";
           }
@@ -42,19 +66,35 @@
       }
 
       &__count {
+        position: absolute;
+        top: auto;
+        right: 0;
         flex-grow: 1;
         text-align: right;
-        font-size: $font-size-small;
+        font-size: $font-size-logs;
         color: $color-text-secondary;
         padding-right: calc($padding-large * 3);
       }
 
       &__pair {
+        order: 0;
         padding: calc($padding-global / 2) $padding-global;
         background-color: lighten($color-tropical-blue, 6%);
         color: darken($color-dodger-blue, 20%);
         border-radius: $border-radius-medium;
         transition: background-color 0.3s ease-in, transform 0.1s ease-in, opacity 0.3s ease-in;
+        white-space: nowrap;
+
+        &_hide {
+          order: 2;
+          visibility: hidden;
+          opacity: 0;
+          pointer-events: none;
+        }
+
+        &_more {
+          order: 1;
+        }
 
         &:hover {
           background-color: $color-tropical-blue;
@@ -84,13 +124,19 @@
 
   &-row {
     position: relative;
-    border-bottom: $border-divider;
+
+    &:last-child {
+      margin-bottom: $padding-small;
+    }
 
     &-content {
       position: relative;
       display: grid;
-      grid-template-columns: auto minmax(180px, max-content) 1fr;
-      padding: $padding-global 0;
+      grid-template-columns: auto max-content 1fr;
+      padding: calc($padding-small / 4) 0;
+      font-size: $font-size-logs;
+      font-variant-numeric: tabular-nums;
+      line-height: 1.3;
       cursor: pointer;
       transition: background-color 0.2s ease-in;
 
@@ -116,8 +162,7 @@
         display: flex;
         align-items: flex-start;
         justify-content: flex-end;
-        margin-right: $padding-small;
-        line-height: 1;
+        padding: 0 $padding-global 0 $padding-small;
         white-space: nowrap;
 
         &_missing {
@@ -130,7 +175,12 @@
       &__msg {
         font-family: $font-family-monospace;
         overflow-wrap: anywhere;
-        line-height: 1.1;
+
+        &_single-line {
+          overflow: hidden;
+          text-overflow: ellipsis;
+          white-space: nowrap;
+        }
 
         &_empty-msg {
           overflow: hidden;
@@ -158,7 +208,7 @@
           border-radius: $border-radius-small;
           tab-size: 4;
           font-variant-ligatures: none;
-          margin: calc($padding-small/4) 0;
+          margin: calc($padding-small / 4) 0;
         }
 
         p {
@@ -171,7 +221,7 @@
         }
 
         code {
-          font-size: $font-size-small;
+          font-size: $font-size-logs;
           padding: calc($padding-small / 4) calc($padding-small / 2);
         }
 
@@ -194,25 +244,35 @@
 
         blockquote {
           border-left: 4px solid $color-hover-black;
-          margin: calc($padding-small/2) $padding-small;
-          padding: calc($padding-small/2) $padding-small;
+          margin: calc($padding-small / 2) $padding-small;
+          padding: calc($padding-small / 2) $padding-small;
         }
 
         ul, ol {
           list-style-position: inside;
         }
+
         /* end styles for markdown */
       }
+
+      &__sub-msg {
+        padding-right: $padding-global;
+      }
     }
 
     &-fields {
+      position: relative;
       grid-row: 2;
       padding: $padding-small 0;
-      margin-bottom: $padding-small;
+      margin: $padding-small 0 $padding-small calc($padding-global * 2);
       border: $border-divider;
       border-radius: $border-radius-small;
       overflow: auto;
       max-height: 300px;
+      resize: vertical;
+      font-family: $font-family-monospace;
+      font-size: $font-size-logs;
+      font-variant-numeric: tabular-nums;
 
       &-item {
         border-radius: $border-radius-small;
@@ -223,19 +283,26 @@
         }
 
         &-controls {
-          padding: 0;
+          padding: 0 calc($padding-small / 2);
 
           &__wrapper {
             display: flex;
             align-items: center;
             justify-content: center;
           }
+
+          &__button.vm-button_small {
+            width: 22px;
+            height: 22px;
+            min-height: 22px;
+          }
         }
 
         &__key,
         &__value {
           vertical-align: top;
-          padding: calc($padding-small / 2) $padding-global;
+          line-height: $font-size;
+          padding: calc($padding-small / 2);
         }
 
         &__key {

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogHits.ts

@@ -4,6 +4,8 @@ import { ErrorTypes, TimeParams } from "../../../types";
 import { LogHits } from "../../../api/types";
 import { useSearchParams } from "react-router-dom";
 import { getHitsTimeParams } from "../../../utils/logs";
+import { LOGS_GROUP_BY, LOGS_LIMIT_HITS } from "../../../constants/logs";
+import { isEmptyObject } from "../../../utils/object";
 
 export const useFetchLogHits = (server: string, query: string) => {
   const [searchParams] = useSearchParams();
@@ -30,46 +32,12 @@ export const useFetchLogHits = (server: string, query: string) => {
         step: `${step}ms`,
         start: start.toISOString(),
         end: end.toISOString(),
-        field: "_stream" // In the future, this field can be made configurable
-
+        fields_limit: `${LOGS_LIMIT_HITS}`,
+        field: LOGS_GROUP_BY,
       })
     };
   };
 
-  const accumulateHits = (resultHit: LogHits, hit: LogHits) => {
-    resultHit.total = (resultHit.total || 0) + (hit.total || 0);
-    hit.timestamps.forEach((timestamp, i) => {
-      const index = resultHit.timestamps.findIndex(t => t === timestamp);
-      if (index === -1) {
-        resultHit.timestamps.push(timestamp);
-        resultHit.values.push(hit.values[i]);
-      } else {
-        resultHit.values[index] += hit.values[i];
-      }
-    });
-    return resultHit;
-  };
-
-  const getHitsWithTop = (hits: LogHits[]) => {
-    const topN = 5;
-    const defaultHit = { fields: {}, timestamps: [], values: [], total: 0 };
-
-    const hitsByTotal = hits.sort((a, b) => (b.total || 0) - (a.total || 0));
-    const result = [];
-
-    const otherHits: LogHits = hitsByTotal.slice(topN).reduce(accumulateHits, defaultHit);
-    if (otherHits.total) {
-      result.push(otherHits);
-    }
-
-    const topHits: LogHits[] = hitsByTotal.slice(0, topN);
-    if (topHits.length) {
-      result.push(...topHits);
-    }
-
-    return result;
-  };
-
   const fetchLogHits = useCallback(async (period: TimeParams) => {
     abortControllerRef.current.abort();
     abortControllerRef.current = new AbortController();
@@ -98,7 +66,7 @@ export const useFetchLogHits = (server: string, query: string) => {
         setError(error);
       }
 
-      setLogHits(!hits ? [] : getHitsWithTop(hits));
+      setLogHits(hits.map(hit => ({ ...hit, _isOther: isEmptyObject(hit.fields) })));
     } catch (e) {
       if (e instanceof Error && e.name !== "AbortError") {
         setError(String(e));

app/vmui/packages/vmui/src/utils/logs.ts

@@ -1,6 +1,8 @@
 import { TimeParams } from "../types";
 import dayjs from "dayjs";
-import { LOGS_BARS_VIEW } from "../constants/logs";
+import { LOGS_BARS_VIEW, LOGS_GROUP_BY } from "../constants/logs";
+import { LogHits } from "../api/types";
+import { OTHER_HITS_LABEL } from "../components/Chart/BarHitsChart/hooks/useBarHitsOptions";
 
 export const getStreamPairs = (value: string): string[] => {
   const pairs = /^{.+}$/.test(value) ? value.slice(1, -1).split(",") : [value];
@@ -14,3 +16,27 @@ export const getHitsTimeParams = (period: TimeParams) => {
   const step = Math.ceil(totalSeconds / LOGS_BARS_VIEW) || 1;
   return { start, end, step };
 };
+
+export const convertToFieldFilter = (value: string, field = LOGS_GROUP_BY) => {
+  const isKeyValue = /(.+)?=(".+")/.test(value);
+
+  if (isKeyValue) {
+    return value.replace(/=/, ": ");
+  }
+
+  return `${field}: "${value}"`;
+};
+
+export const calculateTotalHits = (hits: LogHits[]): number => {
+  return hits.reduce((acc, item) => acc + (item.total || 0), 0);
+};
+
+export const sortLogHits = <T extends { label?: string }>(key: keyof T) => (a: T, b: T): number => {
+  if (a.label === OTHER_HITS_LABEL) return 1;
+  if (b.label === OTHER_HITS_LABEL) return -1;
+
+  const aValue = a[key] as unknown as number;
+  const bValue = b[key] as unknown as number;
+
+  return bValue - aValue;
+};

app/vmui/packages/vmui/src/utils/math.ts

@@ -57,3 +57,15 @@ export const getLastFromArray = (a: number[]) => {
     }
   }
 };
+
+export const formatNumberShort = (value: number) => {
+  if (value >= 1_000_000_000) {
+    return (value / 1_000_000_000).toFixed(1).replace(/\.0$/, "") + "B"; // Миллиарды
+  } else if (value >= 1_000_000) {
+    return (value / 1_000_000).toFixed(1).replace(/\.0$/, "") + "M"; // Миллионы
+  } else if (value >= 1_000) {
+    return (value / 1_000).toFixed(1).replace(/\.0$/, "") + "K"; // Тысячи
+  } else {
+    return value.toString(); // Для чисел меньше 1000
+  }
+};

app/vmui/packages/vmui/src/utils/object.ts

@@ -14,3 +14,7 @@ export function filterObject<T extends object>(
 export function compactObject<T extends object>(obj: T) {
   return filterObject(obj, (entry) => !!entry[1] || typeof entry[1] === "number");
 }
+
+export function isEmptyObject(obj: object) {
+  return Object.keys(obj).length === 0;
+}

app/vmui/packages/vmui/src/utils/storage.ts

@@ -3,7 +3,6 @@ export type StorageKeys = "AUTOCOMPLETE"
     | "QUERY_TRACING"
     | "SERIES_LIMITS"
     | "TABLE_COMPACT"
-    | "TABLE_COLUMNS"
     | "TIMEZONE"
     | "DISABLED_DEFAULT_TIMEZONE"
     | "THEME"

app/vmui/packages/vmui/web/main.go

@@ -10,7 +10,7 @@ import (
 // specific files
 // static content
 //
-//go:embed favicon-32x32.png robots.txt index.html manifest.json asset-manifest.json
+//go:embed favicon.svg robots.txt index.html manifest.json asset-manifest.json
 //go:embed static
 var files embed.FS
 

apptest/tests/multitenant_test.go

@@ -171,4 +171,49 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
+	// Delete series from specific tenant
+	vmselect.DeleteSeries(t, "foo_bar", apptest.QueryOpts{
+		Tenant: "5:15",
+	})
+	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"},
+        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
+              ]
+     }`)
+	wantSR.Sort()
+
+	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+		Tenant: "multitenant",
+		Start:  "2022-05-10T08:03:00.000Z",
+	})
+	gotSR.Sort()
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Delete series for multitenant with tenant filter
+	vmselect.DeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+		Tenant: "multitenant",
+	})
+
+	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
+        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
+              ]
+     }`)
+	wantSR.Sort()
+
+	gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{
+		Tenant: "multitenant",
+		Start:  "2022-05-10T08:03:00.000Z",
+	})
+	gotSR.Sort()
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
 }

apptest/tests/sparse_cache_test.go

@@ -0,0 +1,76 @@
+package tests
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestStorageUsesSparseCacheForFinalMerge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartVmsingle("sparse-cache-final-merge", []string{`-retentionPeriod=100y`, `-downsampling.period={__name__=~"metric.*"}:5m:1s`})
+
+	// insert metrics daily over past 2 months
+	const metricsPerStep = 10
+	const steps = 35
+	const stepSize = int64(1 * 24 * 60 * 60 * 1000)
+	records := make([]string, metricsPerStep)
+	ts := time.Now().Add(-2 * 30 * 24 * time.Hour).UnixMilli()
+	for range steps {
+		for i := range metricsPerStep {
+			name := fmt.Sprintf("metric_%d", i)
+			records[i] = fmt.Sprintf("%s %d %d", name, rand.IntN(1000), ts)
+		}
+		sut.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
+		ts += stepSize
+	}
+	sut.ForceFlush(t)
+	sut.ForceMerge(t)
+
+	// todo: replace with a more reliable way to check if the merge is completed
+	// wait for merge to be completed
+	time.Sleep(5 * time.Second)
+
+	v := sut.GetIntMetric(t, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`)
+	if v <= 0 {
+		t.Fatalf(`unexpected vm_cache_requests_total{type="indexdb/dataBlocksSparse"} value: %d`, v)
+	}
+}
+
+func TestStorageDoesNotUseSparseCacheForRegularMerge(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	sut := tc.MustStartVmsingle("sparse-cache-regular-merge", []string{`-retentionPeriod=100y`, `-downsampling.period={__name__=~"metric.*"}:5m:1s`})
+
+	// insert metrics into current month only
+	const metricsPerStep = 10
+	const steps = 2
+	const stepSize = 60 * 1000
+	records := make([]string, metricsPerStep)
+	ts := time.Now().Add(-1 * time.Hour).UnixMilli()
+	for range steps {
+		for i := range metricsPerStep {
+			name := fmt.Sprintf("metric_%d", i)
+			records[i] = fmt.Sprintf("%s %d %d", name, rand.IntN(1000), ts)
+		}
+		sut.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
+		ts += stepSize
+	}
+	sut.ForceFlush(t)
+	sut.ForceMerge(t)
+
+	// todo: replace with a more reliable way to check if the merge is completed
+	// wait for merge to be completed
+	time.Sleep(5 * time.Second)
+
+	v := sut.GetIntMetric(t, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`)
+	if v == 0 {
+		t.Fatalf(`unexpected vm_cache_requests_total{type="indexdb/dataBlocksSparse"} value: %d`, v)
+	}
+}