app/vminsert: moves replication at datapointWrite level

* It allows to tolerate slow storage node for replicationFactor > 1 and disableRerouting=false
* It allows to have partial replication with replicationFactor > and dropSamplesOnOverload=true
* It increases cpu and memory usage by 30-50% per replica

Signed-off-by: f41gh7 <nik@victoriametrics.com>

.gitignore

@@ -7,6 +7,7 @@
 .vscode
 *.test
 *.swp
+/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data

Makefile

@@ -234,7 +234,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`
@@ -253,34 +253,3 @@ install-wwhrd:
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
-
-copy-docs:
-# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
-# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
-	echo "---" > ${DST}
-	@if [ ${ORDER} -ne 0 ]; then \
-		echo "sort: ${ORDER}" >> ${DST}; \
-		echo "weight: ${ORDER}" >> ${DST}; \
-		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
-	fi
-
-	echo "title: ${TITLE}" >> ${DST}
-	@if [ ${OLD_URL} ]; then \
-		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
-	fi
-	echo "---" >> ${DST}
-	cat ${SRC} >> ${DST}
-	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"\//' ${DST}
-	sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"\//' ${DST}
-	sed -i='.tmp' 's/](docs\//](/' ${DST}
-	rm -rf docs/*.tmp
-
-# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
-# For ORDER=0 it adds no order tag/weight tag.
-# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
-# Images starting with <img src="docs/ are replaced with <img src="
-# Cluster docs are supposed to be ordered as 2nd.
-# The rest of docs is ordered manually.
-docs-sync:
-	SRC=README.md DST=docs/Cluster-VictoriaMetrics.md OLD_URL='/Cluster-VictoriaMetrics.html' ORDER=2 TITLE='Cluster version' $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
+| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to security@victoriametrics.com
+Please report any security issues to <security@victoriametrics.com>

app/vlinsert/elasticsearch/elasticsearch.go

@@ -57,6 +57,12 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
+	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
+		// Return fake response for Logstash APIs requests.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
 	switch path {
 	case "/":
 		switch r.Method {

app/vmagent/main.go

@@ -318,6 +318,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -564,6 +568,10 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -674,7 +682,8 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/remotewrite/remotewrite.go

@@ -441,7 +441,7 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 {
+	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}

app/vmagent/remotewrite/streamaggr.go

@@ -50,7 +50,7 @@ var (
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
 		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
@@ -133,7 +133,7 @@ func initStreamAggrConfigGlobal() {
 	} else {
 		dedupInterval := streamAggrGlobalDedupInterval.Duration()
 		if dedupInterval > 0 {
-			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrDropInputLabels, "dedup-global")
+			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 		}
 	}
 }
@@ -202,6 +202,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 		DropInputLabels:      *streamAggrGlobalDropInputLabels,
 		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+		KeepInput:            *streamAggrGlobalKeepInput,
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
@@ -229,7 +230,8 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
+		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

app/vmalert/datasource/init.go

@@ -17,7 +17,7 @@ import (
 
 var (
 	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
@@ -99,7 +99,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
 	}
 	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive

app/vmalert/datasource/vm_prom_api_timing_test.go

@@ -13,7 +13,7 @@ func BenchmarkMetrics(b *testing.B) {
 
 	var pi promInstant
 	if err := pi.Unmarshal(payload); err != nil {
-		b.Fatalf(err.Error())
+		b.Fatal(err.Error())
 	}
 	b.Run("Instant", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {

app/vmalert/notifier/alertmanager.go

@@ -76,9 +76,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	for key, value := range headers {
-		req.Header.Set(key, value)
-	}
 
 	if am.timeout > 0 {
 		var cancel context.CancelFunc
@@ -94,6 +91,11 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 			return err
 		}
 	}
+	// external headers have higher priority
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
 	resp, err := am.client.Do(req)
 	if err != nil {
 		return err
@@ -130,7 +132,8 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
+
 	}
 
 	ba := new(promauth.BasicAuthConfig)
@@ -145,7 +148,9 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
+		utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/notifier/alertmanager_test.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -48,6 +49,9 @@ func TestAlertManager_Send(t *testing.T) {
 			conn, _, _ := w.(http.Hijacker).Hijack()
 			_ = conn.Close()
 		case 1:
+			if r.Header.Get(headerKey) != headerValue {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			}
 			w.WriteHeader(500)
 		case 2:
 			var a []struct {
@@ -72,6 +76,9 @@ func TestAlertManager_Send(t *testing.T) {
 			if a[0].EndAt.IsZero() {
 				t.Fatalf("expected non-zero end time")
 			}
+			if r.Header.Get(headerKey) != "bar" {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			}
 		case 3:
 			if r.Header.Get(headerKey) != headerValue {
 				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
@@ -86,6 +93,7 @@ func TestAlertManager_Send(t *testing.T) {
 			Username: baUser,
 			Password: promauth.NewSecret(baPass),
 		},
+		Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
@@ -105,7 +113,7 @@ func TestAlertManager_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, nil); err != nil {
+	}}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 	if c != 2 {

app/vmalert/notifier/init.go

@@ -25,6 +25,9 @@ var (
 		"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
 		"-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.")
 
+	headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
+		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
@@ -171,6 +174,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
 				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
 			},
+			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
 		addr = strings.TrimSuffix(addr, "/")

app/vmalert/remoteread/init.go

@@ -17,7 +17,7 @@ var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
 		"Remote read is used to restore alerts state."+
 		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
 
 	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
@@ -68,7 +68,7 @@ func Init() (datasource.QuerierBuilder, error) {
 	}
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
 	tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")

app/vmalert/remotewrite/debug_client.go

@@ -32,7 +32,7 @@ func NewDebugClient() (*DebugClient, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
 	c := &DebugClient{
 		c: &http.Client{

app/vmalert/remotewrite/init.go

@@ -15,7 +15,7 @@ import (
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -72,7 +72,7 @@ func Init(ctx context.Context) (*Client, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
 	t.IdleConnTimeout = *idleConnectionTimeout
 	t.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")

app/vmalert/rule/alerting.go

@@ -441,9 +441,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			}
 			a.Value = m.Values[0]
 			a.Annotations = annotations
-			if err != nil {
-				return nil, err
-			}
 			a.KeepFiringSince = time.Time{}
 			continue
 		}

app/vmctl/backoff/backoff.go

@@ -10,12 +10,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-const (
-	backoffRetries     = 10
-	backoffFactor      = 1.8
-	backoffMinDuration = time.Second * 2
-)
-
 // retryableFunc describes call back which will repeat on errors
 type retryableFunc func() error
 
@@ -30,12 +24,22 @@ type Backoff struct {
 }
 
 // New initialize backoff object
-func New() *Backoff {
-	return &Backoff{
-		retries:     backoffRetries,
-		factor:      backoffFactor,
-		minDuration: backoffMinDuration,
+func New(retries int, factor float64, minDuration time.Duration) (*Backoff, error) {
+	if retries <= 0 {
+		return nil, fmt.Errorf("number of backoff retries must be greater than 0")
 	}
+	if factor <= 1 {
+		return nil, fmt.Errorf("backoff retry factor must be greater than 1")
+	}
+	if minDuration <= 0 {
+		return nil, fmt.Errorf("backoff retry minimum duration must be greater than 0")
+	}
+
+	return &Backoff{
+		retries:     retries,
+		factor:      factor,
+		minDuration: minDuration,
+	}, nil
 }
 
 // Retry process retries until all attempts are completed

app/vmctl/backoff/backoff_test.go

@@ -3,6 +3,7 @@ package backoff
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 )
@@ -110,3 +111,32 @@ func TestBackoffRetry_Success(t *testing.T) {
 	resultExpected := 1
 	f(retryFunc, resultExpected)
 }
+
+func TestBackoff_New(t *testing.T) {
+	f := func(retries int, factor float64, minDuration time.Duration, errExpected string) {
+		t.Helper()
+
+		_, err := New(retries, factor, minDuration)
+		if err == nil {
+			if errExpected != "" {
+				t.Fatalf("expecting non-nil error")
+			}
+			return
+		}
+		if !strings.Contains(err.Error(), errExpected) {
+			t.Fatalf("unexpected error: got %q; want %q", err.Error(), errExpected)
+		}
+	}
+
+	// empty retries
+	f(0, 1.1, time.Millisecond*10, "retries must be greater than 0")
+
+	// empty factor
+	f(1, 0, time.Millisecond*10, "factor must be greater than 1")
+
+	// empty minDuration
+	f(1, 1.1, 0, "minimum duration must be greater than 0")
+
+	// no errors
+	f(1, 1.1, time.Millisecond*10, "")
+}

app/vmctl/flags.go

@@ -56,6 +56,10 @@ const (
 	vmRateLimit  = "vm-rate-limit"
 
 	vmInterCluster = "vm-intercluster"
+
+	vmBackoffRetries     = "vm-backoff-retries"
+	vmBackoffFactor      = "vm-backoff-factor"
+	vmBackoffMinDuration = "vm-backoff-min-duration"
 )
 
 var (
@@ -146,6 +150,21 @@ var (
 			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
 			Value: false,
 		},
+		&cli.IntFlag{
+			Name:  vmBackoffRetries,
+			Value: 10,
+			Usage: "How many import retries to perform before giving up.",
+		},
+		&cli.Float64Flag{
+			Name:  vmBackoffFactor,
+			Value: 1.8,
+			Usage: "Factor to multiply the base duration after each failed import retry. Must be greater than 1.0",
+		},
+		&cli.DurationFlag{
+			Name:  vmBackoffMinDuration,
+			Value: time.Second * 2,
+			Usage: "Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'.",
+		},
 	}
 )
 
@@ -430,6 +449,10 @@ const (
 	vmNativeDstCAFile             = "vm-native-dst-ca-file"
 	vmNativeDstServerName         = "vm-native-dst-server-name"
 	vmNativeDstInsecureSkipVerify = "vm-native-dst-insecure-skip-verify"
+
+	vmNativeBackoffRetries     = "vm-native-backoff-retries"
+	vmNativeBackoffFactor      = "vm-native-backoff-factor"
+	vmNativeBackoffMinDuration = "vm-native-backoff-min-duration"
 )
 
 var (
@@ -599,6 +622,21 @@ var (
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},
+		&cli.IntFlag{
+			Name:  vmNativeBackoffRetries,
+			Value: 10,
+			Usage: "How many export/import retries to perform before giving up.",
+		},
+		&cli.Float64Flag{
+			Name:  vmNativeBackoffFactor,
+			Value: 1.8,
+			Usage: "Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0",
+		},
+		&cli.DurationFlag{
+			Name:  vmNativeBackoffMinDuration,
+			Value: time.Second * 2,
+			Usage: "Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'.",
+		},
 	}
 )
 

app/vmctl/main.go

@@ -68,7 +68,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create Transport: %s", err)
+						return fmt.Errorf("failed to create transport for -%s=%q: %s", otsdbAddr, addr, err)
 					}
 					oCfg := opentsdb.Config{
 						Addr:       addr,
@@ -90,6 +90,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -143,6 +144,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -178,7 +180,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create transport: %s", err)
+						return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
 					}
 
 					rr, err := remoteread.NewClient(remoteread.Config{
@@ -201,6 +203,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -233,6 +236,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -272,6 +276,14 @@ func main() {
 						return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
 					}
 
+					bfRetries := c.Int(vmNativeBackoffRetries)
+					bfFactor := c.Float64(vmNativeBackoffFactor)
+					bfMinDuration := c.Duration(vmNativeBackoffMinDuration)
+					bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
+					if err != nil {
+						return fmt.Errorf("failed to create backoff object: %s", err)
+					}
+
 					disableKeepAlive := c.Bool(vmNativeDisableHTTPKeepAlive)
 
 					var srcExtraLabels []string
@@ -350,7 +362,7 @@ func main() {
 							ExtraLabels: dstExtraLabels,
 							HTTPClient:  dstHTTPClient,
 						},
-						backoff:                  backoff.New(),
+						backoff:                  bf,
 						cc:                       c.Int(vmConcurrency),
 						disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration),
 						isNative:                 !c.Bool(vmNativeDisableBinaryProtocol),
@@ -426,7 +438,15 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 
 	tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 	if err != nil {
-		return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err)
+		return vm.Config{}, fmt.Errorf("failed to create transport for -%s=%q: %s", vmAddr, addr, err)
+	}
+
+	bfRetries := c.Int(vmBackoffRetries)
+	bfFactor := c.Float64(vmBackoffFactor)
+	bfMinDuration := c.Duration(vmBackoffMinDuration)
+	bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
+	if err != nil {
+		return vm.Config{}, fmt.Errorf("failed to create backoff object: %s", err)
 	}
 
 	return vm.Config{
@@ -442,5 +462,6 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		RoundDigits:        c.Int(vmRoundDigits),
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
 		RateLimit:          c.Int64(vmRateLimit),
+		Backoff:            bf,
 	}, nil
 }

app/vmctl/vm/vm.go

@@ -54,6 +54,8 @@ type Config struct {
 	// RateLimit defines a data transfer speed in bytes per second.
 	// Is applied to each worker (see Concurrency) independently.
 	RateLimit int64
+	// Backoff defines backoff policy for retries
+	Backoff *backoff.Backoff
 }
 
 // Importer performs insertion of timeseries
@@ -144,7 +146,7 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		close:      make(chan struct{}),
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
-		backoff:    backoff.New(),
+		backoff:    cfg.Backoff,
 	}
 	if err := im.Ping(); err != nil {
 		return nil, fmt.Errorf("ping to %q failed: %s", addr, err)

app/vminsert/clusternative/request_handler.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
@@ -14,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/clusternative/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -26,7 +25,7 @@ var (
 func InsertHandler(c net.Conn) error {
 	// There is no need in response compression, since
 	// lower-level vminsert sends only small packets to upper-level vminsert.
-	bc, err := handshake.VMInsertServer(c, 0, netstorage.GetNodeID())
+	bc, err := handshake.VMInsertServer(c, 0)
 	if err != nil {
 		if errors.Is(err, handshake.ErrIgnoreHealthcheck) {
 			return nil

app/vminsert/main.go

@@ -299,6 +299,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		addInfluxResponseHeaders(w)
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -423,7 +427,8 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/query", protocol="influx"}`)
+	influxQueryRequests  = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/query", protocol="influx"}`)
+	influxHealthRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/influx/health", protocol="influx"}`)
 
 	opentelemetryPushRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/opentelemetry/v1/metrics", protocol="opentelemetry"}`)

app/vminsert/netstorage/consistent_hash.go

@@ -1,5 +1,9 @@
 package netstorage
 
+import (
+	"github.com/cespare/xxhash/v2"
+)
+
 // See the following docs:
 // - https://www.eecs.umich.edu/techreports/cse/96/CSE-TR-316-96.pdf
 // - https://github.com/dgryski/go-rendezvous
@@ -9,10 +13,14 @@ type consistentHash struct {
 	nodeHashes []uint64
 }
 
-func newConsistentHash(ids []uint64, hashSeed uint64) *consistentHash {
+func newConsistentHash(nodes []string, hashSeed uint64) *consistentHash {
+	nodeHashes := make([]uint64, len(nodes))
+	for i, node := range nodes {
+		nodeHashes[i] = xxhash.Sum64([]byte(node))
+	}
 	return &consistentHash{
 		hashSeed:   hashSeed,
-		nodeHashes: ids,
+		nodeHashes: nodeHashes,
 	}
 }
 

app/vminsert/netstorage/consistent_hash_test.go

@@ -4,18 +4,16 @@ import (
 	"math"
 	"math/rand"
 	"testing"
-
-	"github.com/cespare/xxhash/v2"
 )
 
 func TestConsistentHash(t *testing.T) {
 	r := rand.New(rand.NewSource(1))
 
-	nodes := []uint64{
-		xxhash.Sum64String("node1"),
-		xxhash.Sum64String("node2"),
-		xxhash.Sum64String("node3"),
-		xxhash.Sum64String("node4"),
+	nodes := []string{
+		"node1",
+		"node2",
+		"node3",
+		"node4",
 	}
 	rh := newConsistentHash(nodes, 0)
 

app/vminsert/netstorage/consistent_hash_timing_test.go

@@ -4,19 +4,16 @@ import (
 	"math/rand"
 	"sync/atomic"
 	"testing"
-
-	"github.com/cespare/xxhash/v2"
 )
 
 func BenchmarkConsistentHash(b *testing.B) {
-	nodes := []uint64{
-		xxhash.Sum64String("node1"),
-		xxhash.Sum64String("node2"),
-		xxhash.Sum64String("node3"),
-		xxhash.Sum64String("node4"),
+	nodes := []string{
+		"node1",
+		"node2",
+		"node3",
+		"node4",
 	}
 	rh := newConsistentHash(nodes, 0)
-
 	b.ReportAllocs()
 	b.SetBytes(int64(len(benchKeys)))
 	b.RunParallel(func(pb *testing.PB) {

app/vminsert/netstorage/insert_ctx.go

@@ -5,8 +5,6 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/cespare/xxhash/v2"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -14,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/cespare/xxhash/v2"
 )
 
 // InsertCtx is a generic context for inserting data.
@@ -123,11 +122,48 @@ func (ctx *InsertCtx) ApplyRelabeling() {
 func (ctx *InsertCtx) WriteDataPoint(at *auth.Token, labels []prompb.Label, timestamp int64, value float64) error {
 	ctx.MetricNameBuf = storage.MarshalMetricNameRaw(ctx.MetricNameBuf[:0], at.AccountID, at.ProjectID, labels)
 	storageNodeIdx := ctx.GetStorageNodeIdx(at, labels)
-	return ctx.WriteDataPointExt(storageNodeIdx, ctx.MetricNameBuf, timestamp, value)
+	return ctx.writeDataPointToReplicas(storageNodeIdx, ctx.MetricNameBuf, timestamp, value)
 }
 
 // WriteDataPointExt writes the given metricNameRaw with (timestmap, value) to ctx buffer with the given storageNodeIdx.
 func (ctx *InsertCtx) WriteDataPointExt(storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
+	return ctx.writeDataPointToReplicas(storageNodeIdx, metricNameRaw, timestamp, value)
+}
+
+func (ctx *InsertCtx) writeDataPointToReplicas(storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
+	var firstErr error
+	var failsCount int
+	for i := 0; i < replicas; i++ {
+		snIdx := storageNodeIdx + i
+		if snIdx >= len(ctx.snb.sns) {
+			snIdx %= len(ctx.snb.sns)
+		}
+
+		if err := ctx.writeDataPointExt(snIdx, metricNameRaw, timestamp, value); err != nil {
+			if replicas == 1 {
+				return fmt.Errorf("cannot write datapoint: %w", err)
+			}
+			if firstErr == nil {
+				firstErr = err
+			}
+			failsCount++
+			// The data is partially replicated, so just emit a warning and return true.
+			// We could retry sending the data again, but this may result in uncontrolled duplicate data.
+			// So it is better returning true.
+			br := &ctx.bufRowss[snIdx]
+			rowsIncompletelyReplicatedTotal.Add(br.rows)
+			incompleteReplicationLogger.Warnf("cannot make a copy #%d out of %d copies according to -replicationFactor=%d, used_nodes=%d for %d bytes with %d rows, "+
+				"since a part of storage nodes is temporarily unavailable", i+1, replicas, *replicationFactor, len(br.buf), br.rows)
+			continue
+		}
+	}
+	if failsCount == replicas {
+		return fmt.Errorf("cannot write datapoint to any replicas: %w", firstErr)
+	}
+	return nil
+}
+
+func (ctx *InsertCtx) writeDataPointExt(storageNodeIdx int, metricNameRaw []byte, timestamp int64, value float64) error {
 	br := &ctx.bufRowss[storageNodeIdx]
 	snb := ctx.snb
 	sn := snb.sns[storageNodeIdx]

app/vminsert/netstorage/insert_ctx_pool.go

@@ -2,16 +2,23 @@ package netstorage
 
 import (
 	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 )
 
 // GetInsertCtx returns InsertCtx from the pool.
 //
 // Call PutInsertCtx for returning it to the pool.
 func GetInsertCtx() *InsertCtx {
-	if v := insertCtxPool.Get(); v != nil {
-		return v.(*InsertCtx)
+	select {
+	case ctx := <-insertCtxPoolCh:
+		return ctx
+	default:
+		if v := insertCtxPool.Get(); v != nil {
+			return v.(*InsertCtx)
+		}
+		return &InsertCtx{}
 	}
-	return &InsertCtx{}
 }
 
 // PutInsertCtx returns ctx to the pool.
@@ -19,7 +26,14 @@ func GetInsertCtx() *InsertCtx {
 // ctx cannot be used after the call.
 func PutInsertCtx(ctx *InsertCtx) {
 	ctx.Reset()
-	insertCtxPool.Put(ctx)
+	select {
+	case insertCtxPoolCh <- ctx:
+	default:
+		insertCtxPool.Put(ctx)
+	}
 }
 
-var insertCtxPool sync.Pool
+var (
+	insertCtxPool   sync.Pool
+	insertCtxPoolCh = make(chan *InsertCtx, cgroup.AvailableCPUs())
+)

app/vminsert/netstorage/netstorage.go

@@ -6,14 +6,10 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consts"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -25,6 +21,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 )
 
 var (
@@ -45,11 +43,10 @@ var (
 		"On the other side, disabled re-routing minimizes the number of active time series in the cluster "+
 		"during rolling restarts and during spikes in series churn rate. "+
 		"See also -disableRerouting")
-	usePersistentStorageNodeID = flag.Bool("vmstorageUsePersistentID", false, "Whether to use persistent storage node ID for -storageNode instances. "+
-		"If set to false uses storage node address in order to generate an ID. "+
-		"Using persistent node ID is useful if vmstorage node address changes over time, e.g. due to dynamic IP addresses or DNS names. ")
 )
 
+var replicas int
+
 var errStorageReadOnly = errors.New("storage node is read only")
 
 func (sn *storageNode) isReady() bool {
@@ -145,15 +142,6 @@ again:
 }
 
 func (sn *storageNode) run(snb *storageNodesBucket, snIdx int) {
-	replicas := *replicationFactor
-	if replicas <= 0 {
-		replicas = 1
-	}
-	sns := snb.sns
-	if replicas > len(sns) {
-		replicas = len(sns)
-	}
-
 	sn.readOnlyCheckerWG.Add(1)
 	go func() {
 		defer sn.readOnlyCheckerWG.Done()
@@ -198,7 +186,7 @@ func (sn *storageNode) run(snb *storageNodesBucket, snIdx int) {
 			continue
 		}
 		// Send br to replicas storage nodes starting from snIdx.
-		for !sendBufToReplicasNonblocking(snb, &br, snIdx, replicas) {
+		for !sendBufToSnNonblocking(snb, &br, snIdx) {
 			d := timeutil.AddJitterToDuration(time.Millisecond * 200)
 			t := timerpool.Get(d)
 			select {
@@ -211,50 +199,38 @@ func (sn *storageNode) run(snb *storageNodesBucket, snIdx int) {
 				sn.checkHealth()
 			}
 		}
+		if sn.isBufferFull.CompareAndSwap(true, false) {
+			logger.Infof("transited node=%s to non-full", sn.dialer.Addr())
+		}
+
 		br.reset()
 	}
 }
 
-func sendBufToReplicasNonblocking(snb *storageNodesBucket, br *bufRows, snIdx, replicas int) bool {
-	usedStorageNodes := make(map[*storageNode]struct{}, replicas)
+func sendBufToSnNonblocking(snb *storageNodesBucket, br *bufRows, snIdx int) bool {
 	sns := snb.sns
-	for i := 0; i < replicas; i++ {
-		idx := snIdx + i
-		attempts := 0
-		for {
-			attempts++
-			if attempts > len(sns) {
-				if i == 0 {
-					// The data wasn't replicated at all.
-					cannotReplicateLogger.Warnf("cannot push %d bytes with %d rows to storage nodes, since all the nodes are temporarily unavailable; "+
-						"re-trying to send the data soon", len(br.buf), br.rows)
-					return false
-				}
-				// The data is partially replicated, so just emit a warning and return true.
-				// We could retry sending the data again, but this may result in uncontrolled duplicate data.
-				// So it is better returning true.
-				rowsIncompletelyReplicatedTotal.Add(br.rows)
-				incompleteReplicationLogger.Warnf("cannot make a copy #%d out of %d copies according to -replicationFactor=%d for %d bytes with %d rows, "+
-					"since a part of storage nodes is temporarily unavailable", i+1, replicas, *replicationFactor, len(br.buf), br.rows)
-				return true
-			}
-			if idx >= len(sns) {
-				idx %= len(sns)
-			}
-			sn := sns[idx]
-			idx++
-			if _, ok := usedStorageNodes[sn]; ok {
-				// The br has been already replicated to sn. Skip it.
-				continue
-			}
-			if !sn.sendBufRowsNonblocking(br) {
-				// Cannot send data to sn. Go to the next sn.
-				continue
-			}
-			// Successfully sent data to sn.
-			usedStorageNodes[sn] = struct{}{}
-			break
+	idx := snIdx
+	attempts := 0
+	for {
+		attempts++
+		if attempts > len(sns) {
+			// The data wasn't replicated at all.
+			cannotReplicateLogger.Warnf("cannot push %d bytes with %d rows to storage nodes, since all the nodes are temporarily unavailable; "+
+				"re-trying to send the data soon", len(br.buf), br.rows)
+			return false
 		}
+		if idx >= len(sns) {
+			idx %= len(sns)
+		}
+		sn := sns[idx]
+		idx++
+
+		if !sn.sendBufRowsNonblocking(br) {
+			// Cannot send data to sn. Go to the next sn.
+			continue
+		}
+		// Successfully sent data to sn.
+		break
 	}
 	return true
 }
@@ -283,7 +259,7 @@ func (sn *storageNode) checkHealth() {
 		}
 		return
 	}
-	logger.Infof("successfully dialed -storageNode=%q (node ID: %d)", sn.dialer.Addr(), sn.id.Load())
+	logger.Infof("successfully dialed -storageNode=%q", sn.dialer.Addr())
 	sn.lastDialErr = nil
 	sn.bc = bc
 	sn.isBroken.Store(false)
@@ -403,30 +379,23 @@ func (sn *storageNode) dial() (*handshake.BufferedConn, error) {
 	if *disableRPCCompression {
 		compressionLevel = 0
 	}
-	bc, id, err := handshake.VMInsertClient(c, compressionLevel)
+	bc, err := handshake.VMInsertClient(c, compressionLevel)
 	if err != nil {
 		_ = c.Close()
 		sn.handshakeErrors.Inc()
 		return nil, fmt.Errorf("handshake error: %w", err)
 	}
-	sn.id.CompareAndSwap(0, id)
 	return bc, nil
 }
 
-func (sn *storageNode) getID() uint64 {
-	// Ensure that the id is populated
-	if sn.id.Load() == 0 {
-		sn.checkHealth()
-	}
-	return sn.id.Load()
-}
-
 // storageNode is a client sending data to vmstorage node.
 type storageNode struct {
 	// isBroken is set to true if the given vmstorage node is temporarily unhealthy.
 	// In this case the data is re-routed to the remaining healthy vmstorage nodes.
 	isBroken atomic.Bool
 
+	isBufferFull atomic.Bool
+
 	// isReadOnly is set to true if the given vmstorage node is read only
 	// In this case the data is re-routed to the remaining healthy vmstorage nodes.
 	isReadOnly atomic.Bool
@@ -487,9 +456,6 @@ type storageNode struct {
 	// The total duration spent for sending data to vmstorage node.
 	// This metric is useful for determining the saturation of vminsert->vmstorage link.
 	sendDurationSeconds *metrics.FloatCounter
-
-	// id is a unique identifier for the storage node.
-	id atomic.Uint64
 }
 
 type storageNodesBucket struct {
@@ -522,6 +488,13 @@ func setStorageNodesBucket(snb *storageNodesBucket) {
 //
 // Call MustStop when the initialized vmstorage connections are no longer needed.
 func Init(addrs []string, hashSeed uint64) {
+	replicas = *replicationFactor
+	if replicas <= 0 {
+		replicas = 1
+	}
+	if replicas > len(addrs) {
+		replicas = len(addrs)
+	}
 	snb := initStorageNodes(addrs, hashSeed)
 	setStorageNodesBucket(snb)
 }
@@ -532,31 +505,14 @@ func MustStop() {
 	mustStopStorageNodes(snb)
 }
 
-// GetNodeID returns unique identifier for underlying storage nodes.
-func GetNodeID() uint64 {
-	snb := getStorageNodesBucket()
-	snIDs := make([]uint64, 0, len(snb.sns))
-	for _, sn := range snb.sns {
-		snIDs = append(snIDs, sn.getID())
-	}
-	slices.Sort(snIDs)
-	idsM := make([]byte, 0)
-	for _, id := range snIDs {
-		idsM = encoding.MarshalUint64(idsM, id)
-	}
-
-	return xxhash.Sum64(idsM)
-}
-
 func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
 	if len(addrs) == 0 {
 		logger.Panicf("BUG: addrs must be non-empty")
 	}
 	ms := metrics.NewSet()
+	nodesHash := newConsistentHash(addrs, hashSeed)
 	sns := make([]*storageNode, 0, len(addrs))
-	brokenNodes := make([]*storageNode, 0)
 	stopCh := make(chan struct{})
-	nodeIDs := make([]uint64, 0, len(addrs))
 	for _, addr := range addrs {
 		if _, _, err := net.SplitHostPort(addr); err != nil {
 			// Automatically add missing port.
@@ -602,22 +558,10 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
 			}
 			return 0
 		})
-		var nodeID uint64
-		if *usePersistentStorageNodeID {
-			nodeID = sn.getID()
-			if nodeID == 0 {
-				brokenNodes = append(brokenNodes, sn)
-				continue
-			}
-		} else {
-			nodeID = xxhash.Sum64String(addr)
-		}
-		nodeIDs = append(nodeIDs, nodeID)
 		sns = append(sns, sn)
 	}
-	nodesHash := newConsistentHash(nodeIDs, hashSeed)
 
-	maxBufSizePerStorageNode = memory.Allowed() / 8 / len(addrs)
+	maxBufSizePerStorageNode = memory.Allowed() / 8 / len(sns)
 	if maxBufSizePerStorageNode > consts.MaxInsertPacketSizeForVMInsert {
 		maxBufSizePerStorageNode = consts.MaxInsertPacketSizeForVMInsert
 	}
@@ -632,12 +576,7 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
 		wg:        &wg,
 	}
 
-	// add broken nodes to the end of the list
-	// this is needed because consistent hash slots will be populated with IDs of available
-	// storage nodes (if there are any) and indexes of consistent hash must be linked to healthy storage nodes
-	snb.sns = append(snb.sns, brokenNodes...)
-
-	for idx, sn := range snb.sns {
+	for idx, sn := range sns {
 		wg.Add(1)
 		go func(sn *storageNode, idx int) {
 			sn.run(snb, idx)
@@ -645,28 +584,6 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
 		}(sn, idx)
 	}
 
-	// Watch for node become healthy and rebuild snb.
-	for _, sn := range brokenNodes {
-		wg.Add(1)
-		sn := sn
-		go watchStorageNodeHealthy(sn, func() {
-			defer wg.Done()
-			// rebuild snb in order to update consistent hash with an ID of the healthy storage node
-			for {
-				currentSnb := getStorageNodesBucket()
-				newSnb := initStorageNodes(addrs, hashSeed)
-				if !storageNodes.CompareAndSwap(currentSnb, newSnb) {
-					// snb has been changed, so we need to stop the newSnb and try again
-					mustStopStorageNodes(newSnb)
-					continue
-				}
-				// stop previous snb and exit
-				mustStopStorageNodes(currentSnb)
-				break
-			}
-		})
-	}
-
 	return snb
 }
 
@@ -679,34 +596,6 @@ func mustStopStorageNodes(snb *storageNodesBucket) {
 	metrics.UnregisterSet(snb.ms, true)
 }
 
-// watchStorageNodeHealthy watches for sn become healthy and calls cb once it is ready.
-func watchStorageNodeHealthy(sn *storageNode, cb func()) {
-	for {
-		sn.brLock.Lock()
-		for !sn.isReady() {
-			select {
-			case <-sn.stopCh:
-				sn.brLock.Unlock()
-				return
-			default:
-				sn.brCond.Wait()
-			}
-		}
-		sn.brLock.Unlock()
-
-		select {
-		case <-sn.stopCh:
-			return
-		default:
-		}
-
-		if sn.isReady() {
-			cb()
-			return
-		}
-	}
-}
-
 // rerouteRowsToReadyStorageNodes reroutes src from not ready snSource to ready storage nodes.
 //
 // The function blocks until src is fully re-routed.
@@ -873,7 +762,7 @@ var noStorageNodesLogger = logger.WithThrottler("storageNodesUnavailable", 5*tim
 func getNotReadyStorageNodeIdxs(snb *storageNodesBucket, dst []int, snExtra *storageNode) []int {
 	dst = dst[:0]
 	for i, sn := range snb.sns {
-		if sn == snExtra || !sn.isReady() {
+		if sn == snExtra || !sn.isReady() || sn.isBufferFull.Load() {
 			dst = append(dst, i)
 		}
 	}
@@ -888,10 +777,17 @@ func (sn *storageNode) trySendBuf(buf []byte, rows int) bool {
 
 	sent := false
 	sn.brLock.Lock()
-	if sn.isReady() && len(sn.br.buf)+len(buf) <= maxBufSizePerStorageNode {
+	if !sn.isReady() {
+		return sent
+	}
+	if len(sn.br.buf)+len(buf) <= maxBufSizePerStorageNode {
 		sn.br.buf = append(sn.br.buf, buf...)
 		sn.br.rows += rows
 		sent = true
+	} else {
+		if sn.isBufferFull.CompareAndSwap(false, true) {
+			logger.Infof("node: %s transited to full", sn.dialer.Addr())
+		}
 	}
 	sn.brLock.Unlock()
 	return sent

app/vmselect/clusternative/vmselect.go

@@ -31,9 +31,7 @@ var (
 
 // NewVMSelectServer starts new server at the given addr, which serves vmselect requests from netstorage.
 func NewVMSelectServer(addr string) (*vmselectapi.Server, error) {
-	api := &vmstorageAPI{
-		nodeID: netstorage.GetNodeID(),
-	}
+	api := &vmstorageAPI{}
 	limits := vmselectapi.Limits{
 		MaxLabelNames:                 *maxTagKeys,
 		MaxLabelValues:                *maxTagValues,
@@ -47,9 +45,7 @@ func NewVMSelectServer(addr string) (*vmselectapi.Server, error) {
 }
 
 // vmstorageAPI impelements vmselectapi.API
-type vmstorageAPI struct {
-	nodeID uint64
-}
+type vmstorageAPI struct{}
 
 func (api *vmstorageAPI) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
 	denyPartialResponse := httputils.GetDenyPartialResponse(nil)
@@ -116,10 +112,6 @@ func (api *vmstorageAPI) RegisterMetricNames(qt *querytracer.Tracer, mrs []stora
 	return netstorage.RegisterMetricNames(qt, mrs, dl)
 }
 
-func (api *vmstorageAPI) GetID() uint64 {
-	return api.nodeID
-}
-
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
 	workCh chan workItem

app/vmselect/graphite/eval.go

@@ -19,11 +19,12 @@ var maxGraphiteSeries = flag.Int("search.maxGraphiteSeries", 300e3, "The maximum
 	"See https://docs.victoriametrics.com/#graphite-render-api-usage")
 
 type evalConfig struct {
-	at          *auth.Token
-	startTime   int64
-	endTime     int64
-	storageStep int64
-	deadline    searchutils.Deadline
+	at                  *auth.Token
+	startTime           int64
+	endTime             int64
+	storageStep         int64
+	denyPartialResponse bool
+	deadline            searchutils.Deadline
 
 	currentTime time.Time
 
@@ -155,8 +156,7 @@ func evalMetricExpr(ec *evalConfig, me *graphiteql.MetricExpr) (nextSeriesFunc,
 }
 
 func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr graphiteql.Expr) (nextSeriesFunc, error) {
-	denyPartialResponse := true
-	rss, _, err := netstorage.ProcessSearchQuery(nil, denyPartialResponse, sq, ec.deadline)
+	rss, _, err := netstorage.ProcessSearchQuery(nil, ec.denyPartialResponse, sq, ec.deadline)
 	if err != nil {
 		return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}

app/vmselect/graphite/render_api.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -94,19 +95,21 @@ func RenderHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
 	if err != nil {
 		return fmt.Errorf("cannot setup tag filters: %w", err)
 	}
+	denyPartialResponse := httputils.GetDenyPartialResponse(r)
 	var nextSeriess []nextSeriesFunc
 	targets := r.Form["target"]
 	for _, target := range targets {
 		ec := &evalConfig{
-			at:            at,
-			startTime:     fromTime,
-			endTime:       untilTime,
-			storageStep:   storageStep,
-			deadline:      deadline,
-			currentTime:   startTime,
-			xFilesFactor:  xFilesFactor,
-			etfs:          etfs,
-			originalQuery: target,
+			at:                  at,
+			startTime:           fromTime,
+			endTime:             untilTime,
+			storageStep:         storageStep,
+			denyPartialResponse: denyPartialResponse,
+			deadline:            deadline,
+			currentTime:         startTime,
+			xFilesFactor:        xFilesFactor,
+			etfs:                etfs,
+			originalQuery:       target,
 		}
 		nextSeries, err := execExpr(ec, target)
 		if err != nil {

app/vmselect/main.go

@@ -43,7 +43,8 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	cacheDataPath         = flag.String("cacheDataPath", "", "Path to directory for cache files. By default, the cache is not persisted.")
+	cacheDataPath = flag.String("cacheDataPath", "", "Path to directory for cache files and temporary query results. "+
+		"By default, the cache won't be persisted, and temporary query results will be placed under /tmp/searchResults. If set, the cache will be persisted under cacheDataPath/rollupResult, and temporary query results will be placed under cacheDataPath/tmp/searchResults.")
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
@@ -193,7 +194,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	startTime := time.Now()
 	defer requestDuration.UpdateDuration(startTime)
 	tracerEnabled := httputils.GetBool(r, "trace")
-	qt := querytracer.New(tracerEnabled, r.URL.Path)
+	qt := querytracer.New(tracerEnabled, "%s", r.URL.Path)
 
 	// Limit the number of concurrent queries.
 	select {

app/vmselect/netstorage/netstorage.go

@@ -2107,9 +2107,6 @@ type storageNode struct {
 
 	// The number of list tenants errors to storageNode.
 	tenantsErrors *metrics.Counter
-
-	// id is the unique identifier for the storageNode.
-	id uint64
 }
 
 func (sn *storageNode) registerMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline searchutils.Deadline) error {
@@ -2957,12 +2954,6 @@ func getStorageNodes() []*storageNode {
 	return snb.sns
 }
 
-// GetNodeID returns unique identifier of vmselect
-func GetNodeID() uint64 {
-	// Returns a 0 as persistent IDs are not intended to use with multi-level setup
-	return 0
-}
-
 // Init initializes storage nodes' connections to the given addrs.
 //
 // MustStop must be called when the initialized connections are no longer needed.
@@ -3024,7 +3015,6 @@ func newStorageNode(ms *metrics.Set, group *storageNodesGroup, addr string) *sto
 	sn := &storageNode{
 		group:    group,
 		connPool: connPool,
-		id:       connPool.GetTargetNodeID(),
 
 		concurrentQueries: ms.NewCounter(fmt.Sprintf(`vm_concurrent_queries{name="vmselect", addr=%q}`, addr)),
 

app/vmselect/netstorage/tmp_blocks_file.go

@@ -140,7 +140,7 @@ func (tbf *tmpBlocksFile) Finalize() error {
 	tbf.buf = tbf.buf[:0]
 	r := fs.NewReaderAt(tbf.f)
 
-	// Hint the OS that the file is read almost sequentiallly.
+	// Hint the OS that the file is read almost sequentially.
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)

app/vmselect/promql/transform.go

@@ -1052,7 +1052,7 @@ func fixBrokenBuckets(i int, xss []leTimeseries) {
 
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
 	// or are bigger than the lower bucket values.
-	vNext := xss[0].ts.Values[0]
+	vNext := xss[0].ts.Values[i]
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
 		if math.IsNaN(v) || vNext > v {

app/vmselect/promql/transform_test.go

@@ -39,6 +39,30 @@ func TestFixBrokenBuckets(t *testing.T) {
 	f([]float64{5, 10, 4, 3}, []float64{5, 10, 10, 10})
 }
 
+func TestFixBrokenBucketsMultipleValues(t *testing.T) {
+	f := func(values, expectedResult [][]float64) {
+		t.Helper()
+		xss := make([]leTimeseries, len(values))
+		for i, v := range values {
+
+			xss[i].ts = &timeseries{
+				Values: v,
+			}
+		}
+		for i := range len(values) - 1 {
+			fixBrokenBuckets(i, xss)
+		}
+		result := make([][]float64, len(values))
+		for i, xs := range xss {
+			result[i] = xs.ts.Values
+		}
+		if !reflect.DeepEqual(result, expectedResult) {
+			t.Fatalf("unexpected result for values=%v\ngot\n%v\nwant\n%v", values, result, expectedResult)
+		}
+	}
+	f([][]float64{{10, 1}, {11, 2}, {13, 3}}, [][]float64{{10, 1}, {11, 2}, {13, 3}})
+}
+
 func TestVmrangeBucketsToLE(t *testing.T) {
 	f := func(buckets, bucketsExpected string) {
 		t.Helper()

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
     "main.css": "./static/css/main.fce049bf.css",
-    "main.js": "./static/js/main.36983a8a.js",
+    "main.js": "./static/js/main.36501ae8.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
     "static/css/main.fce049bf.css",
-    "static/js/main.36983a8a.js"
+    "static/js/main.36501ae8.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.36983a8a.js"></script><link href="./static/css/main.fce049bf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.36501ae8.js"></script><link href="./static/css/main.fce049bf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmstorage/main.go

@@ -111,8 +111,8 @@ func main() {
 	blocksCount := tm.SmallBlocksCount + tm.BigBlocksCount
 	rowsCount := tm.SmallRowsCount + tm.BigRowsCount
 	sizeBytes := tm.SmallSizeBytes + tm.BigSizeBytes
-	logger.Infof("successfully opened storage %q (node ID: %d) in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
-		*storageDataPath, strg.GetID(), time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
+	logger.Infof("successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
+		*storageDataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
 
 	// register storage metrics
 	storageMetrics := metrics.NewSet()

app/vmstorage/servers/vminsert.go

@@ -101,7 +101,7 @@ func (s *VMInsertServer) run() {
 			// There is no need in response compression, since
 			// vmstorage sends only small packets to vminsert.
 			compressionLevel := 0
-			bc, err := handshake.VMInsertServer(c, compressionLevel, s.storage.GetID())
+			bc, err := handshake.VMInsertServer(c, compressionLevel)
 			if err != nil {
 				if s.isStopping() {
 					// c is stopped inside VMInsertServer.MustStop

app/vmstorage/servers/vmselect.go

@@ -195,10 +195,6 @@ func (api *vmstorageAPI) setupTfss(qt *querytracer.Tracer, sq *storage.SearchQue
 	return tfss, nil
 }
 
-func (api *vmstorageAPI) GetID() uint64 {
-	return api.s.GetID()
-}
-
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
 	sr storage.Search

app/vmui/packages/vmui/src/api/types.ts

@@ -38,6 +38,7 @@ export interface Logs {
 export interface LogHits {
   timestamps: string[];
   values: number[];
+  total?: number;
   fields: {
     [key: string]: string;
   };

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsChart.tsx

@@ -1,32 +1,66 @@
-import React, { FC, useRef, useState } from "preact/compat";
+import React, { FC, useMemo, useRef, useState } from "preact/compat";
 import "./style.scss";
 import "uplot/dist/uPlot.min.css";
 import useElementSize from "../../../hooks/useElementSize";
 import uPlot, { AlignedData } from "uplot";
 import { useEffect } from "react";
 import useBarHitsOptions from "./hooks/useBarHitsOptions";
-import TooltipBarHitsChart from "./TooltipBarHitsChart";
+import BarHitsTooltip from "./BarHitsTooltip/BarHitsTooltip";
 import { TimeParams } from "../../../types";
 import usePlotScale from "../../../hooks/uplot/usePlotScale";
 import useReadyChart from "../../../hooks/uplot/useReadyChart";
 import useZoomChart from "../../../hooks/uplot/useZoomChart";
 import classNames from "classnames";
+import { LogHits } from "../../../api/types";
+import { addSeries, delSeries, setBand } from "../../../utils/uplot";
+import { GraphOptions, GRAPH_STYLES } from "./types";
+import BarHitsOptions from "./BarHitsOptions/BarHitsOptions";
+import stack from "../../../utils/uplot/stack";
+import BarHitsLegend from "./BarHitsLegend/BarHitsLegend";
 
 interface Props {
+  logHits: LogHits[];
   data: AlignedData;
   period: TimeParams;
   setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
+  onApplyFilter: (value: string) => void;
 }
-
-const BarHitsChart: FC<Props> = ({ data, period, setPeriod }) => {
+const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onApplyFilter }) => {
   const [containerRef, containerSize] = useElementSize();
   const uPlotRef = useRef<HTMLDivElement>(null);
   const [uPlotInst, setUPlotInst] = useState<uPlot>();
+  const [graphOptions, setGraphOptions] = useState<GraphOptions>({
+    graphStyle: GRAPH_STYLES.LINE_STEPPED,
+    stacked: false,
+    fill: false,
+  });
 
   const { xRange, setPlotScale } = usePlotScale({ period, setPeriod });
   const { onReadyChart, isPanning } = useReadyChart(setPlotScale);
   useZoomChart({ uPlotInst, xRange, setPlotScale });
-  const { options, focusDataIdx } = useBarHitsOptions({ xRange, containerSize, onReadyChart, setPlotScale });
+
+  const { data, bands } = useMemo(() => {
+    return graphOptions.stacked ? stack(_data, () => false) : { data: _data, bands: [] };
+  }, [graphOptions, _data]);
+
+  const { options, series, focusDataIdx } = useBarHitsOptions({
+    data,
+    logHits,
+    bands,
+    xRange,
+    containerSize,
+    onReadyChart,
+    setPlotScale,
+    graphOptions
+  });
+
+  useEffect(() => {
+    if (!uPlotInst) return;
+    delSeries(uPlotInst);
+    addSeries(uPlotInst, series, true);
+    setBand(uPlotInst, series);
+    uPlotInst.redraw();
+  }, [series]);
 
   useEffect(() => {
     if (!uPlotRef.current) return;
@@ -54,21 +88,31 @@ const BarHitsChart: FC<Props> = ({ data, period, setPeriod }) => {
   }, [data]);
 
   return (
-    <div
-      className={classNames({
-        "vm-bar-hits-chart": true,
-        "vm-bar-hits-chart_panning": isPanning
-      })}
-      ref={containerRef}
-    >
+    <div className="vm-bar-hits-chart__wrapper">
       <div
-        className="vm-line-chart__u-plot"
-        ref={uPlotRef}
-      />
-      <TooltipBarHitsChart
-        uPlotInst={uPlotInst}
-        focusDataIdx={focusDataIdx}
-      />
+        className={classNames({
+          "vm-bar-hits-chart": true,
+          "vm-bar-hits-chart_panning": isPanning
+        })}
+        ref={containerRef}
+      >
+        <div
+          className="vm-line-chart__u-plot"
+          ref={uPlotRef}
+        />
+        <BarHitsTooltip
+          uPlotInst={uPlotInst}
+          data={_data}
+          focusDataIdx={focusDataIdx}
+        />
+      </div>
+      <BarHitsOptions onChange={setGraphOptions}/>
+      {uPlotInst && (
+        <BarHitsLegend
+          uPlotInst={uPlotInst}
+          onApplyFilter={onApplyFilter}
+        />
+      )}
     </div>
   );
 };

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/BarHitsLegend.tsx

@@ -0,0 +1,68 @@
+import React, { FC, useCallback, useEffect, useState } from "preact/compat";
+import uPlot, { Series } from "uplot";
+import "./style.scss";
+import "../../Line/Legend/style.scss";
+import classNames from "classnames";
+import { MouseEvent } from "react";
+import { isMacOs } from "../../../../utils/detect-device";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+
+interface Props {
+  uPlotInst: uPlot;
+  onApplyFilter: (value: string) => void;
+}
+
+const BarHitsLegend: FC<Props> = ({ uPlotInst, onApplyFilter }) => {
+  const [series, setSeries] = useState<Series[]>([]);
+
+  const updateSeries = useCallback(() => {
+    const series = uPlotInst.series.filter(s => s.scale !== "x");
+    setSeries(series);
+  }, [uPlotInst]);
+
+  const handleClick = (target: Series) => (e: MouseEvent<HTMLDivElement>) => {
+    const metaKey = e.metaKey || e.ctrlKey;
+    if (!metaKey) {
+      target.show = !target.show;
+    } else {
+      onApplyFilter(target.label || "");
+    }
+
+    updateSeries();
+    uPlotInst.redraw();
+  };
+
+  useEffect(updateSeries, [uPlotInst]);
+
+  return (
+    <div className="vm-bar-hits-legend">
+      {series.map(s => (
+        <Tooltip
+          key={s.label}
+          title={(
+            <ul className="vm-bar-hits-legend-info">
+              <li>Click to {s.show ? "hide" : "show"} the _stream.</li>
+              <li>{isMacOs() ? "Cmd" : "Ctrl"} + Click to filter by the _stream.</li>
+            </ul>
+          )}
+        >
+          <div
+            className={classNames({
+              "vm-bar-hits-legend-item": true,
+              "vm-bar-hits-legend-item_hide": !s.show,
+            })}
+            onClick={handleClick(s)}
+          >
+            <div
+              className="vm-bar-hits-legend-item__marker"
+              style={{ backgroundColor: `${(s?.stroke as () => string)?.()}` }}
+            />
+            <div>{s.label}</div>
+          </div>
+        </Tooltip>
+      ))}
+    </div>
+  );
+};
+
+export default BarHitsLegend;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/style.scss

@@ -0,0 +1,35 @@
+@use "src/styles/variables" as *;
+
+.vm-bar-hits-legend {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0;
+  padding: 0 $padding-small $padding-small;
+
+  &-item {
+    display: grid;
+    grid-template-columns: auto 1fr;
+    align-items: center;
+    gap: 4px;
+    font-size: 12px;
+    padding: $padding-small;
+    border-radius: $border-radius-small;
+    cursor: pointer;
+    transition: 0.2s;
+
+    &:hover {
+      background-color: rgba(0, 0, 0, 0.05);
+    }
+
+    &_hide {
+      text-decoration: line-through;
+      opacity: 0.5;
+    }
+
+    &__marker {
+      width: 14px;
+      height: 14px;
+      border: $color-background-block;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/BarHitsOptions.tsx

@@ -0,0 +1,116 @@
+import React, { FC, useEffect, useMemo, useRef } from "preact/compat";
+import { GraphOptions, GRAPH_STYLES } from "../types";
+import Switch from "../../../Main/Switch/Switch";
+import "./style.scss";
+import useStateSearchParams from "../../../../hooks/useStateSearchParams";
+import { useSearchParams } from "react-router-dom";
+import Button from "../../../Main/Button/Button";
+import classNames from "classnames";
+import { SettingsIcon } from "../../../Main/Icons";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import Popper from "../../../Main/Popper/Popper";
+import useBoolean from "../../../../hooks/useBoolean";
+
+interface Props {
+  onChange: (options: GraphOptions) => void;
+}
+
+const BarHitsOptions: FC<Props> = ({ onChange }) => {
+  const [searchParams, setSearchParams] = useSearchParams();
+  const optionsButtonRef = useRef<HTMLDivElement>(null);
+  const {
+    value: openOptions,
+    toggle: toggleOpenOptions,
+    setFalse: handleCloseOptions,
+  } = useBoolean(false);
+
+  const [graphStyle, setGraphStyle] = useStateSearchParams(GRAPH_STYLES.LINE_STEPPED, "graph");
+  const [stacked, setStacked] = useStateSearchParams(false, "stacked");
+  const [fill, setFill] = useStateSearchParams(false, "fill");
+
+  const options: GraphOptions = useMemo(() => ({
+    graphStyle,
+    stacked,
+    fill,
+  }), [graphStyle, stacked, fill]);
+
+  const handleChangeGraphStyle = (val: string) => () => {
+    setGraphStyle(val as GRAPH_STYLES);
+    searchParams.set("graph", val);
+    setSearchParams(searchParams);
+  };
+
+  const handleChangeFill = (val: boolean) => {
+    setFill(val);
+    val ? searchParams.set("fill", "true") : searchParams.delete("fill");
+    setSearchParams(searchParams);
+  };
+
+  const handleChangeStacked = (val: boolean) => {
+    setStacked(val);
+    val ? searchParams.set("stacked", "true") : searchParams.delete("stacked");
+    setSearchParams(searchParams);
+  };
+
+  useEffect(() => {
+    onChange(options);
+  }, [options]);
+
+  return (
+    <div
+      className="vm-bar-hits-options"
+      ref={optionsButtonRef}
+    >
+      <Tooltip title="Graph settings">
+        <Button
+          variant="text"
+          color="primary"
+          startIcon={<SettingsIcon/>}
+          onClick={toggleOpenOptions}
+          ariaLabel="settings"
+        />
+      </Tooltip>
+      <Popper
+        open={openOptions}
+        placement="bottom-right"
+        onClose={handleCloseOptions}
+        buttonRef={optionsButtonRef}
+        title={"Graph settings"}
+      >
+        <div className="vm-bar-hits-options-settings">
+          <div className="vm-bar-hits-options-settings-item vm-bar-hits-options-settings-item_list">
+            <p className="vm-bar-hits-options-settings-item__title">Graph style:</p>
+            {Object.values(GRAPH_STYLES).map(style => (
+              <div
+                key={style}
+                className={classNames({
+                  "vm-list-item": true,
+                  "vm-list-item_active": graphStyle === style,
+                })}
+                onClick={handleChangeGraphStyle(style)}
+              >
+                {style}
+              </div>
+            ))}
+          </div>
+          <div className="vm-bar-hits-options-settings-item">
+            <Switch
+              label={"Stacked"}
+              value={stacked}
+              onChange={handleChangeStacked}
+            />
+          </div>
+          <div className="vm-bar-hits-options-settings-item">
+            <Switch
+              label={"Fill"}
+              value={fill}
+              onChange={handleChangeFill}
+            />
+          </div>
+        </div>
+      </Popper>
+    </div>
+  );
+};
+
+export default BarHitsOptions;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsOptions/style.scss

@@ -0,0 +1,35 @@
+@use "src/styles/variables" as *;
+
+.vm-bar-hits-options {
+  position: absolute;
+  top: $padding-small;
+  right: $padding-small;
+  z-index: 2;
+
+  &-settings {
+    display: grid;
+    align-items: flex-start;
+    gap: $padding-global;
+    min-width: 200px;
+
+    &-item {
+      border-bottom: $border-divider;
+      padding: 0 $padding-global $padding-global;
+
+      &_list {
+        padding: 0;
+      }
+
+      &__title {
+        font-size: $font-size-small;
+        color: $color-text-secondary;
+        padding: 0 $padding-small $padding-small;
+      }
+
+      &:last-child {
+        border-bottom: none;
+      }
+
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/BarHitsTooltip.tsx

@@ -0,0 +1,125 @@
+import React, { FC, useMemo, useRef } from "preact/compat";
+import uPlot, { AlignedData } from "uplot";
+import dayjs from "dayjs";
+import { DATE_TIME_FORMAT } from "../../../../constants/date";
+import classNames from "classnames";
+import "./style.scss";
+import "../../ChartTooltip/style.scss";
+
+interface Props {
+  data: AlignedData;
+  uPlotInst?: uPlot;
+  focusDataIdx: number;
+}
+
+const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
+  const tooltipRef = useRef<HTMLDivElement>(null);
+
+  const tooltipData = useMemo(() => {
+    const series = uPlotInst?.series || [];
+    const [time, ...values] = data.map((d) => d[focusDataIdx] || 0);
+
+    const tooltipItems = values.map((value, i) => {
+      const targetSeries = series[i + 1];
+      const stroke = (targetSeries?.stroke as () => string)?.();
+      const label = targetSeries?.label || "other";
+      const show = targetSeries?.show;
+      return {
+        label,
+        stroke,
+        value,
+        show
+      };
+    }).filter(item => item.value > 0 && item.show).sort((a, b) => b.value - a.value);
+
+    const point = {
+      top: tooltipItems[0] ? uPlotInst?.valToPos?.(tooltipItems[0].value, "y") || 0 : 0,
+      left: uPlotInst?.valToPos?.(time, "x") || 0,
+    };
+
+    return {
+      point,
+      values: tooltipItems,
+      total: tooltipItems.reduce((acc, item) => acc + item.value, 0),
+      timestamp: dayjs(time * 1000).tz().format(DATE_TIME_FORMAT),
+    };
+  }, [focusDataIdx, uPlotInst, data]);
+
+  const tooltipPosition = useMemo(() => {
+    if (!uPlotInst || !tooltipData.total || !tooltipRef.current) return;
+
+    const { top, left } = tooltipData.point;
+    const uPlotPosition = {
+      left: parseFloat(uPlotInst.over.style.left),
+      top: parseFloat(uPlotInst.over.style.top)
+    };
+
+    const {
+      width: uPlotWidth,
+      height: uPlotHeight
+    } = uPlotInst.over.getBoundingClientRect();
+
+    const {
+      width: tooltipWidth,
+      height: tooltipHeight
+    } = tooltipRef.current.getBoundingClientRect();
+
+    const margin = 50;
+    const overflowX = left + tooltipWidth >= uPlotWidth ? tooltipWidth + (2 * margin) : 0;
+    const overflowY = top + tooltipHeight >= uPlotHeight ? tooltipHeight + (2 * margin) : 0;
+
+    const position = {
+      top: top + uPlotPosition.top + margin - overflowY,
+      left: left + uPlotPosition.left + margin - overflowX
+    };
+
+    if (position.left < 0) position.left = 20;
+    if (position.top < 0) position.top = 20;
+
+    return position;
+  }, [tooltipData, uPlotInst, tooltipRef.current]);
+
+  return (
+    <div
+      className={classNames({
+        "vm-chart-tooltip": true,
+        "vm-chart-tooltip_hits": true,
+        "vm-bar-hits-tooltip": true,
+        "vm-bar-hits-tooltip_visible": focusDataIdx !== -1 && tooltipData.values.length
+      })}
+      ref={tooltipRef}
+      style={tooltipPosition}
+    >
+      <div>
+        {tooltipData.values.map((item, i) => (
+          <div
+            className="vm-chart-tooltip-data"
+            key={i}
+          >
+            <span
+              className="vm-chart-tooltip-data__marker"
+              style={{ background: item.stroke }}
+            />
+            <p>
+              {item.label}: <b>{item.value}</b>
+            </p>
+          </div>
+        ))}
+      </div>
+      {tooltipData.values.length > 1 && (
+        <div className="vm-chart-tooltip-data">
+          <p>
+              Total records: <b>{tooltipData.total}</b>
+          </p>
+        </div>
+      )}
+      <div className="vm-chart-tooltip-header">
+        <div className="vm-chart-tooltip-header__title">
+          {tooltipData.timestamp}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default BarHitsTooltip;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/style.scss

@@ -0,0 +1,12 @@
+@use "src/styles/variables" as *;
+
+.vm-bar-hits-tooltip {
+  opacity: 0;
+  pointer-events: none;
+  gap: $padding-small;
+
+  &_visible {
+    opacity: 1;
+    pointer-events: auto;
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/TooltipBarHitsChart.tsx

@@ -1,89 +0,0 @@
-import React, { FC, useMemo, useRef } from "preact/compat";
-import uPlot from "uplot";
-import dayjs from "dayjs";
-import { DATE_TIME_FORMAT } from "../../../constants/date";
-import classNames from "classnames";
-import "./style.scss";
-import "../../../components/Chart/ChartTooltip/style.scss";
-
-interface Props {
-  uPlotInst?: uPlot;
-  focusDataIdx: number
-}
-
-const TooltipBarHitsChart: FC<Props> = ({ focusDataIdx, uPlotInst }) => {
-  const tooltipRef = useRef<HTMLDivElement>(null);
-
-  const tooltipData = useMemo(() => {
-    const value = uPlotInst?.data?.[1]?.[focusDataIdx];
-    const timestamp = uPlotInst?.data?.[0]?.[focusDataIdx] || 0;
-    const top = uPlotInst?.valToPos?.((value || 0), "y") || 0;
-    const left = uPlotInst?.valToPos?.(timestamp, "x") || 0;
-
-    return {
-      point: { top, left },
-      value,
-      timestamp: dayjs(timestamp * 1000).tz().format(DATE_TIME_FORMAT),
-    };
-  }, [focusDataIdx, uPlotInst]);
-
-  const tooltipPosition = useMemo(() => {
-    if (!uPlotInst || !tooltipData.value || !tooltipRef.current) return;
-
-    const { top, left } = tooltipData.point;
-    const uPlotPosition = {
-      left: parseFloat(uPlotInst.over.style.left),
-      top: parseFloat(uPlotInst.over.style.top)
-    };
-
-    const {
-      width: uPlotWidth,
-      height: uPlotHeight
-    } = uPlotInst.over.getBoundingClientRect();
-
-    const {
-      width: tooltipWidth,
-      height: tooltipHeight
-    } = tooltipRef.current.getBoundingClientRect();
-
-    const margin = 10;
-    const overflowX = left + tooltipWidth >= uPlotWidth ? tooltipWidth + (2 * margin) : 0;
-    const overflowY = top + tooltipHeight >= uPlotHeight ? tooltipHeight + (2 * margin) : 0;
-
-    const position = {
-      top: top + uPlotPosition.top + margin - overflowY,
-      left: left + uPlotPosition.left + margin - overflowX
-    };
-
-    if (position.left < 0) position.left = 20;
-    if (position.top < 0) position.top = 20;
-
-    return position;
-  }, [tooltipData, uPlotInst, tooltipRef.current]);
-
-  return (
-    <div
-      className={classNames({
-        "vm-chart-tooltip": true,
-        "vm-bar-hits-chart-tooltip": true,
-        "vm-bar-hits-chart-tooltip_visible": focusDataIdx !== -1
-      })}
-      ref={tooltipRef}
-      style={tooltipPosition}
-    >
-      <div className="vm-chart-tooltip-data">
-        Count of records:
-        <p className="vm-chart-tooltip-data__value">
-          <b>{tooltipData.value}</b>
-        </p>
-      </div>
-      <div className="vm-chart-tooltip-header">
-        <div className="vm-chart-tooltip-header__title">
-          {tooltipData.timestamp}
-        </div>
-      </div>
-    </div>
-  );
-};
-
-export default TooltipBarHitsChart;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/hooks/useBarHitsOptions.ts

@@ -2,42 +2,81 @@ import { useMemo, useState } from "preact/compat";
 import { getAxes, handleDestroy, setSelect } from "../../../../utils/uplot";
 import dayjs from "dayjs";
 import { dateFromSeconds, formatDateForNativeInput } from "../../../../utils/time";
-import uPlot, { Options } from "uplot";
+import uPlot, { AlignedData, Band, Options, Series } from "uplot";
 import { getCssVariable } from "../../../../utils/theme";
-import { barPaths } from "../../../../utils/uplot/bars";
 import { useAppState } from "../../../../state/common/StateContext";
 import { MinMax, SetMinMax } from "../../../../types";
+import { LogHits } from "../../../../api/types";
+import getSeriesPaths from "../../../../utils/uplot/paths";
+import { GraphOptions, GRAPH_STYLES } from "../types";
+
+const seriesColors = [
+  "color-log-hits-bar-1",
+  "color-log-hits-bar-2",
+  "color-log-hits-bar-3",
+  "color-log-hits-bar-4",
+  "color-log-hits-bar-5",
+];
+
+const strokeWidth = {
+  [GRAPH_STYLES.BAR]: 0.8,
+  [GRAPH_STYLES.LINE_STEPPED]: 1.2,
+  [GRAPH_STYLES.LINE]: 1.2,
+  [GRAPH_STYLES.POINTS]: 0,
+};
 
 interface UseGetBarHitsOptionsArgs {
+  data: AlignedData;
+  logHits: LogHits[];
   xRange: MinMax;
+  bands?: Band[];
   containerSize: { width: number, height: number };
   setPlotScale: SetMinMax;
   onReadyChart: (u: uPlot) => void;
+  graphOptions: GraphOptions;
 }
 
-const useBarHitsOptions = ({ xRange, containerSize, onReadyChart, setPlotScale }: UseGetBarHitsOptionsArgs) => {
+const useBarHitsOptions = ({
+  data,
+  logHits,
+  xRange,
+  bands,
+  containerSize,
+  onReadyChart,
+  setPlotScale,
+  graphOptions
+}: UseGetBarHitsOptionsArgs) => {
   const { isDarkTheme } = useAppState();
 
   const [focusDataIdx, setFocusDataIdx] = useState(-1);
 
-  const series = useMemo(() => [
-    {},
-    {
-      label: "y",
-      width: 1,
-      stroke: getCssVariable("color-log-hits-bar"),
-      fill: getCssVariable("color-log-hits-bar"),
-      paths: barPaths,
-    }
-  ], [isDarkTheme]);
-
   const setCursor = (u: uPlot) => {
     const dataIdx = u.cursor.idx ?? -1;
     setFocusDataIdx(dataIdx);
   };
 
+  const series: Series[] = useMemo(() => {
+    let colorN = 0;
+    return data.map((_d, i) => {
+      if (i === 0) return {}; // 0 index is xAxis(timestamps)
+      const fields = Object.values(logHits?.[i - 1]?.fields || {});
+      const label = fields.map((value) => value || "\"\"").join(", ");
+      const color = getCssVariable(label ? seriesColors[colorN] : "color-log-hits-bar-0");
+      if (label) colorN++;
+      return {
+        label: label || "other",
+        width: strokeWidth[graphOptions.graphStyle],
+        spanGaps: true,
+        stroke: color,
+        fill: graphOptions.fill ? color + "80" : "",
+        paths: getSeriesPaths(graphOptions.graphStyle),
+      };
+    });
+  }, [isDarkTheme, data, graphOptions]);
+
   const options: Options = useMemo(() => ({
     series,
+    bands,
     width: containerSize.width || (window.innerWidth / 2),
     height: containerSize.height || 200,
     cursor: {
@@ -55,6 +94,7 @@ const useBarHitsOptions = ({ xRange, containerSize, onReadyChart, setPlotScale }
       }
     },
     hooks: {
+      drawSeries: [],
       ready: [onReadyChart],
       setCursor: [setCursor],
       setSelect: [setSelect(setPlotScale)],
@@ -63,10 +103,11 @@ const useBarHitsOptions = ({ xRange, containerSize, onReadyChart, setPlotScale }
     legend: { show: false },
     axes: getAxes([{}, { scale: "y" }]),
     tzDate: ts => dayjs(formatDateForNativeInput(dateFromSeconds(ts))).local().toDate(),
-  }), [isDarkTheme]);
+  }), [isDarkTheme, series, bands]);
 
   return {
     options,
+    series,
     focusDataIdx,
   };
 };

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/style.scss

@@ -1,22 +1,18 @@
 @use "src/styles/variables" as *;
 
 .vm-bar-hits-chart {
-  height: 100%;
+  position: relative;
   width: 100%;
+  height: 200px;
+
+  &__wrapper {
+    display: flex;
+    flex-direction: column;
+    width: 100%;
+    height: 100%;
+  }
 
   &_panning {
     pointer-events: none;
   }
-
-  &-tooltip {
-    opacity: 0;
-    pointer-events: none;
-    width: 240px;
-    gap: $padding-small;
-
-    &_visible {
-      opacity: 1;
-      pointer-events: auto;
-    }
-  }
 }

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/types.ts

@@ -0,0 +1,12 @@
+export enum GRAPH_STYLES {
+  BAR = "Bars",
+  LINE = "Lines",
+  LINE_STEPPED = "Stepped lines",
+  POINTS = "Points",
+}
+
+export interface GraphOptions {
+  graphStyle: GRAPH_STYLES;
+  stacked: boolean;
+  fill: boolean;
+}

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -25,6 +25,12 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
   user-select: text;
   pointer-events: none;
 
+  &_hits {
+    white-space: pre-wrap;
+    word-break: break-all;
+    width: auto;
+  }
+
   &_sticky {
     pointer-events: auto;
     z-index: 99;
@@ -74,10 +80,22 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     justify-content: flex-start;
     gap: $padding-small;
 
+    &_margin-bottom {
+      margin-bottom: $padding-global;
+    }
+
+    &_margin-top {
+      margin-top: $padding-global;
+    }
+
     &__marker {
       width: $font-size;
       height: $font-size;
       border: 1px solid rgba($color-white, 0.5);
+
+      &_tranparent {
+        opacity: 0;
+      }
     }
 
     &__value {

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -13,6 +13,7 @@ import ThemeControl from "../ThemeControl/ThemeControl";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useBoolean from "../../../hooks/useBoolean";
 import { AppType } from "../../../types/appType";
+import SwitchMarkdownParsing from "../LogsSettings/MarkdownParsing/SwitchMarkdownParsing";
 
 const title = "Settings";
 
@@ -60,6 +61,10 @@ const GlobalSettings: FC = () => {
         onClose={handleClose}
       />
     },
+    {
+      show: isLogsApp,
+      component: <SwitchMarkdownParsing/>
+    },
     {
       show: true,
       component: <Timezones ref={timezoneSettingRef}/>

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsFields.tsx

@@ -0,0 +1,150 @@
+import React, { FC, useRef } from "preact/compat";
+import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
+import { ArrowDownIcon, QuestionIcon, StorageIcon } from "../../../Main/Icons";
+import Button from "../../../Main/Button/Button";
+import "./style.scss";
+import "../../TimeRangeSettings/ExecutionControls/style.scss";
+import classNames from "classnames";
+import Popper from "../../../Main/Popper/Popper";
+import { getAppModeEnable } from "../../../../utils/app-mode";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import TextField from "../../../Main/TextField/TextField";
+import useBoolean from "../../../../hooks/useBoolean";
+import useStateSearchParams from "../../../../hooks/useStateSearchParams";
+import { useSearchParams } from "react-router-dom";
+import { useEffect } from "react";
+
+const TenantsFields: FC = () => {
+  const appModeEnable = getAppModeEnable();
+  const { isMobile } = useDeviceDetect();
+  const timeDispatch = useTimeDispatch();
+
+  const [searchParams, setSearchParams] = useSearchParams();
+  const [accountID, setAccountID] = useStateSearchParams("0", "accountID");
+  const [projectID, setProjectID] = useStateSearchParams("0", "projectID");
+  const formattedTenant = `${accountID}:${projectID}`;
+
+  const buttonRef = useRef<HTMLDivElement>(null);
+
+  const {
+    value: openPopup,
+    toggle: toggleOpenPopup,
+    setFalse: handleClosePopup,
+  } = useBoolean(false);
+
+  const applyChanges = () => {
+    searchParams.set("accountID", accountID);
+    searchParams.set("projectID", projectID);
+    setSearchParams(searchParams);
+    handleClosePopup();
+    timeDispatch({ type: "RUN_QUERY" });
+  };
+
+  const handleReset = () => {
+    setAccountID(searchParams.get("accountID") || "0");
+    setProjectID(searchParams.get("projectID") || "0");
+  };
+
+  useEffect(() => {
+    if (openPopup) return;
+    handleReset();
+  }, [openPopup]);
+
+  return (
+    <div className="vm-tenant-input">
+      <Tooltip title="Define Tenant ID if you need request to another storage">
+        <div ref={buttonRef}>
+          {isMobile ? (
+            <div
+              className="vm-mobile-option"
+              onClick={toggleOpenPopup}
+            >
+              <span className="vm-mobile-option__icon"><StorageIcon/></span>
+              <div className="vm-mobile-option-text">
+                <span className="vm-mobile-option-text__label">Tenant ID</span>
+                <span className="vm-mobile-option-text__value">{formattedTenant}</span>
+              </div>
+              <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
+            </div>
+          ) : (
+            <Button
+              className={appModeEnable ? "" : "vm-header-button"}
+              variant="contained"
+              color="primary"
+              fullWidth
+              startIcon={<StorageIcon/>}
+              endIcon={(
+                <div
+                  className={classNames({
+                    "vm-execution-controls-buttons__arrow": true,
+                    "vm-execution-controls-buttons__arrow_open": openPopup,
+                  })}
+                >
+                  <ArrowDownIcon/>
+                </div>
+              )}
+              onClick={toggleOpenPopup}
+            >
+              {formattedTenant}
+            </Button>
+          )}
+        </div>
+      </Tooltip>
+      <Popper
+        open={openPopup}
+        placement="bottom-right"
+        onClose={handleClosePopup}
+        buttonRef={buttonRef}
+        title={isMobile ? "Define Tenant ID" : undefined}
+      >
+        <div
+          className={classNames({
+            "vm-list vm-tenant-input-list": true,
+            "vm-list vm-tenant-input-list_mobile": isMobile,
+            "vm-tenant-input-list_inline": true,
+          })}
+        >
+          <TextField
+            autofocus
+            label="accountID"
+            value={accountID}
+            onChange={setAccountID}
+            type="number"
+          />
+          <TextField
+            autofocus
+            label="projectID"
+            value={projectID}
+            onChange={setProjectID}
+            type="number"
+          />
+          <div className="vm-tenant-input-list__buttons">
+            <Tooltip title="Multitenancy in VictoriaLogs documentation">
+              <a
+                href="https://docs.victoriametrics.com/victorialogs/#multitenancy"
+                target="_blank"
+                rel="help noreferrer"
+              >
+                <Button
+                  variant="text"
+                  color="gray"
+                  startIcon={<QuestionIcon/>}
+                />
+              </a>
+            </Tooltip>
+            <Button
+              variant="contained"
+              color="primary"
+              onClick={applyChanges}
+            >
+              Apply
+            </Button>
+          </div>
+        </div>
+      </Popper>
+    </div>
+  );
+};
+
+export default TenantsFields;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/style.scss

@@ -17,11 +17,23 @@
       padding: 0 $padding-global $padding-small;
     }
 
+    &_inline {
+      display: grid;
+      gap: calc($padding-small/2);
+      padding: $padding-global;
+    }
+
     &__search {
       position: sticky;
       top: 0;
       padding: $padding-small $padding-global;
       background-color: $color-background-block;
     }
+
+    &__buttons {
+      display: flex;
+      justify-content: space-between;
+      gap: $padding-small;
+    }
   }
 }

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/style.scss

@@ -4,7 +4,7 @@
   display: flex;
   flex-direction: column;
   align-items: center;
-  gap: $padding-medium;
+  gap: $padding-large;
   width: 600px;
   padding-bottom: $padding-medium;
 
@@ -39,6 +39,13 @@
     margin-bottom: $padding-global;
   }
 
+  &__info {
+    padding-top: $padding-small;
+    font-size: $font-size-small;
+    color: $color-text-secondary;
+    line-height: 130%;
+  }
+
   &-url {
     display: flex;
     align-items: flex-start;

app/vmui/packages/vmui/src/components/Configurators/LogsSettings/MarkdownParsing/SwitchMarkdownParsing.tsx

@@ -0,0 +1,35 @@
+import React, { FC } from "preact/compat";
+import Switch from "../../../Main/Switch/Switch";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import { useLogsDispatch, useLogsState } from "../../../../state/logsPanel/LogsStateContext";
+
+const SwitchMarkdownParsing: FC = () => {
+  const { isMobile } = useDeviceDetect();
+  const { markdownParsing } = useLogsState();
+  const dispatch = useLogsDispatch();
+
+
+  const handleChangeMarkdownParsing = (val: boolean) => {
+    dispatch({ type: "SET_MARKDOWN_PARSING", payload: val });
+  };
+
+  return (
+    <div>
+      <div className="vm-server-configurator__title">
+        Markdown Parsing for Logs
+      </div>
+      <Switch
+        label={markdownParsing ? "Disable markdown parsing" : "Enable markdown parsing"}
+        value={markdownParsing}
+        onChange={handleChangeMarkdownParsing}
+        fullWidth={isMobile}
+      />
+      <div className="vm-server-configurator__info">
+        Toggle this switch to enable or disable the Markdown formatting for log entries.
+        Enabling this will parse log texts to Markdown.
+      </div>
+    </div>
+  );
+};
+
+export default SwitchMarkdownParsing;

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -34,14 +34,25 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   }, [value, caretPosition]);
 
   const exprLastPart = useMemo(() => {
-    const parts = values.beforeCursor.split("}");
+    const regexpSplit = /\s(or|and|unless|default|ifnot|if|group_left|group_right)\s|}|\+|\|-|\*|\/|\^/i;
+    const parts = values.beforeCursor.split(regexpSplit);
     return parts[parts.length - 1];
   }, [values]);
 
   const metric = useMemo(() => {
-    const regexp = /\b[^{}(),\s]+(?={|$)/g;
-    const match = exprLastPart.match(regexp);
-    return match ? match[0] : "";
+    const regex1 = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
+    const matchAlt = [...exprLastPart.matchAll(regex1)];
+    if (matchAlt.length > 0 && matchAlt[0].groups && matchAlt[0].groups.metricName) {
+      return matchAlt[0].groups.metricName;
+    }
+
+    const regex2 = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
+    const match = [...exprLastPart.matchAll(regex2)];
+    if (match.length > 0 && match[0].groups && match[0].groups.metricName) {
+      return match[0].groups.metricName;
+    }
+
+    return "";
   }, [exprLastPart]);
 
   const label = useMemo(() => {
@@ -51,7 +62,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   }, [exprLastPart]);
 
   const shouldSuppressAutoSuggestion = (value: string) => {
-    const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right)\b)/;
+    const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right|by|without|on|ignoring)\b)/i;
     const parts = value.split(/\s+/);
     const partsCount = parts.length;
     const lastPart = parts[partsCount - 1];
@@ -63,12 +74,16 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   };
 
   const context = useMemo(() => {
-    if (!values.beforeCursor || values.beforeCursor.endsWith("}") || shouldSuppressAutoSuggestion(values.beforeCursor)) {
+    const valueBeforeCursor = values.beforeCursor.trim();
+    const endOfClosedBrackets = ["}", ")"].some(char => valueBeforeCursor.endsWith(char));
+    const endOfClosedQuotes = !hasUnclosedQuotes(valueBeforeCursor) && ["`", "'", "\""].some(char => valueBeforeCursor.endsWith(char));
+    if (!values.beforeCursor || endOfClosedBrackets || endOfClosedQuotes || shouldSuppressAutoSuggestion(values.beforeCursor)) {
       return QueryContextType.empty;
     }
 
-    const labelRegexp = /\{[^}]*$/;
-    const labelValueRegexp = new RegExp(`(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`, "g");
+    const labelRegexp = /(?:by|without|on|ignoring)\s*\(\s*[^)]*$|\{[^}]*$/i;
+    const patternLabelValue = `(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`;
+    const labelValueRegexp = new RegExp(patternLabelValue, "g");
 
     switch (true) {
       case labelValueRegexp.test(values.beforeCursor):
@@ -81,7 +96,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   }, [values, metric, label]);
 
   const valueByContext = useMemo(() => {
-    const wordMatch = values.beforeCursor.match(/([\w_\-.:/]+(?![},]))$/);
+    const wordMatch = values.beforeCursor.match(/([\w_.:]+(?![},]))$/);
     return wordMatch ? wordMatch[0] : "";
   }, [values.beforeCursor]);
 
@@ -119,9 +134,10 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     // Add quotes around the value if the context is labelValue
     if (context === QueryContextType.labelValue) {
       const quote = "\"";
-      const needsQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext);
       valueAfterCursor = valueAfterCursor.replace(/^[^\s"|},]*/, "");
-      insert = `${needsQuote ? quote : ""}${insert}`;
+      const needsOpenQuote = /(?:=|!=|=~|!~)$/.test(beforeValueByContext);
+      const needsCloseQuote = valueAfterCursor.trim()[0] !== "\"";
+      insert = `${needsOpenQuote ? quote : ""}${insert}${needsCloseQuote ? quote : ""}`;
     }
 
     if (context === QueryContextType.label) {

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -520,3 +520,25 @@ export const DownloadIcon = () => (
     <path d="M19 9h-4V3H9v6H5l7 7 7-7zM5 18v2h14v-2H5z"></path>
   </svg>
 );
+
+export const ExpandIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M12 5.83 15.17 9l1.41-1.41L12 3 7.41 7.59 8.83 9zm0 12.34L8.83 15l-1.41 1.41L12 21l4.59-4.59L15.17 15z"
+    ></path>
+  </svg>
+);
+
+export const CollapseIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M7.41 18.59 8.83 20 12 16.83 15.17 20l1.41-1.41L12 14zm9.18-13.18L15.17 4 12 7.17 8.83 4 7.41 5.41 12 10z"
+    ></path>
+  </svg>
+);

app/vmui/packages/vmui/src/components/Main/Popper/style.scss

@@ -32,19 +32,19 @@
 
   &-header {
     display: grid;
-    grid-template-columns: 1fr auto;
-    gap: $padding-small;
+    grid-template-columns: 1fr 25px;
+    gap: $padding-global;
     align-items: center;
     justify-content: space-between;
     background-color: $color-background-block;
-    padding: $padding-small $padding-small $padding-small $padding-global;
+    padding: $padding-small $padding-global;
     border-radius: $border-radius-small $border-radius-small 0 0;
     color: $color-text;
     border-bottom: $border-divider;
     margin-bottom: $padding-global;
-    min-height: 51px;
 
     &__title {
+      font-size: $font-size-small;
       font-weight: bold;
       user-select: none;
     }

app/vmui/packages/vmui/src/components/Table/Table.tsx

@@ -6,11 +6,14 @@ import Tooltip from "../Main/Tooltip/Tooltip";
 import Button from "../Main/Button/Button";
 import { useEffect } from "preact/compat";
 
+type OrderDir = "asc" | "desc"
+
 interface TableProps<T> {
   rows: T[];
   columns: { title?: string, key: keyof Partial<T>, className?: string }[];
   defaultOrderBy: keyof T;
   copyToClipboard?: keyof T;
+  defaultOrderDir?: OrderDir;
   // TODO: Remove when pagination is implemented on the backend.
   paginationOffset: {
     startIndex: number;
@@ -18,9 +21,9 @@ interface TableProps<T> {
   }
 }
 
-const Table = <T extends object>({ rows, columns, defaultOrderBy, copyToClipboard, paginationOffset }: TableProps<T>) => {
+const Table = <T extends object>({ rows, columns, defaultOrderBy, defaultOrderDir, copyToClipboard, paginationOffset }: TableProps<T>) => {
   const [orderBy, setOrderBy] = useState<keyof T>(defaultOrderBy);
-  const [orderDir, setOrderDir] = useState<"asc" | "desc">("desc");
+  const [orderDir, setOrderDir] = useState<OrderDir>(defaultOrderDir || "desc");
   const [copied, setCopied] = useState<number | null>(null);
 
   // const sortedList = useMemo(() => stableSort(rows as [], getComparator(orderDir, orderBy)),

app/vmui/packages/vmui/src/components/Table/TableSettings/TableSettings.tsx

@@ -44,6 +44,14 @@ const TableSettings: FC<TableSettingsProps> = ({
     onChangeColumns(defaultColumns.includes(key) ? defaultColumns.filter(col => col !== key) : [...defaultColumns, key]);
   };
 
+  const toggleAllColumns = () => {
+    if (defaultColumns.length === columns.length) {
+      onChangeColumns([]);
+    } else {
+      onChangeColumns(columns);
+    }
+  };
+
   const handleResetColumns = () => {
     handleClose();
     onChangeColumns(columns);
@@ -105,6 +113,16 @@ const TableSettings: FC<TableSettingsProps> = ({
                 />
               </Tooltip>
             </div>
+            <div
+              className="vm-table-settings-popper-list__item vm-table-settings-popper-list__check_all"
+            >
+              <Checkbox
+                checked={defaultColumns.length === columns.length}
+                onChange={toggleAllColumns}
+                label="Check all"
+                disabled={tableCompact}
+              />
+            </div>
             {columns.map(col => (
               <div
                 className="vm-table-settings-popper-list__item"

app/vmui/packages/vmui/src/components/Table/TableSettings/style.scss

@@ -39,5 +39,9 @@
     &__item {
       font-size: $font-size;
     }
+    &__check_all {
+      padding: 0 0 $padding-global;
+      border-bottom: $border-divider;
+    }
   }
 }

app/vmui/packages/vmui/src/components/TraceQuery/TracingsView.tsx

@@ -1,7 +1,7 @@
 import React, { FC, useState } from "preact/compat";
 import Trace from "./Trace";
 import Button from "../Main/Button/Button";
-import { ArrowDownIcon, CodeIcon, DeleteIcon, DownloadIcon } from "../Main/Icons";
+import { CodeIcon, CollapseIcon, DeleteIcon, DownloadIcon, ExpandIcon } from "../Main/Icons";
 import "./style.scss";
 import NestedNav from "./NestedNav/NestedNav";
 import Alert from "../Main/Alert/Alert";
@@ -89,13 +89,7 @@ const TracingsView: FC<TraceViewProps> = ({ traces, jsonEditor = false, onDelete
               <Tooltip title={expandedTraces.includes(trace.idValue) ? "Collapse All" : "Expand All"}>
                 <Button
                   variant="text"
-                  startIcon={(
-                    <div
-                      className={classNames({
-                        "vm-tracings-view-trace-header__expand-icon": true,
-                        "vm-tracings-view-trace-header__expand-icon_open": expandedTraces.includes(trace.idValue) })}
-                    ><ArrowDownIcon/></div>
-                  )}
+                  startIcon={expandedTraces.includes(trace.idValue) ? <CollapseIcon/> : <ExpandIcon/> }
                   onClick={handleExpandAll(trace)}
                   ariaLabel={expandedTraces.includes(trace.idValue) ? "Collapse All" : "Expand All"}
                 />

app/vmui/packages/vmui/src/constants/palette.ts

@@ -15,7 +15,13 @@ export const darkPalette = {
   "box-shadow-popper": "rgba(0, 0, 0, 0.2) 0px 2px 8px 0px",
   "border-divider": "1px solid rgba(99, 110, 123, 0.5)",
   "color-hover-black": "rgba(0, 0, 0, 0.12)",
-  "color-log-hits-bar": "rgba(255, 255, 255, 0.18)"
+  // log hits chart colors
+  "color-log-hits-bar-0": "rgba(255, 255, 255, 0.18)",
+  "color-log-hits-bar-1": "#FFB74D",
+  "color-log-hits-bar-2": "#81C784",
+  "color-log-hits-bar-3": "#64B5F6",
+  "color-log-hits-bar-4": "#E57373",
+  "color-log-hits-bar-5": "#8a62f0",
 };
 
 export const lightPalette = {
@@ -35,5 +41,12 @@ export const lightPalette = {
   "box-shadow-popper": "rgba(0, 0, 0, 0.1) 0px 2px 8px 0px",
   "border-divider": "1px solid rgba(0, 0, 0, 0.15)",
   "color-hover-black": "rgba(0, 0, 0, 0.06)",
-  "color-log-hits-bar": "rgba(0, 0, 0, 0.18)"
+  // log hits chart colors
+  "color-log-hits-bar-0": "rgba(0, 0, 0, 0.18)",
+  "color-log-hits-bar-1": "#FFB74D",
+  "color-log-hits-bar-2": "#81C784",
+  "color-log-hits-bar-3": "#64B5F6",
+  "color-log-hits-bar-4": "#E57373",
+  "color-log-hits-bar-5": "#8a62f0",
+
 };

app/vmui/packages/vmui/src/contexts/AppContextProvider.tsx

@@ -3,10 +3,11 @@ import { TimeStateProvider } from "../state/time/TimeStateContext";
 import { QueryStateProvider } from "../state/query/QueryStateContext";
 import { CustomPanelStateProvider } from "../state/customPanel/CustomPanelStateContext";
 import { GraphStateProvider } from "../state/graph/GraphStateContext";
+import { DashboardsStateProvider } from "../state/dashboards/DashboardsStateContext";
+import { LogsStateProvider } from "../state/logsPanel/LogsStateContext";
 import { SnackbarProvider } from "./Snackbar";
 
 import { combineComponents } from "../utils/combine-components";
-import { DashboardsStateProvider } from "../state/dashboards/DashboardsStateContext";
 
 const providers = [
   AppStateProvider,
@@ -15,7 +16,8 @@ const providers = [
   CustomPanelStateProvider,
   GraphStateProvider,
   SnackbarProvider,
-  DashboardsStateProvider
+  DashboardsStateProvider,
+  LogsStateProvider
 ];
 
 export default combineComponents(...providers);

app/vmui/packages/vmui/src/hooks/useFetchQueryOptions.tsx

@@ -137,7 +137,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
 
   // fetch labels
   useEffect(() => {
-    if (!serverUrl || !metric || context !== QueryContextType.label) {
+    if (!serverUrl || context !== QueryContextType.label) {
       return;
     }
     setLabels([]);
@@ -149,7 +149,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
       urlSuffix: "labels",
       setter: setLabels,
       type: TypeData.label,
-      params: getQueryParams({ "match[]": `{__name__="${metricEscaped}"}` })
+      params: getQueryParams(metric ? { "match[]": `{__name__="${metricEscaped}"}` } : undefined)
     });
 
     return () => abortControllerRef.current?.abort();
@@ -157,20 +157,23 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
 
   // fetch labelValues
   useEffect(() => {
-    if (!serverUrl || !metric || !label || context !== QueryContextType.labelValue) {
+    if (!serverUrl || !label || context !== QueryContextType.labelValue) {
       return;
     }
     setLabelValues([]);
 
     const metricEscaped = escapeDoubleQuotes(metric);
     const valueReEscaped = escapeDoubleQuotes(escapeRegexp(value));
+    const matchMetric = metric ? `__name__="${metricEscaped}"` : "";
+    const matchLabel = `${label}=~".*${valueReEscaped}.*"`;
+    const matchValue = [matchMetric, matchLabel].filter(Boolean).join(",");
 
     fetchData({
       value,
       urlSuffix: `label/${label}/values`,
       setter: setLabelValues,
       type: TypeData.labelValue,
-      params: getQueryParams({ "match[]": `{__name__="${metricEscaped}", ${label}=~".*${valueReEscaped}.*"}` })
+      params: getQueryParams({ "match[]": `{${matchValue}}` })
     });
 
     return () => abortControllerRef.current?.abort();

app/vmui/packages/vmui/src/hooks/useSortedCategories.ts

@@ -1,5 +1,5 @@
-import { useMemo } from "preact/compat";
-import { MetricBase } from "../api/types";
+import {useMemo} from "preact/compat";
+import {MetricBase} from "../api/types";
 
 export type MetricCategory = {
   key: string;
@@ -10,7 +10,7 @@ export const getColumns = (data: MetricBase[]): MetricCategory[] => {
   const columns: { [key: string]: { options: Set<string> } } = {};
   data.forEach(d =>
     Object.entries(d.metric).forEach(e =>
-      columns[e[0]] ? columns[e[0]].options.add(e[1]) : columns[e[0]] = { options: new Set([e[1]]) }
+      columns[e[0]] ? columns[e[0]].options.add(e[1]) : columns[e[0]] = {options: new Set([e[1]])}
     )
   );
 
@@ -22,7 +22,8 @@ export const getColumns = (data: MetricBase[]): MetricCategory[] => {
 
 export const useSortedCategories = (data: MetricBase[], displayColumns?: string[]): MetricCategory[] => (
   useMemo(() => {
+    if (!displayColumns) return [];
     const sortedColumns = getColumns(data);
-    return displayColumns ? sortedColumns.filter(col => displayColumns.includes(col.key)) : sortedColumns;
+    return sortedColumns.filter(col => displayColumns.includes(col.key));
   }, [data, displayColumns])
 );

app/vmui/packages/vmui/src/layouts/LogsLayout/ControlsLogsLayout.tsx

@@ -3,6 +3,7 @@ import classNames from "classnames";
 import GlobalSettings from "../../components/Configurators/GlobalSettings/GlobalSettings";
 import { ControlsProps } from "../Header/HeaderControls/HeaderControls";
 import { TimeSelector } from "../../components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector";
+import TenantsFields from "../../components/Configurators/GlobalSettings/TenantsConfiguration/TenantsFields";
 
 const ControlsLogsLayout: FC<ControlsProps> = ({ isMobile }) => {
 
@@ -13,6 +14,7 @@ const ControlsLogsLayout: FC<ControlsProps> = ({ isMobile }) => {
         "vm-header-controls_mobile": isMobile,
       })}
     >
+      <TenantsFields/>
       <TimeSelector/>
       <GlobalSettings/>
     </div>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useCallback, useEffect } from "preact/compat";
+import React, { FC, useCallback, useEffect, useState } from "preact/compat";
 import ExploreLogsBody from "./ExploreLogsBody/ExploreLogsBody";
 import useStateSearchParams from "../../hooks/useStateSearchParams";
 import useSearchParamsFromObject from "../../hooks/useSearchParamsFromObject";
@@ -9,7 +9,6 @@ import Alert from "../../components/Main/Alert/Alert";
 import ExploreLogsHeader from "./ExploreLogsHeader/ExploreLogsHeader";
 import "./style.scss";
 import { ErrorTypes, TimeParams } from "../../types";
-import { useState } from "react";
 import { useTimeState } from "../../state/time/TimeStateContext";
 import { getFromStorage, saveToStorage } from "../../utils/storage";
 import ExploreLogsBarChart from "./ExploreLogsBarChart/ExploreLogsBarChart";
@@ -27,11 +26,12 @@ const ExploreLogs: FC = () => {
 
   const [limit, setLimit] = useStateSearchParams(defaultLimit, "limit");
   const [query, setQuery] = useStateSearchParams("*", "query");
+  const [tmpQuery, setTmpQuery] = useState("");
   const [period, setPeriod] = useState<TimeParams>(periodState);
+  const [queryError, setQueryError] = useState<ErrorTypes | string>("");
+
   const { logs, isLoading, error, fetchLogs } = useFetchLogs(serverUrl, query, limit);
   const { fetchLogHits, ...dataLogHits } = useFetchLogHits(serverUrl, query);
-  const [queryError, setQueryError] = useState<ErrorTypes | string>("");
-  const [markdownParsing, setMarkdownParsing] = useState(getFromStorage("LOGS_MARKDOWN") === "true");
 
   const getPeriod = useCallback(() => {
     const relativeTimeOpts = relativeTimeOptions.find(d => d.id === relativeTime);
@@ -45,6 +45,7 @@ const ExploreLogs: FC = () => {
       setQueryError(ErrorTypes.validQuery);
       return;
     }
+    setQueryError("");
 
     const newPeriod = getPeriod();
     setPeriod(newPeriod);
@@ -65,9 +66,13 @@ const ExploreLogs: FC = () => {
     saveToStorage("LOGS_LIMIT", `${limit}`);
   };
 
-  const handleChangeMarkdownParsing = (val: boolean) => {
-    saveToStorage("LOGS_MARKDOWN", `${val}`);
-    setMarkdownParsing(val);
+  const handleApplyFilter = (val: string) => {
+    setQuery(prev => `_stream: ${val === "other" ? "{}" : val} AND (${prev})`);
+  };
+
+  const handleUpdateQuery = () => {
+    setQuery(tmpQuery);
+    handleRunQuery();
   };
 
   useEffect(() => {
@@ -75,20 +80,19 @@ const ExploreLogs: FC = () => {
   }, [periodState]);
 
   useEffect(() => {
-    setQueryError("");
+    handleRunQuery();
+    setTmpQuery(query);
   }, [query]);
 
   return (
     <div className="vm-explore-logs">
       <ExploreLogsHeader
-        query={query}
+        query={tmpQuery}
         error={queryError}
         limit={limit}
-        markdownParsing={markdownParsing}
-        onChange={setQuery}
+        onChange={setTmpQuery}
         onChangeLimit={handleChangeLimit}
-        onRun={handleRunQuery}
-        onChangeMarkdownParsing={handleChangeMarkdownParsing}
+        onRun={handleUpdateQuery}
       />
       {isLoading && <Spinner message={"Loading logs..."}/>}
       {error && <Alert variant="error">{error}</Alert>}
@@ -97,13 +101,11 @@ const ExploreLogs: FC = () => {
           {...dataLogHits}
           query={query}
           period={period}
+          onApplyFilter={handleApplyFilter}
           isLoading={isLoading ? false : dataLogHits.isLoading}
         />
       )}
-      <ExploreLogsBody
-        data={logs}
-        markdownParsing={markdownParsing}
-      />
+      <ExploreLogsBody data={logs}/>
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBarChart/ExploreLogsBarChart.tsx

@@ -17,19 +17,34 @@ interface Props {
   period: TimeParams;
   error?: string;
   isLoading: boolean;
+  onApplyFilter: (value: string) => void;
 }
 
-const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading }) => {
+const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading, onApplyFilter }) => {
   const { isMobile } = useDeviceDetect();
   const timeDispatch = useTimeDispatch();
 
+  const getXAxis = (timestamps: string[]): number[] => {
+    return (timestamps.map(t => t ? dayjs(t).unix() : null)
+      .filter(Boolean) as number[])
+      .sort((a, b) => a - b);
+  };
+
+  const getYAxes = (logHits: LogHits[], timestamps: string[]) => {
+    return logHits.map(hits => {
+      return timestamps.map(t => {
+        const index = hits.timestamps.findIndex(ts => ts === t);
+        return index === -1 ? null : hits.values[index] || null;
+      });
+    });
+  };
+
   const data = useMemo(() => {
-    const hits = logHits[0];
-    if (!hits) return [[], []] as AlignedData;
-    const { values, timestamps } = hits;
-    const xAxis = timestamps.map(t => t ? dayjs(t).unix() : null).filter(Boolean);
-    const yAxis = values.map(v => v || null);
-    return [xAxis, yAxis] as AlignedData;
+    if (!logHits.length) return [[], []] as AlignedData;
+    const timestamps = Array.from(new Set(logHits.map(l => l.timestamps).flat()));
+    const xAxis = getXAxis(timestamps);
+    const yAxes = getYAxes(logHits, timestamps);
+    return [xAxis, ...yAxes] as AlignedData;
   }, [logHits]);
 
   const noDataMessage: string = useMemo(() => {
@@ -75,9 +90,11 @@ const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading }) =
 
       {data && (
         <BarHitsChart
+          logHits={logHits}
           data={data}
           period={period}
           setPeriod={setPeriod}
+          onApplyFilter={onApplyFilter}
         />
       )}
     </section>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBarChart/style.scss

@@ -5,7 +5,6 @@
   display: flex;
   align-items: center;
   justify-content: center;
-  height: 200px;
   padding: 0 0 0 $padding-small !important;
   width: calc(100vw - ($padding-medium * 2));
 

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useState, useMemo } from "preact/compat";
+import React, { FC, useState, useMemo, useRef } from "preact/compat";
 import JsonView from "../../../components/Views/JsonView/JsonView";
 import { CodeIcon, ListIcon, TableIcon } from "../../../components/Main/Icons";
 import Tabs from "../../../components/Main/Tabs/Tabs";
@@ -19,7 +19,6 @@ import { marked } from "marked";
 
 export interface ExploreLogBodyProps {
   data: Logs[];
-  markdownParsing: boolean;
 }
 
 enum DisplayType {
@@ -34,10 +33,11 @@ const tabs = [
   { label: "JSON", value: DisplayType.json, icon: <CodeIcon/> },
 ];
 
-const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) => {
+const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data }) => {
   const { isMobile } = useDeviceDetect();
   const { timezone } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
+  const groupSettingsRef = useRef<HTMLDivElement>(null);
 
   const [activeTab, setActiveTab] = useStateSearchParams(DisplayType.group, "view");
   const [displayColumns, setDisplayColumns] = useState<string[]>([]);
@@ -88,6 +88,9 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) =>
             items={tabs}
             onChange={handleChangeTab}
           />
+          <div className="vm-explore-logs-body-header__log-info">
+            Total logs returned: <b>{data.length}</b>
+          </div>
         </div>
         {activeTab === DisplayType.table && (
           <div className="vm-explore-logs-body-header__settings">
@@ -100,6 +103,12 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) =>
             />
           </div>
         )}
+        {activeTab === DisplayType.group && (
+          <div
+            className="vm-explore-logs-body-header__settings"
+            ref={groupSettingsRef}
+          />
+        )}
       </div>
 
       <div
@@ -123,7 +132,7 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, markdownParsing }) =>
               <GroupLogs
                 logs={logs}
                 columns={columns}
-                markdownParsing={markdownParsing}
+                settingsRef={groupSettingsRef}
               />
             )}
             {activeTab === DisplayType.json && (

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/TableLogs.tsx

@@ -39,7 +39,8 @@ const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, col
 
 
   const filteredColumns = useMemo(() => {
-    if (!displayColumns?.length || tableCompact) return tableColumns;
+    if (tableCompact) return tableColumns;
+    if (!displayColumns?.length) return [];
     return tableColumns.filter(c => displayColumns.includes(c.key as string));
   }, [tableColumns, displayColumns, tableCompact]);
 
@@ -48,7 +49,8 @@ const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, col
       <Table
         rows={logs}
         columns={filteredColumns}
-        defaultOrderBy={"_vmui_time"}
+        defaultOrderBy={"_time"}
+        defaultOrderDir={"desc"}
         copyToClipboard={"_vmui_data"}
         paginationOffset={{ startIndex: 0, endIndex: Infinity }}
       />

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/style.scss

@@ -13,6 +13,14 @@
       align-items: center;
       gap: $padding-small;
     }
+
+    &__log-info {
+      flex-grow: 1;
+      text-align: right;
+      padding-right: $padding-global;
+      color: $color-text-secondary;
+
+    }
   }
 
   &__empty {

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx

@@ -6,28 +6,23 @@ import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import Button from "../../../components/Main/Button/Button";
 import QueryEditor from "../../../components/Configurators/QueryEditor/QueryEditor";
 import TextField from "../../../components/Main/TextField/TextField";
-import Switch from "../../../components/Main/Switch/Switch";
 
 export interface ExploreLogHeaderProps {
   query: string;
   limit: number;
   error?: string;
-  markdownParsing: boolean;
   onChange: (val: string) => void;
   onChangeLimit: (val: number) => void;
   onRun: () => void;
-  onChangeMarkdownParsing: (val: boolean) => void;
 }
 
 const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
   query,
   limit,
   error,
-  markdownParsing,
   onChange,
   onChangeLimit,
   onRun,
-  onChangeMarkdownParsing,
 }) => {
   const { isMobile } = useDeviceDetect();
 
@@ -78,14 +73,7 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
         />
       </div>
       <div className="vm-explore-logs-header-bottom">
-        <div className="vm-explore-logs-header-bottom-contols">
-          <Switch
-            label={"Markdown parsing"}
-            value={markdownParsing}
-            onChange={onChangeMarkdownParsing}
-            fullWidth={isMobile}
-          />
-        </div>
+        <div className="vm-explore-logs-header-bottom-contols"></div>
         <div className="vm-explore-logs-header-bottom-helpful">
           <a
             className="vm-link vm-link_with-icon"

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useEffect, useMemo } from "preact/compat";
+import React, { FC, useCallback, useEffect, useMemo, useRef } from "preact/compat";
 import { MouseEvent, useState } from "react";
 import "./style.scss";
 import { Logs } from "../../../api/types";
@@ -9,89 +9,213 @@ import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 import GroupLogsItem from "./GroupLogsItem";
 import { useAppState } from "../../../state/common/StateContext";
 import classNames from "classnames";
+import Button from "../../../components/Main/Button/Button";
+import { CollapseIcon, ExpandIcon, StorageIcon } from "../../../components/Main/Icons";
+import Popper from "../../../components/Main/Popper/Popper";
+import TextField from "../../../components/Main/TextField/TextField";
+import useBoolean from "../../../hooks/useBoolean";
+import useStateSearchParams from "../../../hooks/useStateSearchParams";
+import { useSearchParams } from "react-router-dom";
+
+const WITHOUT_GROUPING = "No Grouping";
 
 interface TableLogsProps {
   logs: Logs[];
   columns: string[];
-  markdownParsing: boolean;
+  settingsRef: React.Ref<HTMLDivElement>;
 }
 
-const GroupLogs: FC<TableLogsProps> = ({ logs, markdownParsing }) => {
+const GroupLogs: FC<TableLogsProps> = ({ logs, settingsRef }) => {
   const { isDarkTheme } = useAppState();
   const copyToClipboard = useCopyToClipboard();
+  const [searchParams, setSearchParams] = useSearchParams();
 
+  const [expandGroups, setExpandGroups] = useState<boolean[]>([]);
+  const [groupBy, setGroupBy] = useStateSearchParams("_stream", "groupBy");
   const [copied, setCopied] = useState<string | null>(null);
+  const [searchKey, setSearchKey] = useState("");
+  const optionsButtonRef = useRef<HTMLDivElement>(null);
+
+  const {
+    value: openOptions,
+    toggle: toggleOpenOptions,
+    setFalse: handleCloseOptions,
+  } = useBoolean(false);
+
+  const expandAll = useMemo(() => expandGroups.every(Boolean), [expandGroups]);
+
+  const logsKeys = useMemo(() => {
+    const excludeKeys = ["_msg", "_time", "_vmui_time", "_vmui_data", "_vmui_markdown"];
+    const uniqKeys = Array.from(new Set(logs.map(l => Object.keys(l)).flat()));
+    const keys = [WITHOUT_GROUPING, ...uniqKeys.filter(k => !excludeKeys.includes(k))];
+
+    if (!searchKey) return keys;
+    try {
+      const regexp = new RegExp(searchKey, "i");
+      const found = keys.filter((item) => regexp.test(item));
+      return found.sort((a,b) => (a.match(regexp)?.index || 0) - (b.match(regexp)?.index || 0));
+    } catch (e) {
+      return [];
+    }
+  }, [logs, searchKey]);
 
   const groupData = useMemo(() => {
-    return groupByMultipleKeys(logs, ["_stream"]).map((item) => {
-      const streamValue = item.values[0]?._stream || "";
-      const pairs = streamValue.slice(1, -1).match(/(?:[^\\,]+|\\,)+?(?=,|$)/g) || [streamValue];
+    return groupByMultipleKeys(logs, [groupBy]).map((item) => {
+      const streamValue = item.values[0]?.[groupBy] || "";
+      const pairs = /^{.+}$/.test(streamValue)
+        ? streamValue.slice(1, -1).match(/(\\.|[^,])+/g) || [streamValue]
+        : [streamValue];
       return {
         ...item,
         pairs: pairs.filter(Boolean),
       };
     });
-  }, [logs]);
+  }, [logs, groupBy]);
 
-  const handleClickByPair = (pair: string) => async (e: MouseEvent<HTMLDivElement>) => {
+  const handleClickByPair = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
     e.stopPropagation();
-    const isCopied = await copyToClipboard(`${pair.replace(/=/, ": ")}`);
+    const isKeyValue = /(.+)?=(".+")/.test(value);
+    const copyValue = isKeyValue ? `${value.replace(/=/, ": ")}` : `${groupBy}: "${value}"`;
+    const isCopied = await copyToClipboard(copyValue);
     if (isCopied) {
-      setCopied(pair);
+      setCopied(value);
     }
   };
 
+  const handleSelectGroupBy = (key: string) => () => {
+    setGroupBy(key);
+    searchParams.set("groupBy", key);
+    setSearchParams(searchParams);
+    handleCloseOptions();
+  };
+
+  const handleToggleExpandAll = useCallback(() => {
+    setExpandGroups(new Array(groupData.length).fill(!expandAll));
+  }, [expandAll]);
+
+  const handleChangeExpand = (i: number) => (value: boolean) => {
+    setExpandGroups((prev) => {
+      const newExpandGroups = [...prev];
+      newExpandGroups[i] = value;
+      return newExpandGroups;
+    });
+
+  };
+
   useEffect(() => {
     if (copied === null) return;
     const timeout = setTimeout(() => setCopied(null), 2000);
     return () => clearTimeout(timeout);
   }, [copied]);
 
+  useEffect(() => {
+    setExpandGroups(new Array(groupData.length).fill(true));
+  }, [groupData]);
+
   return (
-    <div className="vm-group-logs">
-      {groupData.map((item) => (
-        <div
-          className="vm-group-logs-section"
-          key={item.keys.join("")}
-        >
-          <Accordion
-            defaultExpanded={true}
-            title={(
-              <div className="vm-group-logs-section-keys">
-                <span className="vm-group-logs-section-keys__title">Group by _stream:</span>
-                {item.pairs.map((pair) => (
-                  <Tooltip
-                    title={copied === pair ? "Copied" : "Copy to clipboard"}
-                    key={`${item.keys.join("")}_${pair}`}
-                    placement={"top-center"}
-                  >
-                    <div
-                      className={classNames({
-                        "vm-group-logs-section-keys__pair": true,
-                        "vm-group-logs-section-keys__pair_dark": isDarkTheme
-                      })}
-                      onClick={handleClickByPair(pair)}
+    <>
+      <div className="vm-group-logs">
+        {groupData.map((item, i) => (
+          <div
+            className="vm-group-logs-section"
+            key={item.keys.join("")}
+          >
+            <Accordion
+              key={String(expandGroups[i])}
+              defaultExpanded={expandGroups[i]}
+              onChange={handleChangeExpand(i)}
+              title={groupBy !== WITHOUT_GROUPING && (
+                <div className="vm-group-logs-section-keys">
+                  <span className="vm-group-logs-section-keys__title">Group by <code>{groupBy}</code>:</span>
+                  {item.pairs.map((pair) => (
+                    <Tooltip
+                      title={copied === pair ? "Copied" : "Copy to clipboard"}
+                      key={`${item.keys.join("")}_${pair}`}
+                      placement={"top-center"}
                     >
-                      {pair}
-                    </div>
-                  </Tooltip>
+                      <div
+                        className={classNames({
+                          "vm-group-logs-section-keys__pair": true,
+                          "vm-group-logs-section-keys__pair_dark": isDarkTheme
+                        })}
+                        onClick={handleClickByPair(pair)}
+                      >
+                        {pair}
+                      </div>
+                    </Tooltip>
+                  ))}
+                  <span className="vm-group-logs-section-keys__count">{item.values.length} entries</span>
+                </div>
+              )}
+            >
+              <div className="vm-group-logs-section-rows">
+                {item.values.map((value) => (
+                  <GroupLogsItem
+                    key={`${value._msg}${value._time}`}
+                    log={value}
+                  />
                 ))}
               </div>
-            )}
-          >
-            <div className="vm-group-logs-section-rows">
-              {item.values.map((value) => (
-                <GroupLogsItem
-                  key={`${value._msg}${value._time}`}
-                  log={value}
-                  markdownParsing={markdownParsing}
-                />
-              ))}
+            </Accordion>
+          </div>
+        ))}
+      </div>
+
+
+      {settingsRef.current && React.createPortal((
+        <div className="vm-group-logs-header">
+          <Tooltip title={expandAll ? "Collapse All" : "Expand All"}>
+            <Button
+              variant="text"
+              startIcon={expandAll ? <CollapseIcon/> : <ExpandIcon/> }
+              onClick={handleToggleExpandAll}
+              ariaLabel={expandAll ? "Collapse All" : "Expand All"}
+            />
+          </Tooltip>
+          <Tooltip title={"Group by"}>
+            <div ref={optionsButtonRef}>
+              <Button
+                variant="text"
+                startIcon={<StorageIcon/> }
+                onClick={toggleOpenOptions}
+                ariaLabel={"Group by"}
+              />
             </div>
-          </Accordion>
+          </Tooltip>
+          {
+            <Popper
+              open={openOptions}
+              placement="bottom-right"
+              onClose={handleCloseOptions}
+              buttonRef={optionsButtonRef}
+            >
+              <div className="vm-list vm-group-logs-header-keys">
+                <div className="vm-group-logs-header-keys__search">
+                  <TextField
+                    label="Search key"
+                    value={searchKey}
+                    onChange={setSearchKey}
+                    type="search"
+                  />
+                </div>
+                {logsKeys.map(id => (
+                  <div
+                    className={classNames({
+                      "vm-list-item": true,
+                      "vm-list-item_active": id === groupBy
+                    })}
+                    key={id}
+                    onClick={handleSelectGroupBy(id)}
+                  >
+                    {id}
+                  </div>
+                ))}
+              </div>
+            </Popper>
+          }
         </div>
-      ))}
-    </div>
+      ), settingsRef.current)}
+    </>
   );
 };
 

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx

@@ -7,19 +7,21 @@ import Tooltip from "../../../components/Main/Tooltip/Tooltip";
 import { ArrowDownIcon, CopyIcon } from "../../../components/Main/Icons";
 import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 import classNames from "classnames";
+import { useLogsState } from "../../../state/logsPanel/LogsStateContext";
 
 interface Props {
   log: Logs;
-  markdownParsing: boolean;
 }
 
-const GroupLogsItem: FC<Props> = ({ log, markdownParsing }) => {
+const GroupLogsItem: FC<Props> = ({ log }) => {
   const {
     value: isOpenFields,
     toggle: toggleOpenFields,
   } = useBoolean(false);
 
-  const excludeKeys = ["_stream", "_msg", "_time", "_vmui_time", "_vmui_data", "_vmui_markdown"];
+  const { markdownParsing } = useLogsState();
+
+  const excludeKeys = ["_msg", "_vmui_time", "_vmui_data", "_vmui_markdown"];
   const fields = Object.entries(log).filter(([key]) => !excludeKeys.includes(key));
   const hasFields = fields.length > 0;
 

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss

@@ -3,6 +3,22 @@
 .vm-group-logs {
   margin-top: calc(-1 * $padding-medium);
 
+  &-header {
+    display: flex;
+    align-items: center;
+    justify-content: flex-end;
+    gap: $padding-global;
+
+    &-keys {
+      max-height: 300px;
+      overflow: auto;
+
+      &__search {
+        padding: $padding-small;
+      }
+    }
+  }
+
   &-section {
     &-keys {
       display: flex;
@@ -14,6 +30,24 @@
 
       &__title {
         font-weight: bold;
+
+        code {
+          font-family: monospace;
+          &:before {
+            content: "\"";
+          }
+          &:after {
+            content: "\"";
+          }
+        }
+      }
+
+      &__count {
+        flex-grow: 1;
+        text-align: right;
+        font-size: $font-size-small;
+        color: $color-text-secondary;
+        padding-right: calc($padding-large * 3);
       }
 
       &__pair {

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogHits.ts

@@ -4,8 +4,11 @@ import { ErrorTypes, TimeParams } from "../../../types";
 import { LogHits } from "../../../api/types";
 import dayjs from "dayjs";
 import { LOGS_BARS_VIEW } from "../../../constants/logs";
+import { useSearchParams } from "react-router-dom";
 
 export const useFetchLogHits = (server: string, query: string) => {
+  const [searchParams] = useSearchParams();
+
   const [logHits, setLogHits] = useState<LogHits[]>([]);
   const [isLoading, setIsLoading] = useState<{[key: number]: boolean;}>([]);
   const [error, setError] = useState<ErrorTypes | string>();
@@ -22,15 +25,55 @@ export const useFetchLogHits = (server: string, query: string) => {
     return {
       signal,
       method: "POST",
+      headers: {
+        AccountID: searchParams.get("accountID") || "0",
+        ProjectID: searchParams.get("projectID") || "0",
+      },
       body: new URLSearchParams({
         query: query.trim(),
         step: `${step}ms`,
         start: start.toISOString(),
         end: end.toISOString(),
+        field: "_stream" // In the future, this field can be made configurable
+
       })
     };
   };
 
+  const accumulateHits = (resultHit: LogHits, hit: LogHits) => {
+    resultHit.total = (resultHit.total || 0) + (hit.total || 0);
+    hit.timestamps.forEach((timestamp, i) => {
+      const index = resultHit.timestamps.findIndex(t => t === timestamp);
+      if (index === -1) {
+        resultHit.timestamps.push(timestamp);
+        resultHit.values.push(hit.values[i]);
+      } else {
+        resultHit.values[index] += hit.values[i];
+      }
+    });
+    return resultHit;
+  };
+
+  const getHitsWithTop = (hits: LogHits[]) => {
+    const topN = 5;
+    const defaultHit = { fields: {}, timestamps: [], values: [], total: 0 };
+
+    const hitsByTotal = hits.sort((a, b) => (b.total || 0) - (a.total || 0));
+    const result = [];
+
+    const otherHits: LogHits = hitsByTotal.slice(topN).reduce(accumulateHits, defaultHit);
+    if (otherHits.total) {
+      result.push(otherHits);
+    }
+
+    const topHits: LogHits[] = hitsByTotal.slice(0, topN);
+    if (topHits.length) {
+      result.push(...topHits);
+    }
+
+    return result;
+  };
+
   const fetchLogHits = useCallback(async (period: TimeParams) => {
     abortControllerRef.current.abort();
     abortControllerRef.current = new AbortController();
@@ -59,7 +102,7 @@ export const useFetchLogHits = (server: string, query: string) => {
         setError(error);
       }
 
-      setLogHits(!hits ? [] : hits);
+      setLogHits(!hits ? [] : getHitsWithTop(hits));
     } catch (e) {
       if (e instanceof Error && e.name !== "AbortError") {
         setError(String(e));
@@ -68,7 +111,7 @@ export const useFetchLogHits = (server: string, query: string) => {
       }
     }
     setIsLoading(prev => ({ ...prev, [id]: false }));
-  }, [url, query]);
+  }, [url, query, searchParams]);
 
   return {
     logHits,

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogs.ts

@@ -3,8 +3,11 @@ import { getLogsUrl } from "../../../api/logs";
 import { ErrorTypes, TimeParams } from "../../../types";
 import { Logs } from "../../../api/types";
 import dayjs from "dayjs";
+import { useSearchParams } from "react-router-dom";
 
 export const useFetchLogs = (server: string, query: string, limit: number) => {
+  const [searchParams] = useSearchParams();
+
   const [logs, setLogs] = useState<Logs[]>([]);
   const [isLoading, setIsLoading] = useState<{[key: number]: boolean;}>([]);
   const [error, setError] = useState<ErrorTypes | string>();
@@ -16,7 +19,9 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
     signal,
     method: "POST",
     headers: {
-      "Accept": "application/stream+json",
+      Accept: "application/stream+json",
+      AccountID: searchParams.get("accountID") || "0",
+      ProjectID: searchParams.get("projectID") || "0",
     },
     body: new URLSearchParams({
       query: query.trim(),
@@ -69,7 +74,7 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
       }
       return false;
     }
-  }, [url, query, limit]);
+  }, [url, query, limit, searchParams]);
 
   return {
     logs,

app/vmui/packages/vmui/src/state/logsPanel/LogsStateContext.tsx

@@ -0,0 +1,24 @@
+import React, { createContext, FC, useContext, useMemo, useReducer } from "preact/compat";
+import { LogsAction, LogsState, initialLogsState, reducer } from "./reducer";
+import { Dispatch } from "react";
+
+type LogsStateContextType = { state: LogsState, dispatch: Dispatch<LogsAction> };
+
+export const LogsStateContext = createContext<LogsStateContextType>({} as LogsStateContextType);
+
+export const useLogsState = (): LogsState => useContext(LogsStateContext).state;
+export const useLogsDispatch = (): Dispatch<LogsAction> => useContext(LogsStateContext).dispatch;
+
+export const LogsStateProvider: FC = ({ children }) => {
+  const [state, dispatch] = useReducer(reducer, initialLogsState);
+
+  const contextValue = useMemo(() => {
+    return { state, dispatch };
+  }, [state, dispatch]);
+
+  return <LogsStateContext.Provider value={contextValue}>
+    {children}
+  </LogsStateContext.Provider>;
+};
+
+

app/vmui/packages/vmui/src/state/logsPanel/reducer.ts

@@ -0,0 +1,26 @@
+import { getFromStorage, saveToStorage } from "../../utils/storage";
+
+export interface LogsState {
+  markdownParsing: boolean;
+}
+
+export type LogsAction =
+  | { type: "SET_MARKDOWN_PARSING", payload: boolean }
+
+
+export const initialLogsState: LogsState = {
+  markdownParsing: getFromStorage("LOGS_MARKDOWN") === "true",
+};
+
+export function reducer(state: LogsState, action: LogsAction): LogsState {
+  switch (action.type) {
+    case "SET_MARKDOWN_PARSING":
+      saveToStorage("LOGS_MARKDOWN", `${ action.payload}`);
+      return {
+        ...state,
+        markdownParsing: action.payload
+      };
+    default:
+      throw new Error();
+  }
+}

app/vmui/packages/vmui/src/utils/regexp.ts

@@ -8,6 +8,6 @@ export const escapeDoubleQuotes = (s: string) => {
 };
 
 export const hasUnclosedQuotes = (str: string) => {
-  const matches = str.match(/"/g);
+  const matches = str.match(/["`']/g);
   return matches ? matches.length % 2 !== 0 : false;
 };

app/vmui/packages/vmui/src/utils/uplot/bars.ts

@@ -1,14 +0,0 @@
-import uPlot from "uplot";
-import { LOGS_BARS_VIEW } from "../../constants/logs";
-
-export const barPaths = (
-  u: uPlot,
-  seriesIdx: number,
-  idx0: number,
-  idx1: number,
-): uPlot.Series.Paths | null => {
-  const barSize = (u.under.clientWidth/LOGS_BARS_VIEW ) - 1;
-  const barsPathBuilderFactory = uPlot?.paths?.bars?.({ size: [0.96, barSize] });
-  return barsPathBuilderFactory ? barsPathBuilderFactory(u, seriesIdx, idx0, idx1) : null;
-};
-

app/vmui/packages/vmui/src/utils/uplot/paths.ts

@@ -0,0 +1,38 @@
+import uPlot, { Series } from "uplot";
+import { LOGS_BARS_VIEW } from "../../constants/logs";
+import { GRAPH_STYLES } from "../../components/Chart/BarHitsChart/types";
+
+const barPaths = (
+  u: uPlot,
+  seriesIdx: number,
+  idx0: number,
+  idx1: number,
+): Series.Paths | null => {
+  const barSize = (u.under.clientWidth/LOGS_BARS_VIEW ) - 1;
+  const pathBuilderFactory = uPlot?.paths?.bars?.({ size: [0.96, barSize] });
+  return pathBuilderFactory ? pathBuilderFactory(u, seriesIdx, idx0, idx1) : null;
+};
+
+const lineSteppedPaths = (
+  u: uPlot,
+  seriesIdx: number,
+  idx0: number,
+  idx1: number,
+): Series.Paths | null => {
+  const pathBuilderFactory = uPlot?.paths?.stepped?.({ align: 1 });
+  return pathBuilderFactory ? pathBuilderFactory(u, seriesIdx, idx0, idx1) : null;
+};
+
+const getSeriesPaths = (type?: GRAPH_STYLES) => {
+  switch (type) {
+    case GRAPH_STYLES.BAR:
+      return barPaths;
+    case GRAPH_STYLES.LINE_STEPPED:
+      return lineSteppedPaths;
+    default:
+      return;
+  }
+};
+
+export default getSeriesPaths;
+

app/vmui/packages/vmui/src/utils/uplot/stack.ts

@@ -0,0 +1,33 @@
+// taken from https://github.com/leeoniya/uPlot/blob/master/demos/stack.js
+
+import { AlignedData, Band } from "uplot";
+
+function stack(data: AlignedData, omit: (i: number) => boolean) {
+  const data2 = [];
+  let bands = [];
+  const d0Len = data[0].length;
+  const accum = Array(d0Len);
+
+  for (let i = 0; i < d0Len; i++)
+    accum[i] = 0;
+
+  for (let i = 1; i < data.length; i++)
+    data2.push(omit(i) ? data[i] : data[i].map((v, i) => (accum[i] += +(v ?? 0))));
+
+  for (let i = 1; i < data.length; i++)
+    !omit(i) && bands.push({
+      series: [
+        data.findIndex((_s, j) => j > i && !omit(j)),
+        i,
+      ],
+    });
+
+  bands = bands.filter(b => b.series[1] > -1);
+
+  return {
+    data: [data[0]].concat(data2) as AlignedData,
+    bands: bands as Band[],
+  };
+}
+
+export default stack;

dashboards/vm/vmagent.json

@@ -3548,7 +3548,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3564,7 +3565,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 13
+            "y": 37
           },
           "id": 48,
           "options": {
@@ -3654,7 +3655,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3670,7 +3672,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 13
+            "y": 37
           },
           "id": 76,
           "options": {
@@ -3758,7 +3760,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3774,7 +3777,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 44
           },
           "id": 132,
           "options": {
@@ -3864,7 +3867,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3880,7 +3884,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 44
           },
           "id": 133,
           "options": {
@@ -3969,7 +3973,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3985,7 +3990,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 51
           },
           "id": 20,
           "options": {
@@ -4073,7 +4078,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4089,7 +4095,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 51
           },
           "id": 126,
           "options": {
@@ -4176,7 +4182,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4192,7 +4199,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 59
           },
           "id": 46,
           "options": {
@@ -4230,6 +4237,110 @@
           "title": "Scrape response size 0.99 quantile ($instance)",
           "type": "timeseries"
         },
+        {
+          "datasource": {
+            "type": "victoriametrics-datasource",
+            "uid": "$ds"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 59
+          },
+          "id": 148,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.2.6",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)",
+              "format": "time_series",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Scrape duration 0.99 quantile ($instance)",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "victoriametrics-datasource",
@@ -4279,7 +4390,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4295,7 +4407,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 67
           },
           "id": 31,
           "options": {
@@ -4575,8 +4687,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4680,8 +4791,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4798,8 +4908,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4934,8 +5043,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5037,8 +5145,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5134,8 +5241,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5238,8 +5344,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5349,8 +5454,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5447,8 +5551,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5545,8 +5648,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5693,8 +5795,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5798,8 +5899,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5903,8 +6003,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6008,8 +6107,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6112,8 +6210,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",
@@ -6315,8 +6412,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",

dashboards/vmagent.json

@@ -3547,7 +3547,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3563,7 +3564,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 13
+            "y": 37
           },
           "id": 48,
           "options": {
@@ -3653,7 +3654,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3669,7 +3671,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 13
+            "y": 37
           },
           "id": 76,
           "options": {
@@ -3757,7 +3759,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3773,7 +3776,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 20
+            "y": 44
           },
           "id": 132,
           "options": {
@@ -3863,7 +3866,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3879,7 +3883,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 20
+            "y": 44
           },
           "id": 133,
           "options": {
@@ -3968,7 +3972,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -3984,7 +3989,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 27
+            "y": 51
           },
           "id": 20,
           "options": {
@@ -4072,7 +4077,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4088,7 +4094,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 27
+            "y": 51
           },
           "id": 126,
           "options": {
@@ -4175,7 +4181,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4191,7 +4198,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 35
+            "y": 59
           },
           "id": 46,
           "options": {
@@ -4229,6 +4236,110 @@
           "title": "Scrape response size 0.99 quantile ($instance)",
           "type": "timeseries"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 59
+          },
+          "id": 148,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "9.2.6",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "expr": "max(histogram_quantile(0.99, sum(rate(vm_promscrape_scrape_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, vmrange))) by(job)",
+              "format": "time_series",
+              "interval": "",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Scrape duration 0.99 quantile ($instance)",
+          "type": "timeseries"
+        },
         {
           "datasource": {
             "type": "prometheus",
@@ -4278,7 +4389,8 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                   },
                   {
                     "color": "red",
@@ -4294,7 +4406,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 35
+            "y": 67
           },
           "id": 31,
           "options": {
@@ -4574,8 +4686,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4679,8 +4790,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4797,8 +4907,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -4933,8 +5042,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5036,8 +5144,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5133,8 +5240,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5237,8 +5343,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5348,8 +5453,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5446,8 +5550,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5544,8 +5647,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5692,8 +5794,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5797,8 +5898,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -5902,8 +6002,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6007,8 +6106,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "green",
-                    "value": null
+                    "color": "green"
                   },
                   {
                     "color": "red",
@@ -6111,8 +6209,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",
@@ -6314,8 +6411,7 @@
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "transparent",
-                    "value": null
+                    "color": "transparent"
                   },
                   {
                     "color": "red",

deployment/docker/docker-compose-cluster.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.102.0
+    image: victoriametrics/vmagent:v1.102.1
     depends_on:
       - "vminsert"
     ports:
@@ -39,7 +39,7 @@ services:
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
     container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.102.0-cluster
+    image: victoriametrics/vmstorage:v1.102.1-cluster
     ports:
       - 8482
       - 8400
@@ -51,7 +51,7 @@ services:
     restart: always
   vmstorage-2:
     container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.102.0-cluster
+    image: victoriametrics/vmstorage:v1.102.1-cluster
     ports:
       - 8482
       - 8400
@@ -66,7 +66,7 @@ services:
   # pre-process them and distributes across configured vmstorage shards.
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.102.0-cluster
+    image: victoriametrics/vminsert:v1.102.1-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -81,7 +81,7 @@ services:
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
     container_name: vmselect-1
-    image: victoriametrics/vmselect:v1.102.0-cluster
+    image: victoriametrics/vmselect:v1.102.1-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -94,7 +94,7 @@ services:
     restart: always
   vmselect-2:
     container_name: vmselect-2
-    image: victoriametrics/vmselect:v1.102.0-cluster
+    image: victoriametrics/vmselect:v1.102.1-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -112,7 +112,7 @@ services:
   # It can be used as an authentication proxy.
   vmauth:
     container_name: vmauth
-    image: victoriametrics/vmauth:v1.102.0
+    image: victoriametrics/vmauth:v1.102.1
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -127,7 +127,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.102.0
+    image: victoriametrics/vmalert:v1.102.1
     depends_on:
       - "vmauth"
     ports:

deployment/docker/docker-compose-victorialogs.yml

@@ -55,7 +55,7 @@ services:
   # scraping, storing metrics and serve read requests.
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.102.0
+    image: victoriametrics/victoria-metrics:v1.102.1
     ports:
       - 8428:8428
     volumes:

deployment/docker/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.102.0
+    image: victoriametrics/vmagent:v1.102.1
     depends_on:
       - "victoriametrics"
     ports:
@@ -22,7 +22,7 @@ services:
   # storing metrics and serve read requests.
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.102.0
+    image: victoriametrics/victoria-metrics:v1.102.1
     ports:
       - 8428:8428
       - 8089:8089
@@ -65,7 +65,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.102.0
+    image: victoriametrics/vmalert:v1.102.1
     depends_on:
       - "victoriametrics"
       - "alertmanager"

docs/Articles.md

@@ -1,5 +1,4 @@
 ---
-sort: 29
 weight: 29
 title: Articles
 menu:
@@ -50,7 +49,7 @@ See also [case studies](https://docs.victoriametrics.com/casestudies/).
 * [Percona: Percona monitoring and management migration from Prometheus to VictoriaMetrics FAQ](https://www.percona.com/blog/2020/12/16/percona-monitoring-and-management-migration-from-prometheus-to-victoriametrics-faq/)
 * [Percona: Compiling a Percona Monitoring and Management v2 Client in ARM: Raspberry Pi 3 Reprise](https://www.percona.com/blog/2021/05/26/compiling-a-percona-monitoring-and-management-v2-client-in-arm-raspberry-pi-3/)
 * [Percona: Tame Kubernetes Costs with Percona Monitoring and Management and Prometheus Operator](https://www.percona.com/blog/2021/02/12/tame-kubernetes-costs-with-percona-monitoring-and-management-and-prometheus-operator/)
-* [Making peace with Prometheus rate()](https://blog.doit-intl.com/making-peace-with-prometheus-rate-43a3ea75c4cf)
+* [Making peace with Prometheus rate()](https://www.doit.com/making-peace-with-prometheus-rate/)
 * [Disk usage: VictoriaMetrics vs Prometheus](https://stas.starikevich.com/posts/disk-usage-for-vm-versus-prometheus/)
 * [Benchmarking time series workloads on Apache Kudu using TSBS](https://blog.cloudera.com/benchmarking-time-series-workloads-on-apache-kudu-using-tsbs/)
 * [What are Open Source Time Series Databases?](https://www.iunera.com/kraken/fabric/time-series-database/)
@@ -59,12 +58,12 @@ See also [case studies](https://docs.victoriametrics.com/casestudies/).
 * [Calculating the Error of Quantile Estimation with Histograms](https://linuxczar.net/blog/2020/08/13/histogram-error/)
 * [Monitoring private clouds with VictoriaMetrics at LeroyMerlin](https://www.youtube.com/watch?v=74swsWqf0Uc)
 * [Monitoring Kubernetes with VictoriaMetrics+Prometheus](https://speakerdeck.com/bo0km4n/victoriametrics-plus-prometheusdegou-zhu-surufu-shu-kubernetesfalsejian-shi-ji-pan)
-* [High-performance Graphite storage solution on top of VictoriaMetrics](https://golangexample.com/a-high-performance-graphite-storage-solution/)
+* [High-performance Graphite storage solution on top of VictoriaMetrics](https://github.com/zhihu/promate)
 * [Cloud Native Model Driven Telemetry Stack on OpenShift](https://cer6erus.medium.com/cloud-native-model-driven-telemetry-stack-on-openshift-80712621f5bc)
 * [Prometheus VictoriaMetrics On AWS ECS](https://dalefro.medium.com/prometheus-victoria-metrics-on-aws-ecs-62448e266090)
 * [Solving Metrics at scale with VictoriaMetrics](https://www.youtube.com/watch?v=QgLMztnj7-8)
 * [Monitoring as Code на базе VictoriaMetrics и Grafana](https://habr.com/ru/post/568090/)
-* [Push Prometheus metrics to VictoriaMetrics or other exporters](https://pythonawesome.com/push-prometheus-metrics-to-victoriametrics-or-other-exporters/)
+* [Push Prometheus metrics to VictoriaMetrics or other exporters](https://github.com/gistart/prometheus-push-client)
 * [Install and configure VictoriaMetrics on Debian](https://www.vultr.com/docs/install-and-configure-victoriametrics-on-debian)
 * [Superset BI with Victoria Metrics](https://cer6erus.medium.com/superset-bi-with-victoria-metrics-a109d3e91bc6)
 * [VictoriaMetrics Source Code Analysis - Bloom filter](https://www.sobyte.net/post/2022-05/victoriametrics-bloomfilter/)
@@ -85,6 +84,7 @@ See also [case studies](https://docs.victoriametrics.com/casestudies/).
 * [Supercharge your Monitoring: Migrate from Prometheus to VictoriaMetrics for optimized CPU and Memory usage - Part 2](https://zetablogs.medium.com/part-2-supercharge-your-monitoring-migrate-from-prometheus-to-victoriametrics-for-optimised-cpu-9a90c015ccba)
 * [Persistent Data Structures in VictoriaMetrics (Part 1): vmagent](https://medium.com/devops-dev/persistent-data-structures-in-victoriametrics-part-1-vmagent-2e9c7681a6f0)
 * [Persistent Data Structures in VictoriaMetrics (Part 2): vmselect](https://medium.com/@jiekun/persistent-data-structures-in-victoriametrics-part-2-vmselect-9e3de39a4d20)
+* [Migrating to VictoriaMetrics (by Zomato): A Complete Overhaul for Enhanced Observability](https://blog.zomato.com/migrating-to-victoriametrics-a-complete-overhaul-for-enhanced-observability)
 
 ## Our articles
 

docs/BestPractices.md

@@ -1,9 +1,9 @@
 ---
-sort: 32
 weight: 32
-title: VictoriaMetrics best practices
+title: Best practices
 menu:
   docs:
+    identifier: vm-best-practices
     parent: 'victoriametrics'
     weight: 32
 aliases: