add changelog

Makefile

@@ -513,19 +513,19 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test: victoria-metrics vmagent vmalert vmauth
 	go test ./apptest/... -skip="^TestCluster.*"

app/vlinsert/insertutils/common_params.go

@@ -40,40 +40,6 @@ type CommonParams struct {
 	DebugRemoteAddr string
 }
 
-// GetJaegerCommonParams returns CommonParams for Jaeger.
-func GetJaegerCommonParams() (*CommonParams, error) {
-	// Extract tenantID
-	var tenantID = logstorage.TenantID{
-		AccountID: 0,
-		ProjectID: 0,
-	}
-	timeField := "_time"
-
-	msgFields := []string{"_msg"}
-	streamFields := []string{"_stream"}
-	ignoreFields := []string{"_ignore"}
-
-	extraFields := []logstorage.Field{}
-
-	debug := false
-	debugRequestURI := ""
-	debugRemoteAddr := ""
-
-	cp := &CommonParams{
-		TenantID:        tenantID,
-		TimeField:       timeField,
-		MsgFields:       msgFields,
-		StreamFields:    streamFields,
-		IgnoreFields:    ignoreFields,
-		ExtraFields:     extraFields,
-		Debug:           debug,
-		DebugRequestURI: debugRequestURI,
-		DebugRemoteAddr: debugRemoteAddr,
-	}
-
-	return cp, nil
-}
-
 // GetCommonParams returns CommonParams from r.
 func GetCommonParams(r *http.Request) (*CommonParams, error) {
 	// Extract tenantID
@@ -233,8 +199,8 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 	lmp.bytesIngestedTotal.Add(n)
 
 	if len(fields) > *MaxFieldsPerLine {
-		rf := logstorage.RowFormatter(fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		line := logstorage.MarshalFieldsToJSON(nil, fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
 		rowsDroppedTotalTooManyFields.Inc()
 		return
 	}

app/vlinsert/insertutils/flags.go

@@ -8,8 +8,10 @@ import (
 
 var (
 	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
 
 	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
 )

app/vlinsert/insertutils/timestamp.go

@@ -2,20 +2,19 @@ package insertutils
 
 import (
 	"fmt"
-	"math"
 	"strconv"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+// ExtractTimestampFromFields extracts timestamp in nanoseconds from the field with the name timeField at fields.
 //
 // The value for the timeField is set to empty string after returning from the function,
 // so it could be ignored during data ingestion.
 //
 // The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+func ExtractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
 	for i := range fields {
 		f := &fields[i]
 		if f.Name != timeField {
@@ -48,22 +47,24 @@ func parseTimestamp(s string) (int64, error) {
 	return nsecs, nil
 }
 
-// ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.
+// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
 func ParseUnixTimestamp(s string) (int64, error) {
 	n, err := strconv.ParseInt(s, 10, 64)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
 	}
 	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds. Convert it to milliseconds
-		n *= 1e3
+		// The timestamp is in seconds.
+		return n * 1e9, nil
 	}
-	if n > int64(math.MaxInt64)/1e6 {
-		return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
+		// The timestamp is in milliseconds.
+		return n * 1e6, nil
 	}
-	if n < int64(math.MinInt64)/1e6 {
-		return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
+		// The timestamp is in microseconds.
+		return n * 1e3, nil
 	}
-	n *= 1e6
+	// The timestamp is in nanoseconds
 	return n, nil
 }

app/vlinsert/insertutils/timestamp_test.go

@@ -6,11 +6,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
+func TestExtractTimestampFromFields_Success(t *testing.T) {
 	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
 		t.Helper()
 
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
+		nsecs, err := ExtractTimestampFromFields(timeField, fields)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -51,6 +51,18 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 		{Name: "foo", Value: "bar"},
 	}, 1718773640123456789)
 
+	// Unix timestamp in nanoseconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640123456789"},
+	}, 1718773640123456789)
+
+	// Unix timestamp in microseconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640123456"},
+	}, 1718773640123456000)
+
 	// Unix timestamp in milliseconds
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
@@ -64,14 +76,14 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 	}, 1718773640000000000)
 }
 
-func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
+func TestExtractTimestampFromFields_Error(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 
 		fields := []logstorage.Field{
 			{Name: "time", Value: s},
 		}
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
+		nsecs, err := ExtractTimestampFromFields("time", fields)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -80,6 +92,7 @@ func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
 		}
 	}
 
+	// invalid time
 	f("foobar")
 
 	// incomplete time

app/vlinsert/jaeger/jaeger.go

@@ -1,45 +0,0 @@
-package jaeger
-
-import (
-	"fmt"
-	"github.com/jaegertracing/jaeger/storage/dependencystore"
-	"github.com/jaegertracing/jaeger/storage/spanstore"
-	"log"
-	"net"
-
-	jaeger2 "github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/jaeger"
-	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
-
-	"google.golang.org/grpc"
-)
-
-// MustInit - init for Jaeger gRPC storage backend
-func MustInit() {
-	lis, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", 17271))
-	if err != nil {
-		log.Fatalf("failed to listen: %v", err)
-	}
-	var opts []grpc.ServerOption
-	grpcServer := grpc.NewServer(opts...)
-
-	handler := shared.NewGRPCHandler(&shared.GRPCHandlerStorageImpl{
-		SpanReader:          func() spanstore.Reader { return &jaeger2.SpanReaderPluginServer{} },
-		SpanWriter:          func() spanstore.Writer { return &SpanWriterPluginServer{} },
-		DependencyReader:    func() dependencystore.Reader { return &jaeger2.SpanReaderPluginServer{} },
-		ArchiveSpanReader:   func() spanstore.Reader { return nil },
-		ArchiveSpanWriter:   func() spanstore.Writer { return nil },
-		StreamingSpanWriter: func() spanstore.Writer { return nil },
-	})
-
-	//proto.RegisterSpanWriterPluginServer(grpcServer, &SpanWriterPluginServer{})
-	//proto.RegisterSpanReaderPluginServer(grpcServer, &jaeger2.SpanReaderPluginServer{})
-	err = handler.Register(grpcServer)
-	if err != nil {
-		panic("unable to register Jaeger gRPC handler: " + err.Error())
-	}
-
-	go grpcServer.Serve(lis)
-}
-
-// MustStop - stop for Jaeger gRPC storage backend
-func MustStop() {}

app/vlinsert/jaeger/span_writer_plugin_server.go

@@ -1,44 +0,0 @@
-package jaeger
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/jaegertracing/jaeger/model"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jaeger"
-)
-
-// A SpanWriterPluginServer represents plugin Jaeger interface to write gRPC storage backend
-type SpanWriterPluginServer struct {
-}
-
-var bbufPool bytesutil.ByteBufferPool
-
-// WriteSpan writes spans
-func (s *SpanWriterPluginServer) WriteSpan(ctx context.Context, span *model.Span) error {
-	if span == nil {
-		return fmt.Errorf("span not found")
-	}
-
-	cp, err := insertutils.GetJaegerCommonParams()
-	if err != nil {
-		return err
-	}
-	lmp := cp.NewLogMessageProcessor("jaeger")
-	defer lmp.MustClose()
-	// bytes buf here
-	bbuf := bbufPool.Get()
-	defer bbufPool.Put(bbuf)
-	bbuf.Reset()
-	fields, streamFields, err := jaeger.SpanToFields(span)
-	lmp.AddRow(span.StartTime.UnixNano(), fields, streamFields)
-	// bytes bf clear
-	return nil
-}
-
-func (s *SpanWriterPluginServer) Close() error {
-	return nil
-}

app/vlinsert/jaeger/test_client/grpc_client_test.go

@@ -1,42 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jaeger/proto"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-	"testing"
-)
-
-func newSpanReaderPluginClient(t *testing.T) proto.SpanReaderPluginClient {
-	conn, err := grpc.NewClient(fmt.Sprintf("0.0.0.0:%d", 17271), grpc.WithTransportCredentials(insecure.NewCredentials()))
-	if err != nil {
-		t.Fatalf("cannot connect to server: %v", err)
-	}
-	return proto.NewSpanReaderPluginClient(conn)
-}
-
-func newSpanWriterPluginClient(t *testing.T) proto.SpanWriterPluginClient {
-	conn, err := grpc.NewClient(fmt.Sprintf("0.0.0.0:%d", 17271), grpc.WithTransportCredentials(insecure.NewCredentials()))
-	if err != nil {
-		t.Fatalf("cannot connect to server: %v", err)
-	}
-	return proto.NewSpanWriterPluginClient(conn)
-}
-
-func TestSpanWriter(t *testing.T) {
-	// This is NOT a unit test. Please run the VictoriaLogs before executing this test.
-	client := newSpanWriterPluginClient(t)
-	req := &proto.WriteSpanRequest{}
-	resp, err := client.WriteSpan(context.Background(), req)
-	fmt.Println(resp, err)
-}
-
-func TestSpanReaderGetOperations(t *testing.T) {
-	// This is NOT a unit test. Please run the VictoriaLogs before executing this test.
-	client := newSpanReaderPluginClient(t)
-	req := &proto.GetOperationsRequest{}
-	resp, err := client.GetOperations(context.Background(), req)
-	fmt.Println(resp, err)
-}

app/vlinsert/jsonline/jsonline.go

@@ -99,7 +99,7 @@ func readLine(lr *insertutils.LineReader, timeField string, msgFields []string,
 	if err := p.ParseLogMessage(line); err != nil {
 		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
 	}
-	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
+	ts, err := insertutils.ExtractTimestampFromFields(timeField, p.Fields)
 	if err != nil {
 		return false, fmt.Errorf("cannot get timestamp: %w", err)
 	}

app/vlinsert/main.go

@@ -2,7 +2,6 @@ package vlinsert
 
 import (
 	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jaeger"
 	"net/http"
 	"strings"
 
@@ -18,13 +17,11 @@ import (
 // Init initializes vlinsert
 func Init() {
 	syslog.MustInit()
-	jaeger.MustInit()
 }
 
 // Stop stops vlinsert
 func Stop() {
 	syslog.MustStop()
-	jaeger.MustStop()
 }
 
 // RequestHandler handles insert requests for VictoriaLogs

app/vlinsert/syslog/syslog.go

@@ -560,7 +560,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	if useLocalTimestamp {
 		ts = time.Now().UnixNano()
 	} else {
-		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
+		nsecs, err := insertutils.ExtractTimestampFromFields("timestamp", p.Fields)
 		if err != nil {
 			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
 		}

app/vlogscli/main.go

@@ -270,7 +270,7 @@ func printCommandsHelp(w io.Writer) {
 \h - show this help
 \s - singleline json output mode
 \m - multiline json output mode
-\c - compact output
+\c - compact output mode
 \logfmt - logfmt output mode
 \wrap_long_lines - toggles wrapping long lines
 \tail <query> - live tail <query> results

app/vlselect/jaeger/span_reader_plugin_server.go

@@ -1,293 +0,0 @@
-package jaeger
-
-import (
-	"context"
-	"fmt"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jaeger"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/jaegertracing/jaeger/model"
-	"github.com/jaegertracing/jaeger/storage/spanstore"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-)
-
-// A SpanReaderPluginServer represents a Jaeger interface to read from gRPC storage backend
-type SpanReaderPluginServer struct{}
-
-type row struct {
-	timestamp int64
-	fields    []logstorage.Field
-}
-
-func (s *SpanReaderPluginServer) GetTrace(ctx context.Context, traceID model.TraceID) (*model.Trace, error) {
-	start := time.Now()
-	defer func() {
-		logger.Infof("GetTrace finished in %dms", time.Since(start).Milliseconds())
-	}()
-	qStr := fmt.Sprintf("%s:%s", jaeger.TraceID, traceID.String())
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, time.Now().UnixNano())
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
-	}
-
-	var rows []row
-	var rowsLock sync.Mutex
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
-
-		for i, timestamp := range timestamps {
-			fields := make([]logstorage.Field, len(columns))
-			for j := range columns {
-				if columns[j].Values[i] != "" {
-					fields = append(fields, logstorage.Field{Name: clonedColumnNames[j], Value: strings.Clone(columns[j].Values[i])})
-				}
-			}
-
-			rowsLock.Lock()
-			rows = append(rows, row{
-				timestamp: timestamp,
-				fields:    fields,
-			})
-			rowsLock.Unlock()
-		}
-	}
-	logger.Infof("GetTrace query: %s", q.String())
-	if err = vlstorage.RunQuery(context.TODO(), []logstorage.TenantID{{AccountID: 0, ProjectID: 0}}, q, writeBlock); err != nil {
-		return nil, err
-	}
-
-	spans := make([]*model.Span, 0, len(rows))
-	for i := range rows {
-		sp, err := jaeger.FieldsToSpan(rows[i].fields)
-		if err != nil {
-			logger.Errorf("cannot unmarshal log fields [%v] to span: %s", rows[i].fields, err)
-			continue
-		}
-		spans = append(spans, sp)
-	}
-	trace := &model.Trace{
-		Spans: spans,
-	}
-	return trace, nil
-}
-
-func (s *SpanReaderPluginServer) GetServices(ctx context.Context) ([]string, error) {
-	start := time.Now()
-	defer func() {
-		logger.Infof("GetServices finished in %dms", time.Since(start).Milliseconds())
-	}()
-	qStr := "*"
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, time.Now().UnixNano())
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
-	}
-	q.AddTimeFilter(0, time.Now().UnixNano())
-	logger.Infof("GetServices StreamFieldValues query: %s", q.String())
-	serviceHits, err := vlstorage.GetStreamFieldValues(ctx, []logstorage.TenantID{{AccountID: 0, ProjectID: 0}}, q, jaeger.ProcessServiceName, uint64(1000))
-	if err != nil {
-		return nil, err
-	}
-
-	serviceList := make([]string, 0)
-	for i := range serviceHits {
-		serviceList = append(serviceList, serviceHits[i].Value)
-	}
-
-	return serviceList, nil
-}
-
-func (s *SpanReaderPluginServer) GetOperations(ctx context.Context, req spanstore.OperationQueryParameters) ([]spanstore.Operation, error) {
-	start := time.Now()
-	defer func() {
-		logger.Infof("GetOperations finished in %dms", time.Since(start).Milliseconds())
-	}()
-	qStr := fmt.Sprintf("_stream:{%s=\"%s\"}", jaeger.ProcessServiceName, req.ServiceName) // todo spankind filter
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, time.Now().UnixNano())
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
-	}
-	logger.Infof("GetOperations StreamFieldValues query: %s", q.String())
-	operationHits, err := vlstorage.GetStreamFieldValues(ctx, []logstorage.TenantID{{AccountID: 0, ProjectID: 0}}, q, jaeger.OperationName, uint64(1000))
-	if err != nil {
-		return nil, err
-	}
-
-	operationList := make([]spanstore.Operation, 0)
-	for i := range operationHits {
-		operationList = append(operationList, spanstore.Operation{Name: operationHits[i].Value})
-	}
-	return operationList, nil
-}
-
-func (s *SpanReaderPluginServer) FindTraces(ctx context.Context, query *spanstore.TraceQueryParameters) ([]*model.Trace, error) {
-	start := time.Now()
-	defer func() {
-		logger.Infof("FindTraces finished in %dms", time.Since(start).Milliseconds())
-	}()
-	traceIDs, err := s.FindTraceIDs(ctx, query)
-	if err != nil {
-		return nil, err
-	}
-	if len(traceIDs) == 0 {
-		return nil, nil
-	}
-	traceIDStrList := make([]string, 0, len(traceIDs))
-	for _, traceID := range traceIDs {
-		traceIDStrList = append(traceIDStrList, traceID.String())
-	}
-	qStr := fmt.Sprintf(jaeger.TraceID+":in(%s)", strings.Join(traceIDStrList, ","))
-
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, time.Now().UnixNano())
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
-	}
-	q.AddTimeFilter(query.StartTimeMin.UnixNano(), query.StartTimeMax.UnixNano())
-
-	var rows []row
-	var rowsLock sync.Mutex
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		clonedColumnNames := make([]string, len(columns))
-		for i, c := range columns {
-			clonedColumnNames[i] = strings.Clone(c.Name)
-		}
-
-		for i, timestamp := range timestamps {
-			fields := make([]logstorage.Field, 0, len(columns))
-			for j := range columns {
-				if columns[j].Values[i] != "" {
-					fields = append(fields, logstorage.Field{Name: clonedColumnNames[j], Value: strings.Clone(columns[j].Values[i])})
-				}
-			}
-
-			rowsLock.Lock()
-			rows = append(rows, row{
-				timestamp: timestamp,
-				fields:    fields,
-			})
-			rowsLock.Unlock()
-		}
-	}
-	logger.Infof("FindTraces query: %s", q.String())
-	if err = vlstorage.RunQuery(context.TODO(), []logstorage.TenantID{{AccountID: 0, ProjectID: 0}}, q, writeBlock); err != nil {
-		return nil, err
-	}
-	tracesMap := make(map[string]*model.Trace)
-	traces := make([]*model.Trace, len(traceIDs), len(traceIDs))
-	for i := range traceIDs {
-		traces[i] = &model.Trace{}
-		tracesMap[traceIDs[i].String()] = traces[i]
-	}
-
-	for i := range rows {
-		sp, err := jaeger.FieldsToSpan(rows[i].fields)
-		if err != nil {
-			logger.Errorf("cannot unmarshal log fields [%v] to span: %s", rows[i].fields, err)
-			continue
-		}
-
-		tracesMap[sp.TraceID.String()].Spans = append(tracesMap[sp.TraceID.String()].Spans, sp)
-	}
-	return traces, nil
-}
-
-func (s *SpanReaderPluginServer) FindTraceIDs(ctx context.Context, query *spanstore.TraceQueryParameters) ([]model.TraceID, error) {
-	start := time.Now()
-	defer func() {
-		logger.Infof("FindTraceIDs finished in %dms", time.Since(start).Milliseconds())
-	}()
-	qStr := ""
-	if svcName := query.ServiceName; svcName != "" {
-		qStr += fmt.Sprintf("AND _stream:{"+jaeger.ProcessServiceName+"=\"%s\"} ", svcName)
-	}
-	if operationName := query.OperationName; operationName != "" {
-		qStr += fmt.Sprintf("AND _stream:{"+jaeger.OperationName+"=\"%s\"} ", operationName)
-	}
-
-	if tags := query.Tags; len(tags) > 0 {
-		for k, v := range tags {
-			qStr += fmt.Sprintf(`AND "`+jaeger.TagKey+`":=%s `, k, v)
-		}
-	}
-	if durationMin := query.DurationMin; durationMin > 0 {
-		qStr += fmt.Sprintf("AND "+jaeger.Duration+":>%d ", durationMin.Nanoseconds())
-	}
-	if durationMax := query.DurationMax; durationMax > 0 {
-		qStr += fmt.Sprintf("AND duration:<%d ", durationMax.Nanoseconds())
-	}
-	qStr = strings.TrimLeft(qStr+" | last 1 by (_time) partition by ("+jaeger.TraceID+") | fields _time, "+jaeger.TraceID+" | sort by (_time) desc", "AND ")
-
-	logger.Infof("FindTraceIDs query debug: %s", qStr)
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, query.StartTimeMax.UnixNano())
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse query [%s]: %s", qStr, err)
-	}
-	q.AddPipeLimit(uint64(query.NumTraces))
-
-	traceIDSs, err := findTraceIDsSplitTimeRange(q, query.StartTimeMin, query.StartTimeMax, query.NumTraces)
-	if err != nil {
-		return nil, err
-	}
-
-	traceIDList := make([]model.TraceID, 0, query.NumTraces)
-	for _, v := range traceIDSs {
-		tid, err := model.TraceIDFromString(v)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unmarshal [%s]: %s", v, err)
-		}
-		traceIDList = append(traceIDList, tid)
-	}
-	return traceIDList, nil
-}
-
-// findTraceIDsSplitTimeRange try to search from the nearest time range of the end time.
-// if the result already met requirement of `limit`, return.
-// otherwise, amplify the time range to 5x and search again, until the start time exceed the input.
-func findTraceIDsSplitTimeRange(q *logstorage.Query, startTime, endTime time.Time, limit int) ([]string, error) {
-	step := time.Minute
-	startTimeCurrent := endTime.Add(-step)
-	traceIDList := make([]string, 0, 10)
-	writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
-		for i := range columns {
-			if columns[i].Name == "trace_id" {
-				for _, v := range columns[i].Values {
-					traceIDList = append(traceIDList, v)
-				}
-
-			}
-		}
-	}
-
-	for startTimeCurrent.After(startTime) {
-		qClone := q.CloneWithTimeFilter(endTime.UnixNano(), startTimeCurrent.UnixNano(), endTime.UnixNano())
-		logger.Infof("FindTraces query: %s", qClone.String())
-		if err := vlstorage.RunQuery(context.TODO(), []logstorage.TenantID{{AccountID: 0, ProjectID: 0}}, qClone, writeBlock); err != nil {
-			return nil, err
-		}
-		if len(traceIDList) == limit {
-			return traceIDList, nil
-		}
-		traceIDList = traceIDList[:0]
-		step *= 5
-		startTimeCurrent = startTimeCurrent.Add(-step)
-	}
-
-	// one last try with input time range
-	qClone := q.CloneWithTimeFilter(endTime.UnixNano(), startTimeCurrent.UnixNano(), endTime.UnixNano())
-	logger.Infof("FindTraces query: %s", qClone.String())
-	if err := vlstorage.RunQuery(context.TODO(), []logstorage.TenantID{{AccountID: 0, ProjectID: 0}}, qClone, writeBlock); err != nil {
-		return nil, err
-	}
-	return traceIDList, nil
-}
-
-func (s *SpanReaderPluginServer) GetDependencies(ctx context.Context, endTs time.Time, lookback time.Duration) ([]model.DependencyLink, error) {
-	return []model.DependencyLink{}, nil
-}

app/vlselect/vmui/asset-manifest.json

@@ -1,12 +1,12 @@
 {
   "files": {
-    "main.css": "./static/css/main.3134e778.css",
-    "main.js": "./static/js/main.82cd6930.js",
+    "main.css": "./static/css/main.02a1c6cb.css",
+    "main.js": "./static/js/main.55c8060b.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.3134e778.css",
-    "static/js/main.82cd6930.js"
+    "static/css/main.02a1c6cb.css",
+    "static/js/main.55c8060b.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.82cd6930.js"></script><link href="./static/css/main.3134e778.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.55c8060b.js"></script><link href="./static/css/main.02a1c6cb.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmalert/remotewrite/init.go

@@ -36,7 +36,7 @@ var (
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
-	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint")
+	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
 	flushInterval = flag.Duration("remoteWrite.flushInterval", defaultFlushInterval, "Defines interval of flushes to remote write endpoint")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")

app/vmauth/main.go

@@ -31,7 +31,11 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+		"By default, serves internal API and proxy requests. "+
+		" See also -tls, -httpListenAddr.useProxyProtocol and -httpInternalListenAddr.")
+	httpInternalListenAddr = flagutil.NewArrayString("httpInternalListenAddr", "TCP address to listen for incoming internal API http requests. Such as /health, /-/reload, /debug/pprof, etc. "+
+		"If flag is set, vmauth no longer serves internal API at -httpListenAddr.")
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
@@ -91,7 +95,21 @@ func main() {
 	logger.Infof("starting vmauth at %q...", listenAddrs)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	disableInternalRoutes := len(*httpInternalListenAddr) > 0
+	rh := requestHandlerWithInternalRoutes
+	if disableInternalRoutes {
+		rh = requestHandler
+	}
+
+	serveOpts := httpserver.ServeOptions{
+		UseProxyProtocol:     useProxyProtocol,
+		DisableBuiltinRoutes: disableInternalRoutes,
+	}
+	go httpserver.ServeWithOpts(listenAddrs, rh, serveOpts)
+
+	if len(*httpInternalListenAddr) > 0 {
+		go httpserver.Serve(*httpInternalListenAddr, nil, internalRequestHandler)
+	}
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -109,7 +127,7 @@ func main() {
 	logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())
 }
 
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+func internalRequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
 		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
@@ -120,6 +138,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
+	return false
+}
+
+func requestHandlerWithInternalRoutes(w http.ResponseWriter, r *http.Request) bool {
+	if internalRequestHandler(w, r) {
+		return true
+	}
+	return requestHandler(w, r)
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	ats := getAuthTokensFromRequest(r)
 	if len(ats) == 0 {

app/vmauth/main_test.go

@@ -52,7 +52,7 @@ func TestRequestHandler(t *testing.T) {
 		r.Header.Set("Pass-Header", "abc")
 
 		w := &fakeResponseWriter{}
-		if !requestHandler(w, r) {
+		if !requestHandlerWithInternalRoutes(w, r) {
 			t.Fatalf("unexpected false is returned from requestHandler")
 		}
 

app/vmctl/flags.go

@@ -596,7 +596,8 @@ var (
 		&cli.Int64Flag{
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
-				"By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases.",
+				"By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases. \n" +
+				"Rate limit is applied per worker, see `--vm-concurrency`.",
 		},
 		&cli.BoolFlag{
 			Name: vmInterCluster,

app/vmselect/netstorage/netstorage.go

@@ -8,7 +8,6 @@ import (
 	"sort"
 	"sync"
 	"sync/atomic"
-	"time"
 	"unsafe"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -1002,9 +1001,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 
 	sr := getStorageSearch()
 	defer putStorageSearch(sr)
-	startTime := time.Now()
 	sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
-	indexSearchDuration.UpdateDuration(startTime)
 
 	// Start workers that call f in parallel on available CPU cores.
 	workCh := make(chan *exportWork, gomaxprocs*8)
@@ -1142,9 +1139,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	defer vmstorage.WG.Done()
 
 	sr := getStorageSearch()
-	startTime := time.Now()
 	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
-	indexSearchDuration.UpdateDuration(startTime)
 	type blockRefs struct {
 		brs []blockRef
 	}
@@ -1296,8 +1291,6 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	return &rss, nil
 }
 
-var indexSearchDuration = metrics.NewHistogram(`vm_index_search_duration_seconds`)
-
 type blockRef struct {
 	partRef storage.PartRef
 	addr    tmpBlockAddr

app/vmselect/prometheus/prometheus.go

@@ -29,6 +29,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -142,10 +143,13 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 		WriteFederate(bb, rs)
 		return sw.maybeFlushBuffer(bb)
 	})
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("error during sending data to remote client: %w", err)
 	}
-	return sw.flush()
+	return nil
 }
 
 var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/federate"}`)
@@ -226,10 +230,13 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 		}()
 	}
 	err = <-doneCh
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("error during sending the exported csv data to remote client: %w", err)
 	}
-	return sw.flush()
+	return nil
 }
 
 var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/csv"}`)
@@ -281,10 +288,13 @@ func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
 		bb.B = dst
 		return sw.maybeFlushBuffer(bb)
 	})
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("error during sending native data to remote client: %w", err)
 	}
-	return sw.flush()
+	return nil
 }
 
 var exportNativeDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/native"}`)
@@ -441,16 +451,19 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, cp *commonPara
 		}()
 	}
 	err := <-doneCh
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err == nil {
+		if format == "promapi" {
+			WriteExportPromAPIFooter(bw, qt)
+		}
+		err = bw.Flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("cannot send data to remote client: %w", err)
 	}
-	if err := sw.flush(); err != nil {
-		return fmt.Errorf("cannot send data to remote client: %w", err)
-	}
-	if format == "promapi" {
-		WriteExportPromAPIFooter(bw, qt)
-	}
-	return bw.Flush()
+	return nil
 }
 
 type exportBlock struct {

app/vmselect/promql/binary_op.go

@@ -483,8 +483,11 @@ func binaryOpOr(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries
 
-	for _, tss := range mLeft {
-		rvs = append(rvs, tss...)
+	for k, tss := range mLeft {
+		tssLeft := removeEmptySeries(tss)
+		// re-assign modified slice to map, since it can be referred later
+		mLeft[k] = tssLeft
+		rvs = append(rvs, tssLeft...)
 	}
 	// Sort left-hand-side series by metric name as Prometheus does.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5393
@@ -497,7 +500,10 @@ func binaryOpOr(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 			rvs = append(rvs, tssRight...)
 			continue
 		}
-		fillLeftNaNsWithRightValues(tssLeft, tssRight)
+		fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight)
+		// tssRight might be filled with NaNs after merge
+		tssRight = removeEmptySeries(tssRight)
+		rvs = append(rvs, tssRight...)
 	}
 	// Sort the added right-hand-side series by metric name as Prometheus does.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5393
@@ -526,6 +532,35 @@ func fillLeftNaNsWithRightValues(tssLeft, tssRight []*timeseries) {
 	}
 }
 
+// fill gaps in tssLeft with values from tssRight when labels match
+// Set NaNs to tssRight when tssLeft has corresponding values
+// or if tssLeft and tssRight can be merged.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759
+// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640
+func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
+	for _, tsLeft := range tssLeft {
+		valuesLeft := tsLeft.Values
+		nameLeft := tsLeft.MetricName.String()
+		for i, v := range valuesLeft {
+			leftIsNaN := math.IsNaN(v)
+			for _, tsRight := range tssRight {
+				canBeMerged := nameLeft == tsRight.MetricName.String()
+				valueRight := tsRight.Values[i]
+				if leftIsNaN && canBeMerged {
+					// fill NaNs with valueRight if labels match
+					valuesLeft[i] = valueRight
+				}
+				if !leftIsNaN || canBeMerged {
+					// set NaN to valueRight if valueLeft is not NaN
+					// or if left and right can be merged
+					tsRight.Values[i] = nan
+				}
+			}
+		}
+	}
+}
+
 func binaryOpIfnot(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries

app/vmselect/promql/exec_test.go

@@ -4461,9 +4461,9 @@ func TestExecSuccess(t *testing.T) {
 	t.Run(`histogram_quantile(nan-bucket-count-some)`, func(t *testing.T) {
 		t.Parallel()
 		q := `round(histogram_quantile(0.6,
-			label_set(90, "foo", "bar", "le", "10")
-			or label_set(NaN, "foo", "bar", "le", "30")
-			or label_set(300, "foo", "bar", "le", "+Inf")
+			union(label_set(90, "foo", "bar", "le", "10"),
+			label_set(NaN, "foo", "bar", "le", "30"),
+			label_set(300, "foo", "bar", "le", "+Inf"))
 		),0.01)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
@@ -9409,7 +9409,384 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`nan or on() series`, func(t *testing.T) {
+		t.Parallel()
+		// left side returns NaNs only, so the right side should replace its values and labels
+		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759
+		q := `(label_set(1, "a", "a", "b", "b1") == 0) or on(a) label_set(2, "a", "a", "b", "b2")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b2"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`series with NaNs or scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `(label_set(time() >= 1600, "a", "a", "b", "b1")) or 1`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
 
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() scalar`, func(t *testing.T) {
+		t.Parallel()
+		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640
+		q := `(label_set(time() > 1200, "a", "a", "b", "b1")) or on() vector(0)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() series`, func(t *testing.T) {
+		t.Parallel()
+		// left side + right side
+		q := `(label_set(time() <= 1200, "a", "a", "b", "b1")) or on(a) label_set(time() > 1200, "a", "a", "b", "b2")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b2"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series with no NaNs or on() series`, func(t *testing.T) {
+		t.Parallel()
+		// left side contains all needed values, so the right side should be dropped
+		q := `(label_set(time() < 3000, "a", "a", "b", "b1")) or on(a) label_set(time() > 3000, "a", "a", "b", "b2")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() series with overlap`, func(t *testing.T) {
+		t.Parallel()
+		// left overlap with right
+		q := `(label_set(time() <= 1500, "a", "a", "b", "b1")) or on(a) label_set(time() > 1100, "a", "a", "b", "b2")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b2"),
+		}}
+
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() series merge`, func(t *testing.T) {
+		t.Parallel()
+		// left + right for same series
+		q := `(label_set(time() <= 1200, "a", "a", "b", "b1")) or on(a) label_set(time() > 1400, "a", "a", "b", "b1")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`scalar or timeseries`, func(t *testing.T) {
+		t.Parallel()
+		q := `time() > 1400 or label_set(123, "foo", "bar")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{123, 123, 123, 123, 123, 123},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or many series`, func(t *testing.T) {
+		//load 1m
+		//    foo{a="a", b="1"} 1 0 1 1 1
+		//    bar{a="a", b="2"} 2 2 2 2 2
+		//    bar{a="a", b="3"} 3 3 3 3 3
+		//
+		//eval range from 0 to 4m step 1m foo!=0 or on (a) bar
+		//    foo{a="a", b="1"} 1 _ 1 1 1
+		//    bar{a="a", b="2"} _ 2 _ _ _
+		//    bar{a="a", b="3"} _ 3 _ _ _
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1200, "x", "foo"),
+		) or on(x) (
+			label_set(time()+1, "x", "foo", "y", "bar"),
+			label_set(time()+2, "y", "baz", "x", "foo"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1201, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+			{Key: []byte("y"), Value: []byte("bar")},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1202, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+			{Key: []byte("y"), Value: []byte("baz")},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`many series or series`, func(t *testing.T) {
+		//load 1m
+		//    foo{a="a", b="1"} 1 0 1 1 1
+		//    foo{a="a", b="2"} 2 2 2 2 2
+		//    bar{a="a", b="3"} 3 3 3 3 3
+		//
+		//eval range from 0 to 4m step 1m foo!=0 or on (a) bar
+		//    foo{a="a", b="1"} 1 _ 1 1 1
+		//    foo{a="a", b="2"} 2 2 2 2 2
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1200, "x", "foo"),
+			label_set(time()+1, "x", "foo", "y","baz"),
+		) or on(x) (
+			label_set(time()+2, "x", "foo", "y", "bar"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1001, 1201, 1401, 1601, 1801, 2001},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+			{Key: []byte("y"), Value: []byte("baz")},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`many series or series with no merge`, func(t *testing.T) {
+		//	load 1m
+		//    foo{job="a1", a="a"} 0 0 1 1 0
+		//    foo{job="a2", a="a"} 1 1 0 0 0
+		//    foo{job="a3", a="a"} 1 1 1 1 1
+		//    foo{job="a4", a="a"} 1 1 1 1 1
+		//
+		//eval range from 0 to 4m step 1m (foo{job=~"a1|a2"} == 0) or on (a) (foo{job=~"a3|a4"} == 1)
+		//    foo{job="a1", a="a"} 0 0 _ _ 0
+		//    foo{job="a2", a="a"} _ _ 0 0 0
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1400, "job", "a1", "a", "a"),
+			label_set(time()>=1400, "job", "a2", "a", "a"),
+		) or on(a) (
+			label_set(time(), "job", "a3", "a", "a"),
+			label_set(time(), "job", "a4", "a", "a"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a1")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a2")},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`many series or series with merge`, func(t *testing.T) {
+		//	load 1m
+		//    foo{job="a1", a="a"} 0 0 1 1 0
+		//    foo{job="a2", a="a"} 1 1 1 0 0
+		//    foo{job="a3", a="a"} 1 1 1 1 1
+		//    foo{job="a4", a="a"} 1 1 1 1 1
+		//
+		//eval range from 0 to 4m step 1m (foo{job=~"a1|a2"} == 0) or on (a) (foo{job=~"a3|a4"} == 1)
+		//    foo{job="a1", a="a"} 0 0 _ _ 0
+		//    foo{job="a2", a="a"} _ _ _ 0 0
+		//    foo{job="a3", a="a"} _ _ 1 _ _
+		//    foo{job="a4", a="a"} _ _ 1 _ _
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1400, "job", "a1", "a", "a"),
+			label_set(time()>=1600, "job", "a2", "a", "a"),
+		) or on(a) (
+			label_set(time(), "job", "a3", "a", "a"),
+			label_set(time(), "job", "a4", "a", "a"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a1")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a2")},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a3")},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a4")},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		f(q, resultExpected)
+	})
 }
 
 func TestExecError(t *testing.T) {

app/vmselect/searchutils/searchutils_test.go

@@ -99,7 +99,8 @@ func TestParseMetricSelectorSuccess(t *testing.T) {
 	f(`{foo="bar"}`)
 	f(`{:f:oo=~"bar.+"}`)
 	f(`foo {bar != "baz"}`)
-	f(` foo { bar !~ "^ddd(x+)$", a="ss", __name__="sffd"}  `)
+	f(` { bar !~ "^ddd(x+)$", a="ss", __name__="sffd"}  `)
+	f(` { bar !~ "^ddd(x+)$", a="ss", "foo"}  `)
 	f(`(foo)`)
 	f(`\п\р\и\в\е\т{\ы="111"}`)
 }

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.63479b72.css",
-    "main.js": "./static/js/main.256ee243.js",
+    "main.css": "./static/css/main.7fa18e1b.css",
+    "main.js": "./static/js/main.ba08300f.js",
     "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
     "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.63479b72.css",
-    "static/js/main.256ee243.js"
+    "static/css/main.7fa18e1b.css",
+    "static/js/main.ba08300f.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.256ee243.js"></script><link href="./static/css/main.63479b72.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.ba08300f.js"></script><link href="./static/css/main.7fa18e1b.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.23.5 AS build-web-stage
+FROM golang:1.23.6 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/src/api/types.ts

@@ -1,3 +1,6 @@
+import uPlot from "uplot";
+import { ReactNode } from "react";
+
 export interface MetricBase {
   group: number;
   metric: {
@@ -6,13 +9,13 @@ export interface MetricBase {
 }
 
 export interface MetricResult extends MetricBase {
-  values: [number, string][]
+  values: [number, string][];
 }
 
 
 export interface InstantMetricResult extends MetricBase {
-  value?: [number, string]
-  values?: [number, string][]
+  value?: [number, string];
+  values?: [number, string][];
 }
 
 export interface ExportMetricResult extends MetricBase {
@@ -43,10 +46,24 @@ export interface Logs {
 export interface LogHits {
   timestamps: string[];
   values: number[];
-  total?: number;
-  fields: {
-    [key: string]: string;
-  };
+  total: number;
+  fields: { [key: string]: string; };
+  _isOther: boolean;
+}
+
+export interface LegendLogHits {
+  label: string;
+  total: number;
+  totalHits: number;
+  isOther: boolean;
+  fields: { [key: string]: string; };
+  stroke?: uPlot.Series.Stroke;
+}
+
+export interface LegendLogHitsMenu {
+  title: string;
+  icon?: ReactNode;
+  handler?: () => void;
 }
 
 export interface ReportMetaData {

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsChart.tsx

@@ -1,22 +1,23 @@
-import React, { FC, useMemo, useRef, useState } from "preact/compat";
+import React, { FC, useCallback, useMemo, useRef, useState } from "preact/compat";
 import "./style.scss";
 import "uplot/dist/uPlot.min.css";
 import useElementSize from "../../../hooks/useElementSize";
 import uPlot, { AlignedData } from "uplot";
 import { useEffect } from "react";
-import useBarHitsOptions from "./hooks/useBarHitsOptions";
+import useBarHitsOptions, { getLabelFromLogHit } from "./hooks/useBarHitsOptions";
 import BarHitsTooltip from "./BarHitsTooltip/BarHitsTooltip";
 import { TimeParams } from "../../../types";
 import usePlotScale from "../../../hooks/uplot/usePlotScale";
 import useReadyChart from "../../../hooks/uplot/useReadyChart";
 import useZoomChart from "../../../hooks/uplot/useZoomChart";
 import classNames from "classnames";
-import { LogHits } from "../../../api/types";
+import { LegendLogHits, LogHits } from "../../../api/types";
 import { addSeries, delSeries, setBand } from "../../../utils/uplot";
 import { GraphOptions, GRAPH_STYLES } from "./types";
 import BarHitsOptions from "./BarHitsOptions/BarHitsOptions";
 import stack from "../../../utils/uplot/stack";
 import BarHitsLegend from "./BarHitsLegend/BarHitsLegend";
+import { calculateTotalHits, sortLogHits } from "../../../utils/logs";
 
 interface Props {
   logHits: LogHits[];
@@ -57,6 +58,29 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
     graphOptions
   });
 
+  const prepareLegend = useCallback((hits: LogHits[], totalHits: number): LegendLogHits[] => {
+    return hits.map((hit) => {
+      const label = getLabelFromLogHit(hit);
+
+      const legendItem: LegendLogHits = {
+        label,
+        isOther: hit._isOther,
+        fields: hit.fields,
+        total: hit.total || 0,
+        totalHits,
+        stroke: series.find((s) => s.label === label)?.stroke,
+      };
+
+      return legendItem;
+    }).sort(sortLogHits("total"));
+  }, [series]);
+
+
+  const legendDetails: LegendLogHits[] = useMemo(() => {
+    const totalHits = calculateTotalHits(logHits);
+    return prepareLegend(logHits, totalHits);
+  }, [logHits, prepareLegend]);
+
   useEffect(() => {
     if (!uPlotInst) return;
     delSeries(uPlotInst);
@@ -121,6 +145,7 @@ const BarHitsChart: FC<Props> = ({ logHits, data: _data, period, setPeriod, onAp
         <BarHitsLegend
           uPlotInst={uPlotInst}
           onApplyFilter={onApplyFilter}
+          legendDetails={legendDetails}
         />
       )}
     </div>

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/BarHitsLegend.tsx

@@ -1,83 +1,53 @@
-import React, { FC, useCallback, useEffect, useState } from "preact/compat";
+import React, { FC, useEffect, useState } from "preact/compat";
 import uPlot, { Series } from "uplot";
 import "./style.scss";
 import "../../Line/Legend/style.scss";
-import classNames from "classnames";
-import { MouseEvent } from "react";
-import { isMacOs } from "../../../../utils/detect-device";
-import Tooltip from "../../../Main/Tooltip/Tooltip";
-import { getStreamPairs } from "../../../../utils/logs";
+import BarHitsLegendItem from "./BarHitsLegendItem";
+import { LegendLogHits } from "../../../../api/types";
 
 interface Props {
   uPlotInst: uPlot;
+  legendDetails: LegendLogHits[];
   onApplyFilter: (value: string) => void;
 }
 
-const BarHitsLegend: FC<Props> = ({ uPlotInst, onApplyFilter }) => {
+const BarHitsLegend: FC<Props> = ({ uPlotInst, legendDetails, onApplyFilter }) => {
   const [series, setSeries] = useState<Series[]>([]);
-  const [pairs, setPairs] = useState<string[][]>([]);
+  const totalHits = legendDetails[0]?.totalHits || 0;
 
-  const updateSeries = useCallback(() => {
-    const series = uPlotInst.series.filter(s => s.scale !== "x");
-    setSeries(series);
-    setPairs(series.map(s => getStreamPairs(s.label || "")));
+  const getSeries = () => {
+    return uPlotInst.series.filter(s => s.scale !== "x");
+  };
+
+  const handleRedrawGraph = () => {
+    uPlotInst.redraw();
+    setSeries(getSeries());
+  };
+
+  useEffect(() => {
+    setSeries(getSeries());
   }, [uPlotInst]);
 
-  const handleClickByValue = (value: string) => (e: MouseEvent<HTMLDivElement>) => {
-    const metaKey = e.metaKey || e.ctrlKey;
-    if (!metaKey) return;
-    onApplyFilter(`{${value}}` || "");
-    updateSeries();
-    uPlotInst.redraw();
-  };
-
-  const handleClickByStream = (target: Series) => (e: MouseEvent<HTMLDivElement>) => {
-    const metaKey = e.metaKey || e.ctrlKey;
-    if (metaKey) return;
-    target.show = !target.show;
-    updateSeries();
-    uPlotInst.redraw();
-  };
-
-  useEffect(updateSeries, [uPlotInst]);
-
   return (
     <div className="vm-bar-hits-legend">
-      {series.map((s, i) => (
-        <Tooltip
-          key={s.label}
-          title={(
-            <ul className="vm-bar-hits-legend-info">
-              <li>Click to {s.show ? "hide" : "show"} the _stream.</li>
-              <li>{isMacOs() ? "Cmd" : "Ctrl"} + Click to filter by the _stream.</li>
-            </ul>
-          )}
-        >
-          <div
-            className={classNames({
-              "vm-bar-hits-legend-item": true,
-              "vm-bar-hits-legend-item_hide": !s.show,
-            })}
-            onClick={handleClickByStream(s)}
-          >
-            <div
-              className="vm-bar-hits-legend-item__marker"
-              style={{ backgroundColor: `${(s?.stroke as () => string)?.()}` }}
-            />
-            <div className="vm-bar-hits-legend-item-pairs">
-              {pairs[i].map(value => (
-                <span
-                  className="vm-bar-hits-legend-item-pairs__value"
-                  key={value}
-                  onClick={handleClickByValue(value)}
-                >
-                  {value}
-                </span>
-              ))}
-            </div>
-          </div>
-        </Tooltip>
+      {legendDetails.map((legend) => (
+        <BarHitsLegendItem
+          key={legend.label}
+          legend={legend}
+          series={series}
+          onRedrawGraph={handleRedrawGraph}
+          onApplyFilter={onApplyFilter}
+        />
       ))}
+      <div className="vm-bar-hits-legend-info">
+        <div>
+          Total hits: <b>{totalHits.toLocaleString("en-US")}</b>
+        </div>
+        <div>
+          <code>L-Click</code> toggles visibility.&nbsp;
+          <code>R-Click</code> opens menu.
+        </div>
+      </div>
     </div>
   );
 };

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/BarHitsLegendItem.tsx

@@ -0,0 +1,92 @@
+import React, { FC, useMemo, useRef, useState } from "preact/compat";
+import classNames from "classnames";
+import { Series } from "uplot";
+import { MouseEvent } from "react";
+import { LegendLogHits } from "../../../../api/types";
+import { getStreamPairs } from "../../../../utils/logs";
+import { formatNumberShort } from "../../../../utils/math";
+import Popper from "../../../Main/Popper/Popper";
+import useBoolean from "../../../../hooks/useBoolean";
+import LegendHitsMenu from "../LegendHitsMenu/LegendHitsMenu";
+
+interface Props {
+  legend: LegendLogHits;
+  series: Series[];
+  onRedrawGraph: () => void;
+  onApplyFilter: (value: string) => void;
+}
+
+const BarHitsLegendItem: FC<Props> = ({ legend, series, onRedrawGraph, onApplyFilter }) => {
+  const {
+    value: openContextMenu,
+    setTrue: handleOpenContextMenu,
+    setFalse: handleCloseContextMenu,
+  } = useBoolean(false);
+
+  const legendRef = useRef<HTMLDivElement>(null);
+  const [clickPosition, setClickPosition] = useState<{ top: number; left: number } | null>(null);
+
+  const targetSeries = useMemo(() => series.find(s => s.label === legend.label), [series]);
+
+  const fields = useMemo(() => getStreamPairs(legend.label), [legend.label]);
+
+  const label = fields.join(", ");
+  const totalShortFormatted = formatNumberShort(legend.total);
+
+  const handleClickByStream = (e: MouseEvent<HTMLDivElement>) => {
+    if (!targetSeries) return;
+
+    if (e.metaKey || e.ctrlKey) {
+      targetSeries.show = !targetSeries.show;
+    } else {
+      const isOnlyTargetVisible = series.every(s => s === targetSeries || !s.show);
+      series.forEach(s => {
+        s.show = isOnlyTargetVisible || (s === targetSeries);
+      });
+    }
+
+    onRedrawGraph();
+  };
+
+  const handleContextMenu = (e: MouseEvent<HTMLDivElement>) => {
+    e.preventDefault();
+    setClickPosition({ top: e.clientY, left: e.clientX });
+    handleOpenContextMenu();
+  };
+
+  return (
+    <div
+      ref={legendRef}
+      className={classNames({
+        "vm-bar-hits-legend-item": true,
+        "vm-bar-hits-legend-item_other": legend.isOther,
+        "vm-bar-hits-legend-item_hide": !targetSeries?.show,
+      })}
+      onClick={handleClickByStream}
+      onContextMenu={handleContextMenu}
+    >
+      <div
+        className="vm-bar-hits-legend-item__marker"
+        style={{ backgroundColor: `${legend.stroke}` }}
+      />
+      <div className="vm-bar-hits-legend-item__label">{label}</div>
+      <span className="vm-bar-hits-legend-item__total">({totalShortFormatted})</span>
+      <Popper
+        placement="fixed"
+        open={openContextMenu}
+        buttonRef={legendRef}
+        placementPosition={clickPosition}
+        onClose={handleCloseContextMenu}
+      >
+        <LegendHitsMenu
+          legend={legend}
+          fields={fields}
+          onApplyFilter={onApplyFilter}
+          onClose={handleCloseContextMenu}
+        />
+      </Popper>
+    </div>
+  );
+};
+
+export default BarHitsLegendItem;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsLegend/style.scss

@@ -3,16 +3,16 @@
 .vm-bar-hits-legend {
   display: flex;
   flex-wrap: wrap;
-  gap: $padding-small;
   padding: 0 $padding-small $padding-small;
+  color: $color-text;
 
   &-item {
-    display: grid;
-    grid-template-columns: auto 1fr;
+    max-width: 50%;
+    display: flex;
     align-items: center;
     gap: $padding-small;
-    font-size: 12px;
-    padding: 0 $padding-small;
+    font-size: $font-size-small;
+    padding: $padding-small $padding-global;
     border-radius: $border-radius-small;
     cursor: pointer;
     transition: 0.2s;
@@ -27,34 +27,44 @@
     }
 
     &__marker {
-      width: 14px;
+      min-width: 14px;
+      max-width: 14px;
       height: 14px;
       border: $color-background-block;
     }
 
-    &-pairs {
-      display: flex;
-      gap: $padding-small;
+    &__label {
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
 
-      &__value {
-        padding: $padding-small 0;
-
-        &:hover {
-          text-decoration: underline;
-        }
-
-        &:after {
-          content: ",";
-        }
-
-        &:last-child:after {
-          content: "";
-        }
-      }
+    &__total {
+      color: $color-text-secondary;
+      font-style: italic;
+      grid-column: 2;
     }
   }
 
   &-info {
-    list-style-position: inside;
+    flex-grow: 1;
+    width: 100%;
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding-top: $padding-small;
+    color: $color-text-secondary;
+    font-size: $font-size-small;
+
+    code {
+      display: inline-block;
+      padding: calc($padding-small / 2) $padding-small;
+      font-size: $font-size-small;
+      text-align: center;
+      background-color: $color-background-body;
+      background-repeat: repeat-x;
+      border: $border-divider;
+      border-radius: 4px;
+    }
   }
 }

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/BarHitsTooltip.tsx

@@ -5,6 +5,7 @@ import { DATE_TIME_FORMAT } from "../../../../constants/date";
 import classNames from "classnames";
 import "./style.scss";
 import "../../ChartTooltip/style.scss";
+import { sortLogHits } from "../../../../utils/logs";
 
 interface Props {
   data: AlignedData;
@@ -26,7 +27,7 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
     const tooltipItems = values.map((value, i) => {
       const targetSeries = series[i + 1];
       const stroke = (targetSeries?.stroke as () => string)?.();
-      const label = targetSeries?.label || "other";
+      const label = targetSeries?.label;
       const show = targetSeries?.show;
       return {
         label,
@@ -34,7 +35,7 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
         value,
         show
       };
-    }).filter(item => item.value > 0 && item.show).sort((a, b) => b.value - a.value);
+    }).filter(item => item.value > 0 && item.show).sort(sortLogHits("value"));
 
     const point = {
       top: tooltipItems[0] ? uPlotInst?.valToPos?.(tooltipItems[0].value, "y") || 0 : 0,
@@ -104,21 +105,24 @@ const BarHitsTooltip: FC<Props> = ({ data, focusDataIdx, uPlotInst }) => {
               className="vm-chart-tooltip-data__marker"
               style={{ background: item.stroke }}
             />
-            <p>
-              {item.label}: <b>{item.value}</b>
+            <p className="vm-bar-hits-tooltip-item">
+              <span className="vm-bar-hits-tooltip-item__label">{item.label}</span>
+              <span>{item.value.toLocaleString("en-US")}</span>
             </p>
           </div>
         ))}
       </div>
       {tooltipData.values.length > 1 && (
         <div className="vm-chart-tooltip-data">
-          <p>
-              Total records: <b>{tooltipData.total}</b>
+          <span/>
+          <p className="vm-bar-hits-tooltip-item">
+            <span className="vm-bar-hits-tooltip-item__label">Total</span>
+            <span>{tooltipData.total.toLocaleString("en-US")}</span>
           </p>
         </div>
       )}
       <div className="vm-chart-tooltip-header">
-        <div className="vm-chart-tooltip-header__title">
+        <div className="vm-chart-tooltip-header__title vm-bar-hits-tooltip__date">
           {tooltipData.timestamp}
         </div>
       </div>

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/BarHitsTooltip/style.scss

@@ -9,4 +9,23 @@
     opacity: 1;
     pointer-events: auto;
   }
+
+  &-item {
+    display: grid;
+    grid-template-columns: 1fr auto;
+    align-items: center;
+    gap: $padding-global;
+    max-width: 100%;
+
+    &__label {
+      display: inline-block;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+    }
+  }
+
+  &__date {
+    white-space: nowrap;
+  }
 }

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenu.tsx

@@ -0,0 +1,50 @@
+import React, { FC } from "preact/compat";
+import "./style.scss";
+import { LegendLogHits } from "../../../../api/types";
+import LegendHitsMenuStats from "./LegendHitsMenuStats";
+import LegendHitsMenuBase from "./LegendHitsMenuBase";
+import LegendHitsMenuRow from "./LegendHitsMenuRow";
+import LegendHitsMenuFields from "./LegendHitsMenuFields";
+import { LOGS_LIMIT_HITS } from "../../../../constants/logs";
+
+const otherDescription = `aggregated results for fields not in the top ${LOGS_LIMIT_HITS}`;
+
+interface Props {
+  legend: LegendLogHits;
+  fields: string[];
+  onApplyFilter: (value: string) => void;
+  onClose: () => void;
+}
+
+const LegendHitsMenu: FC<Props> = ({ legend, fields, onApplyFilter, onClose }) => {
+  return (
+    <div className="vm-legend-hits-menu">
+      <div className="vm-legend-hits-menu-section">
+        <LegendHitsMenuRow
+          className="vm-legend-hits-menu-row_info"
+          title={legend.isOther ? otherDescription : legend.label}
+        />
+      </div>
+
+      {!legend.isOther && (
+        <LegendHitsMenuBase
+          legend={legend}
+          onApplyFilter={onApplyFilter}
+          onClose={onClose}
+        />
+      )}
+
+      {!legend.isOther && (
+        <LegendHitsMenuFields
+          fields={fields}
+          onApplyFilter={onApplyFilter}
+          onClose={onClose}
+        />
+      )}
+
+      <LegendHitsMenuStats legend={legend}/>
+    </div>
+  );
+};
+
+export default LegendHitsMenu;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuBase.tsx

@@ -0,0 +1,64 @@
+import React, { FC } from "preact/compat";
+import LegendHitsMenuRow from "./LegendHitsMenuRow";
+import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
+import { CopyIcon, FilterIcon, FilterOffIcon } from "../../../Main/Icons";
+import { LegendLogHits, LegendLogHitsMenu } from "../../../../api/types";
+import { LOGS_GROUP_BY } from "../../../../constants/logs";
+
+interface Props {
+  legend: LegendLogHits;
+  onApplyFilter: (value: string) => void;
+  onClose: () => void;
+}
+
+const LegendHitsMenuBase: FC<Props> = ({ legend, onApplyFilter, onClose }) => {
+  const copyToClipboard = useCopyToClipboard();
+
+  const handleAddStreamToFilter = () => {
+    onApplyFilter(`${LOGS_GROUP_BY}: ${legend.label}`);
+    onClose();
+  };
+
+  const handleExcludeStreamToFilter = () => {
+    onApplyFilter(`(NOT ${LOGS_GROUP_BY}: ${legend.label})`);
+    onClose();
+  };
+
+  const handlerCopyLabel = async () => {
+    await copyToClipboard(legend.label, `${legend.label} has been copied`);
+    onClose();
+  };
+
+  const options: LegendLogHitsMenu[] = [
+    {
+      title: `Copy ${LOGS_GROUP_BY} name`,
+      icon: <CopyIcon/>,
+      handler: handlerCopyLabel,
+    },
+    {
+      title: `Add ${LOGS_GROUP_BY} to filter`,
+      icon: <FilterIcon/>,
+      handler: handleAddStreamToFilter,
+    },
+    {
+      title: `Exclude ${LOGS_GROUP_BY} to filter`,
+      icon: <FilterOffIcon/>,
+      handler: handleExcludeStreamToFilter,
+    }
+  ];
+
+  return (
+    <div className="vm-legend-hits-menu-section">
+      {options.map(({ icon, title, handler }) => (
+        <LegendHitsMenuRow
+          key={title}
+          iconStart={icon}
+          title={title}
+          handler={handler}
+        />
+      ))}
+    </div>
+  );
+};
+
+export default LegendHitsMenuBase;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuFields.tsx

@@ -0,0 +1,74 @@
+import React, { FC, useMemo } from "preact/compat";
+import LegendHitsMenuRow from "./LegendHitsMenuRow";
+import { CopyIcon, FilterIcon, FilterOffIcon } from "../../../Main/Icons";
+import { convertToFieldFilter } from "../../../../utils/logs";
+import { LegendLogHitsMenu } from "../../../../api/types";
+import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
+
+interface Props {
+  fields: string[];
+  onApplyFilter: (value: string) => void;
+  onClose: () => void;
+}
+
+const LegendHitsMenuFields: FC<Props> = ({ fields, onApplyFilter, onClose }) => {
+  const copyToClipboard = useCopyToClipboard();
+
+  const handleCopy = (field: string) => async () => {
+    await copyToClipboard(field, `${field} has been copied`);
+    onClose();
+  };
+
+  const handleAddToFilter = (field: string) => () => {
+    onApplyFilter(field);
+    onClose();
+  };
+
+  const handleExcludeToFilter = (field: string) => () => {
+    onApplyFilter(`-${field}`);
+    onClose();
+  };
+
+  const generateFieldMenu = (field: string): LegendLogHitsMenu[] => {
+    return [
+      {
+        title: "Copy",
+        icon: <CopyIcon/>,
+        handler: handleCopy(field),
+      },
+      {
+        title: "Add to filter",
+        icon: <FilterIcon/>,
+        handler: handleAddToFilter(field),
+      },
+      {
+        title: "Exclude to filter",
+        icon: <FilterOffIcon/>,
+        handler: handleExcludeToFilter(field),
+      }
+    ];
+  };
+
+  const fieldsWithMenu: LegendLogHitsMenu[] = useMemo(() => {
+    return fields.map(field => {
+      const title = convertToFieldFilter(field);
+      return {
+        title,
+        submenu: generateFieldMenu(title),
+      };
+    });
+  }, [fields]);
+
+  return (
+    <div className="vm-legend-hits-menu-section">
+      {fieldsWithMenu?.map((field) => (
+        <LegendHitsMenuRow
+          key={field.title}
+          {...field}
+        />
+      ))}
+    </div>
+  );
+};
+
+export default LegendHitsMenuFields;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuRow.tsx

@@ -0,0 +1,116 @@
+import React, { FC, useRef, useState } from "preact/compat";
+import classNames from "classnames";
+import { ReactNode, useEffect } from "react";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import { LegendLogHitsMenu } from "../../../../api/types";
+import { ArrowDropDownIcon } from "../../../Main/Icons";
+import useClickOutside from "../../../../hooks/useClickOutside";
+
+interface Props {
+  title: string | ReactNode;
+  handler?: () => void;
+  iconStart?: ReactNode;
+  iconEnd?: ReactNode;
+  className?: string;
+  submenu?: LegendLogHitsMenu[];
+}
+
+const LegendHitsMenuRow: FC<Props> = ({ title, handler, iconStart, iconEnd, className, submenu }) => {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const titleRef = useRef<HTMLDivElement>(null);
+  const submenuRef = useRef<HTMLDivElement>(null);
+
+  const [isOverflownTitle, setIsOverflownTitle] = useState(false);
+
+  const [openSubmenu, setOpenSubmenu] = useState(false);
+  const [posSubmenuLeft, setPosSubmenuLeft] = useState(false);
+  const hasSubmenu = !!submenu?.length;
+
+  const handleToggleContextMenu = () => {
+    setOpenSubmenu(prev => !prev);
+  };
+
+  const handleCloseContextMenu = () => {
+    setOpenSubmenu(false);
+  };
+
+  const handleClick = () => {
+    handler && handler();
+    hasSubmenu && handleToggleContextMenu();
+  };
+
+
+  useEffect(() => {
+    if (!titleRef.current) return;
+    setIsOverflownTitle(titleRef.current.scrollWidth > titleRef.current.clientWidth);
+  }, [title, titleRef]);
+
+  useEffect(() => {
+    requestAnimationFrame(() => {
+      if (!openSubmenu || !submenuRef.current) {
+        setPosSubmenuLeft(false);
+        return;
+      }
+
+      const { left, width } = submenuRef.current.getBoundingClientRect();
+      setPosSubmenuLeft(left + width > window.innerWidth);
+    });
+  }, [submenuRef, openSubmenu]);
+
+  useClickOutside(containerRef, handleCloseContextMenu);
+
+  const titleContent = (
+    <div
+      ref={titleRef}
+      className="vm-legend-hits-menu-row__title"
+    >
+      {title}
+    </div>
+  );
+
+  return (
+    <div
+      ref={containerRef}
+      className={classNames({
+        "vm-legend-hits-menu-row": true,
+        "vm-legend-hits-menu-row_interactive": !!handler || hasSubmenu,
+        [`${className}`]: className
+      })}
+      onClick={handleClick}
+    >
+      {iconStart && <div className="vm-legend-hits-menu-row__icon">{iconStart}</div>}
+      {isOverflownTitle ? (<Tooltip title={title}>{titleContent}</Tooltip>) : titleContent}
+      {iconEnd && !hasSubmenu && <div className="vm-legend-hits-menu-row__icon">{iconEnd}</div>}
+
+      {hasSubmenu && (
+        <div className="vm-legend-hits-menu-row__icon vm-legend-hits-menu-row__icon_drop">
+          <ArrowDropDownIcon/>
+        </div>
+      )}
+
+      {openSubmenu && submenu && (
+        <div
+          ref={submenuRef}
+          className={classNames({
+            "vm-legend-hits-menu": true,
+            "vm-legend-hits-menu_submenu": true,
+            "vm-legend-hits-menu_submenu_left": posSubmenuLeft
+          })}
+        >
+          <div className="vm-legend-hits-menu-section">
+            {submenu.map(({ icon, title, handler }) => (
+              <LegendHitsMenuRow
+                key={title}
+                iconStart={icon}
+                title={title}
+                handler={handler}
+              />
+            ))}
+          </div>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default LegendHitsMenuRow;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/LegendHitsMenuStats.tsx

@@ -0,0 +1,23 @@
+import React, { FC } from "preact/compat";
+import { LegendLogHits } from "../../../../api/types";
+
+interface Props {
+  legend: LegendLogHits;
+}
+
+const LegendHitsMenuStats: FC<Props> = ({ legend }) => {
+  const totalFormatted = legend.total.toLocaleString("en-US");
+  const percentage = Math.round((legend.total / legend.totalHits) * 100);
+
+  return (
+    <div className="vm-legend-hits-menu-section">
+      <div className="vm-legend-hits-menu-row">
+        <div className="vm-legend-hits-menu-row__title">
+          Total: {totalFormatted} ({percentage}%)
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default LegendHitsMenuStats;

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/LegendHitsMenu/style.scss

@@ -0,0 +1,178 @@
+@use "src/styles/variables" as *;
+
+.vm-legend-hits-menu {
+  min-width: 160px;
+  z-index: 1;
+
+  &_submenu {
+    position: absolute;
+    top: calc(-1 * $padding-small);
+    background-color: $color-background-block;
+    left: calc(100% + ($padding-small / 2));
+    box-shadow: $box-shadow-popper;
+    border-radius: $border-radius-small;
+    animation: vm-submenu-show 150ms cubic-bezier(0.280, 0.840, 0.2, 1);
+    transform-origin: top left;
+
+    &_left {
+      left: auto;
+      right: calc(100% + ($padding-small / 2));
+      transform-origin: top right;
+    }
+  }
+
+  &-section {
+    border-bottom: $border-divider;
+
+    &:last-child {
+      border-bottom: none;
+    }
+  }
+
+  &-row {
+    position: relative;
+    display: flex;
+    gap: $padding-small;
+    align-items: center;
+    justify-content: flex-start;
+    padding: 0 $padding-global;
+    transition: background-color 0.3s;
+    color: $color-text;
+
+    &_interactive {
+      cursor: pointer;
+
+      &:hover {
+        background-color: rgba(0, 0, 0, 0.05);
+      }
+    }
+
+    &_info {
+      font-size: $font-size-small;
+      font-weight: 500;
+      padding-block: $padding-small;
+    }
+
+    &_info &__icon {
+      color: $color-info;
+    }
+
+    &__icon {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      width: 14px;
+      height: 14px;
+
+      &_drop {
+        transform: rotate(-90deg);
+      }
+    }
+
+    &__title {
+      flex-grow: 1;
+      padding: $padding-global 0;
+      position: relative;
+      max-width: 400px;
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+  }
+
+  &-other-list {
+    width: 80vw;
+    height: 80vh;
+    overflow: auto;
+
+    &__search {
+      position: sticky;
+      top: 0;
+      padding: $padding-small 0;
+      background-color: $color-background-block;
+      border-bottom: $border-divider;
+      z-index: 2;
+    }
+
+    &-row {
+      border-bottom: $border-divider;
+
+      &_header {
+        border-bottom: none;
+        position: sticky;
+        top: 65px;
+        background-color: $color-background-block;
+        z-index: 1;
+        width: 100%;
+
+        &:after {
+          content: '';
+          position: absolute;
+          bottom: 0;
+          left: 0;
+          width: 100%;
+          height: 1px;
+          border-bottom: $border-divider;
+        }
+      }
+    }
+
+    &-cell {
+      padding: calc($padding-small / 2) 0;
+      text-align: left;
+
+      &_header {
+        padding: $padding-small;
+        font-weight: 500;
+      }
+
+      &_number {
+        padding: $padding-small;
+        text-align: right;
+        font-variant-numeric: tabular-nums;
+      }
+
+      &_fields {
+        width: 100%;
+      }
+    }
+
+    &-fields {
+      display: flex;
+      flex-wrap: wrap;
+      align-items: center;
+      justify-content: flex-start;
+
+      &__field {
+        padding: calc($padding-small / 2) $padding-small;
+        border-radius: $border-radius-small;
+        transition: background-color 0.3s;
+
+        &:hover {
+          background-color: $color-hover-black;
+        }
+
+        &:not(:last-child) {
+          &:after {
+            content: ',';
+          }
+        }
+      }
+    }
+
+    &-actions {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    }
+  }
+}
+
+@keyframes vm-submenu-show {
+  0% {
+    opacity: 0;
+  }
+  100% {
+    opacity: 1;
+  }
+}

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/hooks/useBarHitsOptions.ts

@@ -36,6 +36,14 @@ interface UseGetBarHitsOptionsArgs {
   graphOptions: GraphOptions;
 }
 
+export const OTHER_HITS_LABEL = "other";
+
+export const getLabelFromLogHit = (logHit: LogHits) => {
+  if (logHit?._isOther) return OTHER_HITS_LABEL;
+  const fields = Object.values(logHit?.fields || {});
+  return fields.map((value) => value || "\"\"").join(", ");
+};
+
 const useBarHitsOptions = ({
   data,
   logHits,
@@ -59,16 +67,16 @@ const useBarHitsOptions = ({
     let colorN = 0;
     return data.map((_d, i) => {
       if (i === 0) return {}; // 0 index is xAxis(timestamps)
-      const fields = Object.values(logHits?.[i - 1]?.fields || {});
-      const label = fields.map((value) => value || "\"\"").join(", ");
-      const color = getCssVariable(label ? seriesColors[colorN] : "color-log-hits-bar-0");
-      if (label) colorN++;
+      const target = logHits?.[i - 1];
+      const label = getLabelFromLogHit(target);
+      const color = getCssVariable(target?._isOther ? "color-log-hits-bar-0" : seriesColors[colorN]);
+      if (!target?._isOther) colorN++;
       return {
-        label: label || "other",
+        label,
         width: strokeWidth[graphOptions.graphStyle],
         spanGaps: true,
         stroke: color,
-        fill: graphOptions.fill ? color + "80" : "",
+        fill: graphOptions.fill ? color + (target?._isOther ? "" : "80") : "",
         paths: getSeriesPaths(graphOptions.graphStyle),
       };
     });

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -32,6 +32,11 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     max-width: calc(100vw/3);
   }
 
+  &_hits &-data {
+    display: grid;
+    grid-template-columns: $font-size 1fr;
+  }
+
   &_sticky {
     pointer-events: auto;
     z-index: 99;
@@ -90,6 +95,8 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     }
 
     &__marker {
+      min-width: $font-size;
+      max-width: $font-size;
       width: $font-size;
       height: $font-size;
       border: 1px solid rgba($color-white, 0.5);

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/AxesLimitsConfigurator/AxesLimitsConfigurator.tsx

@@ -36,35 +36,40 @@ const AxesLimitsConfigurator: FC<AxesLimitsConfiguratorProps> = ({ yaxis, setYax
       "vm-axes-limits_mobile": isMobile
     })}
   >
-    <Switch
-      value={yaxis.limits.enable}
-      onChange={toggleEnableLimits}
-      label="Fix the limits for y-axis"
-      fullWidth={isMobile}
-    />
-    <div className="vm-axes-limits-list">
-      {axes.map(axis => (
-        <div
-          className="vm-axes-limits-list__inputs"
-          key={axis}
-        >
-          <TextField
-            label={`Min ${axis}`}
-            type="number"
-            disabled={!yaxis.limits.enable}
-            value={yaxis.limits.range[axis][0]}
-            onChange={createHandlerOnchangeAxis(axis, 0)}
-          />
-          <TextField
-            label={`Max ${axis}`}
-            type="number"
-            disabled={!yaxis.limits.enable}
-            value={yaxis.limits.range[axis][1]}
-            onChange={createHandlerOnchangeAxis(axis, 1)}
-          />
-        </div>
-      ))}
+    <div className="vm-graph-settings-row">
+      <span className="vm-graph-settings-row__label">Fixed Y-axis limits</span>
+      <Switch
+        value={yaxis.limits.enable}
+        onChange={toggleEnableLimits}
+        label={`${yaxis.limits.enable ? "Fixed" : "Auto"} limits`}
+        fullWidth={isMobile}
+      />
     </div>
+    {yaxis.limits.enable && (
+      <div className="vm-axes-limits-list">
+        {axes.map(axis => (
+          <div
+            className="vm-axes-limits-list__inputs"
+            key={axis}
+          >
+            <TextField
+              label={`Min ${axis}`}
+              type="number"
+              disabled={!yaxis.limits.enable}
+              value={yaxis.limits.range[axis][0]}
+              onChange={createHandlerOnchangeAxis(axis, 0)}
+            />
+            <TextField
+              label={`Max ${axis}`}
+              type="number"
+              disabled={!yaxis.limits.enable}
+              value={yaxis.limits.range[axis][1]}
+              onChange={createHandlerOnchangeAxis(axis, 1)}
+            />
+          </div>
+        ))}
+      </div>
+    )}
   </div>;
 };
 

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/GraphSettings.tsx

@@ -8,10 +8,14 @@ import "./style.scss";
 import Tooltip from "../../Main/Tooltip/Tooltip";
 import useBoolean from "../../../hooks/useBoolean";
 import LinesConfigurator from "./LinesConfigurator/LinesConfigurator";
+import GraphTypeSwitcher from "./GraphTypeSwitcher/GraphTypeSwitcher";
+import { MetricResult } from "../../../api/types";
+import { isHistogramData } from "../../../utils/metric";
 
 const title = "Graph settings";
 
 interface GraphSettingsProps {
+  data: MetricResult[],
   yaxis: YaxisState,
   setYaxisLimits: (limits: AxisRange) => void,
   toggleEnableLimits: () => void,
@@ -19,11 +23,13 @@ interface GraphSettingsProps {
     value: boolean,
     onChange: (value: boolean) => void,
   },
+  isHistogram?: boolean,
 }
 
-const GraphSettings: FC<GraphSettingsProps> = ({ yaxis, setYaxisLimits, toggleEnableLimits, spanGaps }) => {
+const GraphSettings: FC<GraphSettingsProps> = ({ data, yaxis, setYaxisLimits, toggleEnableLimits, spanGaps }) => {
   const popperRef = useRef<HTMLDivElement>(null);
   const buttonRef = useRef<HTMLDivElement>(null);
+  const displayHistogramMode = isHistogramData(data);
 
   const {
     value: openPopper,
@@ -64,6 +70,7 @@ const GraphSettings: FC<GraphSettingsProps> = ({ yaxis, setYaxisLimits, toggleEn
               spanGaps={spanGaps.value}
               onChange={spanGaps.onChange}
             />
+            {displayHistogramMode && <GraphTypeSwitcher onChange={handleClose}/>}
           </div>
         </div>
       </Popper>

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/GraphTypeSwitcher/GraphTypeSwitcher.tsx

@@ -0,0 +1,36 @@
+import React, { FC } from "preact/compat";
+import Switch from "../../../Main/Switch/Switch";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import { useSearchParams } from "react-router-dom";
+import { useChangeDisplayMode } from "./useChangeDisplayMode";
+
+type Props = {
+  onChange: () => void;
+}
+
+const GraphTypeSwitcher: FC<Props> = ({ onChange }) => {
+  const { isMobile } = useDeviceDetect();
+
+  const { handleChange } = useChangeDisplayMode();
+  const [searchParams] = useSearchParams();
+
+  const value = !searchParams.get("display_mode");
+
+  const handleChangeMode = (val: boolean) => {
+    handleChange(val, onChange);
+  };
+
+  return (
+    <div className="vm-graph-settings-row">
+      <span className="vm-graph-settings-row__label">Histogram mode</span>
+      <Switch
+        value={value}
+        onChange={handleChangeMode}
+        label={value ? "Enabled" : "Disabled"}
+        fullWidth={isMobile}
+      />
+    </div>
+  );
+};
+
+export default GraphTypeSwitcher;

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/GraphTypeSwitcher/useChangeDisplayMode.ts

@@ -0,0 +1,16 @@
+import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
+import { useSearchParams } from "react-router-dom";
+
+export const useChangeDisplayMode = () => {
+  const [searchParams, setSearchParams] = useSearchParams();
+  const dispatch = useTimeDispatch();
+
+  const handleChange = (val: boolean, callback?: () => void) => {
+    val ? searchParams.delete("display_mode") : searchParams.set("display_mode", "lines");
+    setSearchParams(searchParams);
+    dispatch({ type: "RUN_QUERY" });
+    callback && callback();
+  };
+
+  return { handleChange };
+};

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/LinesConfigurator/LinesConfigurator.tsx

@@ -10,14 +10,17 @@ interface Props {
 const LinesConfigurator: FC<Props> = ({ spanGaps, onChange }) => {
   const { isMobile } = useDeviceDetect();
 
-  return <div>
-    <Switch
-      value={spanGaps}
-      onChange={onChange}
-      label="Connect null values"
-      fullWidth={isMobile}
-    />
-  </div>;
+  return (
+    <div className="vm-graph-settings-row">
+      <span className="vm-graph-settings-row__label">Connect null values</span>
+      <Switch
+        value={spanGaps}
+        onChange={onChange}
+        label={spanGaps ? "Enabled" : "Disabled"}
+        fullWidth={isMobile}
+      />
+    </div>
+  );
 };
 
 export default LinesConfigurator;

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/style.scss

@@ -1,15 +1,31 @@
 @use "src/styles/variables" as *;
 
 .vm-graph-settings {
+  display: flex;
+  align-items: center;
+  gap: $padding-small;
+
   &-popper {
     display: grid;
     gap: $padding-global;
-    padding: 0 0 $padding-global;
+    padding: $padding-small $padding-large $padding-large;
+    min-width: 300px;
 
     &__body {
       display: grid;
       gap: $padding-large;
-      padding: 0 $padding-global;
+    }
+  }
+
+  &-row {
+    display: grid;
+    gap: $padding-small;
+    grid-template-columns: minmax(150px, max-content) 1fr;
+
+    &__label {
+      &:after{
+        content: ":";
+      }
     }
   }
 }

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditor.tsx

@@ -124,7 +124,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
   };
 
   useEffect(() => {
-    setOpenAutocomplete(!!AutocompleteEl);
+    setOpenAutocomplete(!!AutocompleteEl && autocompleteQuick);
   }, [autocompleteQuick]);
 
   useEffect(() => {

app/vmui/packages/vmui/src/components/LogsConfigurators/GroupLogsConfigurators/GroupLogsConfigurators.tsx

@@ -41,13 +41,13 @@ const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
   const noWrapLines = searchParams.get(NO_WRAP_LINES) === "true";
   const compactGroupHeader = searchParams.get(COMPACT_GROUP_HEADER) === "true";
   const displayFieldsString = searchParams.get(DISPLAY_FIELDS) || "";
-  const displayFields = displayFieldsString ? displayFieldsString.split(",") : [];
+  const displayFields = displayFieldsString ? displayFieldsString.split(",") : [LOGS_DISPLAY_FIELDS];
 
   const [dateFormat, setDateFormat] = useState(searchParams.get(DATE_FORMAT) || LOGS_DATE_FORMAT);
   const [errorFormat, setErrorFormat] = useState("");
 
   const isGroupChanged = groupBy !== LOGS_GROUP_BY;
-  const isDisplayFieldsChanged = displayFields.length > 0;
+  const isDisplayFieldsChanged = displayFields.length !== 1 || displayFields[0] !== LOGS_DISPLAY_FIELDS;
   const isTimeChanged = searchParams.get(DATE_FORMAT) !== LOGS_DATE_FORMAT;
   const hasChanges = [
     isGroupChanged,
@@ -58,9 +58,7 @@ const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
   ].some(Boolean);
 
   const logsKeys = useMemo(() => {
-    const excludeKeys = ["_msg", "_time"];
-    const uniqKeys = Array.from(new Set(logs.map(l => Object.keys(l)).flat()));
-    return uniqKeys.filter(k => !excludeKeys.includes(k));
+    return Array.from(new Set(logs.map(l => Object.keys(l)).flat()));
   }, [logs]);
 
   const {

app/vmui/packages/vmui/src/components/Main/CodeExample/CodeExample.tsx

@@ -4,13 +4,16 @@ import { useState } from "react";
 import Tooltip from "../Tooltip/Tooltip";
 import Button from "../Button/Button";
 import { CopyIcon } from "../Icons";
+import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 
 enum CopyState { copy = "Copy", copied = "Copied" }
 
 const CodeExample: FC<{code: string}> = ({ code }) => {
+  const copyToClipboard = useCopyToClipboard();
+
   const [tooltip, setTooltip] = useState(CopyState.copy);
-  const handlerCopy = () => {
-    navigator.clipboard.writeText(code);
+  const handlerCopy = async () => {
+    await copyToClipboard(code);
     setTooltip(CopyState.copied);
   };
 

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -581,3 +581,45 @@ export const CommentIcon = () => (
     ></path>
   </svg>
 );
+
+export const FilterIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M4.25 5.61C6.27 8.2 10 13 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-6s3.72-4.8 5.74-7.39c.51-.66.04-1.61-.79-1.61H5.04c-.83 0-1.3.95-.79 1.61"
+    ></path>
+  </svg>
+);
+
+export const FilterOffIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M19.79 5.61C20.3 4.95 19.83 4 19 4H6.83l7.97 7.97zM2.81 2.81 1.39 4.22 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-2.17l5.78 5.78 1.41-1.41z"
+    ></path>
+  </svg>
+);
+
+export const OpenNewIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M19 19H5V5h7V3H5c-1.11 0-2 .9-2 2v14c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2v-7h-2zM14 3v2h3.59l-9.83 9.83 1.41 1.41L19 6.41V10h2V3z"
+    ></path>
+  </svg>
+);
+
+export const ModalIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path d="M19 4H5c-1.11 0-2 .9-2 2v12c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2V6c0-1.1-.89-2-2-2m0 14H5V8h14z"></path>
+  </svg>
+);

app/vmui/packages/vmui/src/components/Main/Popper/Popper.tsx

@@ -15,9 +15,10 @@ interface PopperProps {
   open: boolean
   onClose: () => void
   buttonRef: React.RefObject<HTMLElement>
-  placement?: "bottom-right" | "bottom-left" | "top-left" | "top-right"
+  placement?: "bottom-right" | "bottom-left" | "top-left" | "top-right" | "fixed"
+  placementPosition?: { top: number, left: number } | null
   animation?: string
-  offset?: {top: number, left: number}
+  offset?: { top: number, left: number }
   clickOutside?: boolean,
   fullWidth?: boolean
   title?: string
@@ -29,6 +30,7 @@ const Popper: FC<PopperProps> = ({
   children,
   buttonRef,
   placement = "bottom-left",
+  placementPosition,
   open = false,
   onClose,
   offset = { top: 6, left: 0 },
@@ -92,13 +94,18 @@ const Popper: FC<PopperProps> = ({
     if (needAlignRight) position.left = buttonPos.right - popperSize.width;
     if (needAlignTop) position.top = buttonPos.top - popperSize.height - offsetTop;
 
-    const { innerWidth, innerHeight } = window;
-    const margin = 20;
+    if (placement === "fixed" && placementPosition) {
+      position.top = Math.max(placementPosition.top + offset.top, 0);
+      position.left = Math.max(placementPosition.left + offset.left, 0);
+      return position;
+    }
 
-    const isOverflowBottom = (position.top + popperSize.height + margin) > innerHeight;
-    const isOverflowTop = (position.top - margin) < 0;
-    const isOverflowRight = (position.left + popperSize.width + margin) > innerWidth;
-    const isOverflowLeft = (position.left - margin) < 0;
+    const { innerWidth, innerHeight } = window;
+
+    const isOverflowBottom = (position.top + popperSize.height) > innerHeight;
+    const isOverflowTop = (position.top) < 0;
+    const isOverflowRight = (position.left + popperSize.width) > innerWidth;
+    const isOverflowLeft = (position.left) < 0;
 
     if (isOverflowBottom) position.top = buttonPos.top - popperSize.height - offsetTop;
     if (isOverflowTop) position.top = buttonPos.height + buttonPos.top + offsetTop;
@@ -106,11 +113,11 @@ const Popper: FC<PopperProps> = ({
     if (isOverflowLeft) position.left = buttonPos.left + offsetLeft;
 
     if (fullWidth) position.width = `${buttonPos.width}px`;
-    if (position.top < 0) position.top = 20;
-    if (position.left < 0) position.left = 20;
+    if (position.top < 0) position.top = 0;
+    if (position.left < 0) position.left = 0;
 
     return position;
-  },[buttonRef, placement, isOpen, children, fullWidth]);
+  }, [buttonRef, placement, isOpen, children, fullWidth]);
 
   const handleClickClose = (e: ReactMouseEvent<HTMLButtonElement, MouseEvent>) => {
     e.stopPropagation();
@@ -131,10 +138,10 @@ const Popper: FC<PopperProps> = ({
     if (!popperRef.current || !isOpen || (isMobile && !disabledFullScreen)) return;
     const { right, width } = popperRef.current.getBoundingClientRect();
     if (right > window.innerWidth) {
-      const left = window.innerWidth - 20 - width;
-      popperRef.current.style.left = left < window.innerWidth ? "0" : `${left}px`;
+      const left = window.innerWidth - width;
+      popperRef.current.style.left = `${left}px`;
     }
-  }, [isOpen, popperRef]);
+  }, [isOpen, popperRef, placementPosition]);
 
   const handlePopstate = useCallback(() => {
     if (isOpen && isMobile && !disabledFullScreen) {

app/vmui/packages/vmui/src/components/Table/Table.tsx

@@ -5,6 +5,7 @@ import { getComparator, stableSort } from "./helpers";
 import Tooltip from "../Main/Tooltip/Tooltip";
 import Button from "../Main/Button/Button";
 import { useEffect } from "preact/compat";
+import useCopyToClipboard from "../../hooks/useCopyToClipboard";
 
 type OrderDir = "asc" | "desc"
 
@@ -22,6 +23,8 @@ interface TableProps<T> {
 }
 
 const Table = <T extends object>({ rows, columns, defaultOrderBy, defaultOrderDir, copyToClipboard, paginationOffset }: TableProps<T>) => {
+  const handleCopyToClipboard = useCopyToClipboard();
+
   const [orderBy, setOrderBy] = useState<keyof T>(defaultOrderBy);
   const [orderDir, setOrderDir] = useState<OrderDir>(defaultOrderDir || "desc");
   const [copied, setCopied] = useState<number | null>(null);
@@ -42,7 +45,7 @@ const Table = <T extends object>({ rows, columns, defaultOrderBy, defaultOrderDi
   const createCopyHandler = (copyValue:  string | number, rowIndex: number) => async () => {
     if (copied === rowIndex) return;
     try {
-      await navigator.clipboard.writeText(String(copyValue));
+      await handleCopyToClipboard(String(copyValue));
       setCopied(rowIndex);
     } catch (e) {
       console.error(e);

app/vmui/packages/vmui/src/components/Table/TableSettings/TableSettings.tsx

@@ -11,7 +11,7 @@ import useBoolean from "../../../hooks/useBoolean";
 import TextField from "../../Main/TextField/TextField";
 import { KeyboardEvent, useState } from "react";
 import Modal from "../../Main/Modal/Modal";
-import { getFromStorage, removeFromStorage, saveToStorage } from "../../../utils/storage";
+import { useSearchParams } from "react-router-dom";
 
 const title = "Table settings";
 
@@ -30,6 +30,8 @@ const TableSettings: FC<TableSettingsProps> = ({
   onChangeColumns,
   toggleTableCompact
 }) => {
+  const [searchParams, setSearchParams] = useSearchParams();
+
   const buttonRef = useRef<HTMLDivElement>(null);
 
   const {
@@ -38,11 +40,6 @@ const TableSettings: FC<TableSettingsProps> = ({
     setFalse: handleClose,
   } = useBoolean(false);
 
-  const {
-    value: saveColumns,
-    toggle: toggleSaveColumns,
-  } = useBoolean(Boolean(getFromStorage("TABLE_COLUMNS")));
-
   const [searchColumn, setSearchColumn] = useState("");
   const [indexFocusItem, setIndexFocusItem] = useState(-1);
 
@@ -60,15 +57,34 @@ const TableSettings: FC<TableSettingsProps> = ({
     return filteredColumns.every(col => selectedColumns.includes(col));
   }, [selectedColumns, filteredColumns]);
 
+  const handleChangeDisplayColumns = (displayColumns: string[]) => {
+    onChangeColumns(displayColumns);
+
+    const updatedParams = new URLSearchParams(searchParams.toString());
+    const isAllCheck = displayColumns.length === columns.length;
+
+    if (isAllCheck) {
+      updatedParams.delete("columns");
+    } else {
+      updatedParams.set("columns", displayColumns.map(encodeURIComponent).join(","));
+    }
+
+    setSearchParams(updatedParams);
+  };
+
   const handleChange = (key: string) => {
-    onChangeColumns(selectedColumns.includes(key) ? selectedColumns.filter(col => col !== key) : [...selectedColumns, key]);
+    const displayColumns = selectedColumns.includes(key)
+      ? selectedColumns.filter(col => col !== key)
+      : [...selectedColumns, key];
+
+    handleChangeDisplayColumns(displayColumns);
   };
 
   const toggleAllColumns = () => {
     if (isAllChecked) {
-      onChangeColumns(selectedColumns.filter(col => !filteredColumns.includes(col)));
+      handleChangeDisplayColumns(selectedColumns.filter(col => !filteredColumns.includes(col)));
     } else {
-      onChangeColumns(filteredColumns);
+      handleChangeDisplayColumns(filteredColumns);
     }
   };
 
@@ -95,22 +111,16 @@ const TableSettings: FC<TableSettingsProps> = ({
   };
 
   useEffect(() => {
-    if (arrayEquals(columns, selectedColumns) || saveColumns) return;
+    if (arrayEquals(columns, selectedColumns) || searchParams.has("columns")) return;
     onChangeColumns(columns);
   }, [columns]);
 
   useEffect(() => {
-    if (!saveColumns) {
-      removeFromStorage(["TABLE_COLUMNS"]);
-    } else if (selectedColumns.length) {
-      saveToStorage("TABLE_COLUMNS", selectedColumns.join(","));
-    }
-  }, [saveColumns, selectedColumns]);
-
-  useEffect(() => {
-    const saveColumns = getFromStorage("TABLE_COLUMNS") as string;
-    if (!saveColumns) return;
-    onChangeColumns(saveColumns.split(","));
+    const hasColumns = searchParams.has("columns");
+    if (!hasColumns) return;
+    const columnsParam = searchParams.get("columns") || "";
+    const columnsArray = columnsParam.split(",").map(decodeURIComponent).filter(Boolean);
+    onChangeColumns(columnsArray);
   }, []);
 
   return (
@@ -183,19 +193,6 @@ const TableSettings: FC<TableSettingsProps> = ({
                   </div>
                 ))}
               </div>
-              <div className="vm-table-settings-modal-preserve">
-                <Checkbox
-                  checked={saveColumns}
-                  onChange={toggleSaveColumns}
-                  label={"Preserve column settings"}
-                  disabled={tableCompact}
-                  color={"primary"}
-                />
-                <p className="vm-table-settings-modal-preserve__info">
-                  This label indicates that when the checkbox is activated,
-                  the current column configurations will not be reset.
-                </p>
-              </div>
             </div>
           </div>
           <div className="vm-table-settings-modal-section">

app/vmui/packages/vmui/src/components/Table/TableSettings/style.scss

@@ -3,6 +3,7 @@
 .vm-table-settings {
   &-modal {
     .vm-modal-content-body {
+      min-width: clamp(300px, 600px, 90vw);
       padding: 0;
     }
 
@@ -83,16 +84,5 @@
         }
       }
     }
-
-    &-preserve {
-      padding: $padding-global;
-
-      &__info {
-        padding-top: $padding-small;
-        font-size: $font-size-small;
-        color: $color-text-secondary;
-        line-height: 130%;
-      }
-    }
   }
 }

app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx

@@ -26,6 +26,7 @@ import useElementSize from "../../../hooks/useElementSize";
 import { ChartTooltipProps } from "../../Chart/ChartTooltip/ChartTooltip";
 import LegendAnomaly from "../../Chart/Line/LegendAnomaly/LegendAnomaly";
 import { groupByMultipleKeys } from "../../../utils/array";
+import { useGraphDispatch } from "../../../state/graph/GraphStateContext";
 
 export interface GraphViewProps {
   data?: MetricResult[];
@@ -62,6 +63,8 @@ const GraphView: FC<GraphViewProps> = ({
   isAnomalyView,
   spanGaps
 }) => {
+  const graphDispatch = useGraphDispatch();
+
   const { isMobile } = useDeviceDetect();
   const { timezone } = useTimeState();
   const currentStep = useMemo(() => customStep || period.step || "1s", [period.step, customStep]);
@@ -196,6 +199,26 @@ const GraphView: FC<GraphViewProps> = ({
 
   const [containerRef, containerSize] = useElementSize();
 
+  const hasTimeData = dataChart[0]?.length > 0;
+
+  useEffect(() => {
+    const checkEmptyHistogram = () => {
+      if (!isHistogram || !data[1]) {
+        return false;
+      }
+
+      try {
+        const values = (dataChart?.[1]?.[2] || []) as (number | null)[];
+        return values.every(v => v === null);
+      } catch (e) {
+        return false;
+      }
+    };
+
+    const isEmpty = checkEmptyHistogram();
+    graphDispatch({ type: "SET_IS_EMPTY_HISTOGRAM", payload: isEmpty });
+  }, [dataChart, isHistogram]);
+
   return (
     <div
       className={classNames({
@@ -205,7 +228,7 @@ const GraphView: FC<GraphViewProps> = ({
       })}
       ref={containerRef}
     >
-      {!isHistogram && (
+      {!isHistogram && hasTimeData && (
         <LineChart
           data={dataChart}
           series={series}

app/vmui/packages/vmui/src/constants/logs.ts

@@ -2,6 +2,7 @@ import { DATE_TIME_FORMAT } from "./date";
 
 export const LOGS_ENTRIES_LIMIT = 50;
 export const LOGS_BARS_VIEW = 100;
+export const LOGS_LIMIT_HITS = 5;
 
 // "Ungrouped" is a string that is used as a value for the "groupBy" parameter.
 export const WITHOUT_GROUPING = "Ungrouped";

app/vmui/packages/vmui/src/hooks/useFetchQuery.ts

@@ -13,6 +13,7 @@ import { isHistogramData } from "../utils/metric";
 import { useGraphState } from "../state/graph/GraphStateContext";
 import { getStepFromDuration } from "../utils/time";
 import { AppType } from "../types/appType";
+import { getQueryStringValue } from "../utils/query-string";
 
 interface FetchQueryParams {
   predefinedQuery?: string[]
@@ -132,7 +133,8 @@ export const useFetchQuery = ({
             tempTraces.push(trace);
           }
 
-          isHistogramResult = !isAnomalyUI && isDisplayChart && isHistogramData(resp.data.result);
+          const preventChangeType = !!getQueryStringValue("display_mode", null);
+          isHistogramResult = !isAnomalyUI && isDisplayChart && !preventChangeType && isHistogramData(resp.data.result);
           seriesLimit = isHistogramResult ? Infinity : defaultLimit;
           const freeTempSize = seriesLimit - tempData.length;
           resp.data.result.slice(0, freeTempSize).forEach((d: MetricBase) => {

app/vmui/packages/vmui/src/pages/CustomPanel/CustomPanelTabs/GraphTab.tsx

@@ -47,7 +47,9 @@ const GraphTab: FC<Props> = ({ isHistogram, graphData, controlsRef, isAnomalyVie
     <div className="vm-custom-panel-body-header__graph-controls">
       <GraphTips/>
       <GraphSettings
+        data={graphData}
         yaxis={yaxis}
+        isHistogram={isHistogram}
         setYaxisLimits={setYaxisLimits}
         toggleEnableLimits={toggleEnableLimits}
         spanGaps={{ value: spanGaps, onChange: setSpanGaps }}

app/vmui/packages/vmui/src/pages/CustomPanel/WarningHeatmapToLine/WarningHeatmapToLine.tsx

@@ -0,0 +1,37 @@
+import React, { FC } from "preact/compat";
+import Alert from "../../../components/Main/Alert/Alert";
+import { useGraphState } from "../../../state/graph/GraphStateContext";
+import {
+  useChangeDisplayMode
+} from "../../../components/Configurators/GraphSettings/GraphTypeSwitcher/useChangeDisplayMode";
+import Button from "../../../components/Main/Button/Button";
+import "./style.scss";
+
+const WarningHeatmapToLine:FC = () => {
+  const { isEmptyHistogram } = useGraphState();
+  const { handleChange } = useChangeDisplayMode();
+
+  if (!isEmptyHistogram) return null;
+
+  return (
+    <Alert variant="warning">
+      <div className="vm-warning-heatmap-to-line">
+        <p className="vm-warning-heatmap-to-line__text">
+         The expression cannot be displayed as a heatmap.
+         To make the graph work, disable the heatmap in the &quot;Graph settings&quot; or modify the expression.
+        </p>
+
+        <Button
+          size="small"
+          color="primary"
+          variant="text"
+          onClick={() => handleChange(false)}
+        >
+          Switch to line chart
+        </Button>
+      </div>
+    </Alert>
+  );
+};
+
+export default WarningHeatmapToLine;

app/vmui/packages/vmui/src/pages/CustomPanel/WarningHeatmapToLine/style.scss

@@ -0,0 +1,7 @@
+@use "src/styles/variables" as *;
+
+.vm-warning-heatmap-to-line {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+}

app/vmui/packages/vmui/src/pages/CustomPanel/index.tsx

@@ -18,6 +18,7 @@ import WarningLimitSeries from "./WarningLimitSeries/WarningLimitSeries";
 import CustomPanelTabs from "./CustomPanelTabs";
 import { DisplayType } from "../../types";
 import DownloadReport from "./DownloadReport/DownloadReport";
+import WarningHeatmapToLine from "./WarningHeatmapToLine/WarningHeatmapToLine";
 
 const CustomPanel: FC = () => {
   useSetQueryParams();
@@ -93,6 +94,7 @@ const CustomPanel: FC = () => {
       />
       {showError && <Alert variant="error">{error}</Alert>}
       {showInstantQueryTip && <Alert variant="info"><InstantQueryTip/></Alert>}
+      <WarningHeatmapToLine/>
       {warning && (
         <WarningLimitSeries
           warning={warning}

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -69,7 +69,7 @@ const ExploreLogs: FC = () => {
   };
 
   const handleApplyFilter = (val: string) => {
-    setQuery(prev => `_stream: ${val === "other" ? "{}" : val} AND (${prev})`);
+    setQuery(prev => `${val} AND (${prev})`);
     setIsUpdatingQuery(true);
   };
 

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx

@@ -43,7 +43,7 @@ const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
         values,
         pairs,
       };
-    }).sort((a, b) => a.keysString.localeCompare(b.keysString)); // groups sorting
+    }).sort((a, b) => b.values.length - a.values.length); // groups sorting
   }, [logs, groupBy]);
 
   const handleToggleExpandAll = useCallback(() => {

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsHeaderItem.tsx

@@ -6,6 +6,7 @@ import { MouseEvent, useState } from "react";
 import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 import { useSearchParams } from "react-router-dom";
 import { LOGS_GROUP_BY, LOGS_URL_PARAMS } from "../../../constants/logs";
+import { convertToFieldFilter } from "../../../utils/logs";
 
 interface Props {
   pair: string;
@@ -23,8 +24,7 @@ const GroupLogsHeaderItem: FC<Props> = ({ pair, isHide }) => {
 
   const handleClickByPair = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
     e.stopPropagation();
-    const isKeyValue = /(.+)?=(".+")/.test(value);
-    const copyValue = isKeyValue ? `${value.replace(/=/, ": ")}` : `${groupBy}: "${value}"`;
+    const copyValue = convertToFieldFilter(value, groupBy);
     const isCopied = await copyToClipboard(copyValue);
     if (isCopied) {
       setCopied(value);

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx

@@ -40,7 +40,7 @@ const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"] }) => {
     return marked(log._msg.replace(/```/g, "\n```\n")) as string;
   }, [log._msg, markdownParsing]);
 
-  const fields = useMemo(() => Object.entries(log).filter(([key]) => key !== "_msg"), [log]);
+  const fields = useMemo(() => Object.entries(log), [log]);
   const hasFields = fields.length > 0;
 
   const displayMessage = useMemo(() => {

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss

@@ -268,7 +268,7 @@ $font-size-logs: var(--font-size-logs, $font-size-small);
       border: $border-divider;
       border-radius: $border-radius-small;
       overflow: auto;
-      height: 300px;
+      max-height: 300px;
       resize: vertical;
       font-family: $font-family-monospace;
       font-size: $font-size-logs;

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogHits.ts

@@ -4,6 +4,8 @@ import { ErrorTypes, TimeParams } from "../../../types";
 import { LogHits } from "../../../api/types";
 import { useSearchParams } from "react-router-dom";
 import { getHitsTimeParams } from "../../../utils/logs";
+import { LOGS_GROUP_BY, LOGS_LIMIT_HITS } from "../../../constants/logs";
+import { isEmptyObject } from "../../../utils/object";
 
 export const useFetchLogHits = (server: string, query: string) => {
   const [searchParams] = useSearchParams();
@@ -30,46 +32,12 @@ export const useFetchLogHits = (server: string, query: string) => {
         step: `${step}ms`,
         start: start.toISOString(),
         end: end.toISOString(),
-        field: "_stream" // In the future, this field can be made configurable
-
+        fields_limit: `${LOGS_LIMIT_HITS}`,
+        field: LOGS_GROUP_BY,
       })
     };
   };
 
-  const accumulateHits = (resultHit: LogHits, hit: LogHits) => {
-    resultHit.total = (resultHit.total || 0) + (hit.total || 0);
-    hit.timestamps.forEach((timestamp, i) => {
-      const index = resultHit.timestamps.findIndex(t => t === timestamp);
-      if (index === -1) {
-        resultHit.timestamps.push(timestamp);
-        resultHit.values.push(hit.values[i]);
-      } else {
-        resultHit.values[index] += hit.values[i];
-      }
-    });
-    return resultHit;
-  };
-
-  const getHitsWithTop = (hits: LogHits[]) => {
-    const topN = 5;
-    const defaultHit = { fields: {}, timestamps: [], values: [], total: 0 };
-
-    const hitsByTotal = hits.sort((a, b) => (b.total || 0) - (a.total || 0));
-    const result = [];
-
-    const otherHits: LogHits = hitsByTotal.slice(topN).reduce(accumulateHits, defaultHit);
-    if (otherHits.total) {
-      result.push(otherHits);
-    }
-
-    const topHits: LogHits[] = hitsByTotal.slice(0, topN);
-    if (topHits.length) {
-      result.push(...topHits);
-    }
-
-    return result;
-  };
-
   const fetchLogHits = useCallback(async (period: TimeParams) => {
     abortControllerRef.current.abort();
     abortControllerRef.current = new AbortController();
@@ -98,7 +66,7 @@ export const useFetchLogHits = (server: string, query: string) => {
         setError(error);
       }
 
-      setLogHits(!hits ? [] : getHitsWithTop(hits));
+      setLogHits(hits.map(markIsOther).sort(sortHits));
     } catch (e) {
       if (e instanceof Error && e.name !== "AbortError") {
         setError(String(e));
@@ -117,3 +85,18 @@ export const useFetchLogHits = (server: string, query: string) => {
     abortController: abortControllerRef.current
   };
 };
+
+
+// Helper function to check if a hit is "other"
+const markIsOther = (hit: LogHits) => ({
+  ...hit,
+  _isOther: isEmptyObject(hit.fields)
+});
+
+// Comparison function for sorting hits
+const sortHits = (a: LogHits, b: LogHits) => {
+  if (a._isOther !== b._isOther) {
+    return a._isOther ? -1 : 1; // "Other" hits first to avoid graph overlap
+  }
+  return b.total - a.total; // Sort remaining by total for better visibility
+};

app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/PredefinedPanel.tsx

@@ -119,6 +119,7 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({
         {title || ""}
       </h3>
       <GraphSettings
+        data={graphData || []}
         yaxis={yaxis}
         setYaxisLimits={setYaxisLimits}
         toggleEnableLimits={toggleEnableLimits}

app/vmui/packages/vmui/src/pages/QueryAnalyzer/QueryAnalyzerView/QueryAnalyzerView.tsx

@@ -20,6 +20,8 @@ import TableSettings from "../../../components/Table/TableSettings/TableSettings
 import { getColumns } from "../../../hooks/useSortedCategories";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
 import TableView from "../../../components/Views/TableView/TableView";
+import { useSearchParams } from "react-router-dom";
+import WarningHeatmapToLine from "../../CustomPanel/WarningHeatmapToLine/WarningHeatmapToLine";
 
 type Props = {
   data: DataAnalyzerType[];
@@ -28,6 +30,8 @@ type Props = {
 
 const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
   const { isMobile } = useDeviceDetect();
+  const [searchParams, setSearchParams] = useSearchParams();
+
   const { tableCompact } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
@@ -101,11 +105,16 @@ const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
     setQueries(tempQueries);
     setGraphData(tempGraphData);
     setLiveData(tempLiveData);
+
+    // reset display mode
+    searchParams.delete("display_mode");
+    setSearchParams(searchParams);
   }, [data]);
 
   useEffect(() => {
-    setIsHistogram(!!graphData && isHistogramData(graphData));
-  }, [graphData]);
+    const noSpecificDisplayMode = !searchParams.get("display_mode");
+    setIsHistogram(!!graphData && noSpecificDisplayMode && isHistogramData(graphData));
+  }, [graphData, searchParams]);
 
   return (
     <div
@@ -120,6 +129,7 @@ const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
           onDeleteClick={handleTraceDelete}
         />
       )}
+      <WarningHeatmapToLine/>
       <div
         className={classNames({
           "vm-block": true,
@@ -138,7 +148,9 @@ const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
             {displayType === "chart" && <GraphTips/>}
             {displayType === "chart" && (
               <GraphSettings
+                data={graphData || []}
                 yaxis={yaxis}
+                isHistogram={isHistogram}
                 setYaxisLimits={setYaxisLimits}
                 toggleEnableLimits={toggleEnableLimits}
                 spanGaps={{ value: spanGaps, onChange: setSpanGaps }}

app/vmui/packages/vmui/src/state/graph/reducer.ts

@@ -15,6 +15,7 @@ export interface GraphState {
   customStep: string
   yaxis: YaxisState
   isHistogram: boolean
+  isEmptyHistogram: boolean
   /** when true, null data values will not cause line breaks */
   spanGaps: boolean
 }
@@ -24,6 +25,7 @@ export type GraphAction =
   | { type: "SET_YAXIS_LIMITS", payload: AxisRange }
   | { type: "SET_CUSTOM_STEP", payload: string}
   | { type: "SET_IS_HISTOGRAM", payload: boolean }
+  | { type: "SET_IS_EMPTY_HISTOGRAM", payload: boolean }
   | { type: "SET_SPAN_GAPS", payload: boolean }
 
 export const initialGraphState: GraphState = {
@@ -32,6 +34,7 @@ export const initialGraphState: GraphState = {
     limits: { enable: false, range: { "1": [0, 0] } }
   },
   isHistogram: false,
+  isEmptyHistogram: false,
   spanGaps: false,
 };
 
@@ -69,6 +72,11 @@ export function reducer(state: GraphState, action: GraphAction): GraphState {
         ...state,
         isHistogram: action.payload
       };
+    case "SET_IS_EMPTY_HISTOGRAM":
+      return {
+        ...state,
+        isEmptyHistogram: action.payload
+      };
     case "SET_SPAN_GAPS":
       return {
         ...state,

app/vmui/packages/vmui/src/utils/logs.ts

@@ -1,6 +1,8 @@
 import { TimeParams } from "../types";
 import dayjs from "dayjs";
-import { LOGS_BARS_VIEW } from "../constants/logs";
+import { LOGS_BARS_VIEW, LOGS_GROUP_BY } from "../constants/logs";
+import { LogHits } from "../api/types";
+import { OTHER_HITS_LABEL } from "../components/Chart/BarHitsChart/hooks/useBarHitsOptions";
 
 export const getStreamPairs = (value: string): string[] => {
   const pairs = /^{.+}$/.test(value) ? value.slice(1, -1).split(",") : [value];
@@ -14,3 +16,27 @@ export const getHitsTimeParams = (period: TimeParams) => {
   const step = Math.ceil(totalSeconds / LOGS_BARS_VIEW) || 1;
   return { start, end, step };
 };
+
+export const convertToFieldFilter = (value: string, field = LOGS_GROUP_BY) => {
+  const isKeyValue = /(.+)?=(".+")/.test(value);
+
+  if (isKeyValue) {
+    return value.replace(/=/, ": ");
+  }
+
+  return `${field}: "${value}"`;
+};
+
+export const calculateTotalHits = (hits: LogHits[]): number => {
+  return hits.reduce((acc, item) => acc + (item.total || 0), 0);
+};
+
+export const sortLogHits = <T extends { label?: string }>(key: keyof T) => (a: T, b: T): number => {
+  if (a.label === OTHER_HITS_LABEL) return 1;
+  if (b.label === OTHER_HITS_LABEL) return -1;
+
+  const aValue = a[key] as unknown as number;
+  const bValue = b[key] as unknown as number;
+
+  return bValue - aValue;
+};

app/vmui/packages/vmui/src/utils/math.ts

@@ -57,3 +57,15 @@ export const getLastFromArray = (a: number[]) => {
     }
   }
 };
+
+export const formatNumberShort = (value: number) => {
+  if (value >= 1_000_000_000) {
+    return (value / 1_000_000_000).toFixed(1).replace(/\.0$/, "") + "B"; // Миллиарды
+  } else if (value >= 1_000_000) {
+    return (value / 1_000_000).toFixed(1).replace(/\.0$/, "") + "M"; // Миллионы
+  } else if (value >= 1_000) {
+    return (value / 1_000).toFixed(1).replace(/\.0$/, "") + "K"; // Тысячи
+  } else {
+    return value.toString(); // Для чисел меньше 1000
+  }
+};

app/vmui/packages/vmui/src/utils/object.ts

@@ -14,3 +14,7 @@ export function filterObject<T extends object>(
 export function compactObject<T extends object>(obj: T) {
   return filterObject(obj, (entry) => !!entry[1] || typeof entry[1] === "number");
 }
+
+export function isEmptyObject(obj: object) {
+  return Object.keys(obj).length === 0;
+}

app/vmui/packages/vmui/src/utils/storage.ts

@@ -3,7 +3,6 @@ export type StorageKeys = "AUTOCOMPLETE"
     | "QUERY_TRACING"
     | "SERIES_LIMITS"
     | "TABLE_COMPACT"
-    | "TABLE_COLUMNS"
     | "TIMEZONE"
     | "DISABLED_DEFAULT_TIMEZONE"
     | "THEME"

app/vmui/packages/vmui/src/utils/uplot/heatmap.ts

@@ -153,7 +153,10 @@ export const normalizeData = (buckets: MetricResult[], isHistogram?: boolean): M
   const totalHitsPerTimestamp: { [timestamp: number]: number } = {};
   vmBuckets.forEach(bucket =>
     bucket.values.forEach(([timestamp, value]) => {
-      totalHitsPerTimestamp[timestamp] = (totalHitsPerTimestamp[timestamp] || 0) + +value;
+      const valueNum = Number(value);
+      const number = isNaN(valueNum) ? 0 : valueNum;
+      const prevTotal = totalHitsPerTimestamp[timestamp] || 0;
+      totalHitsPerTimestamp[timestamp] = prevTotal + number;
     })
   );
 

apptest/testcase.go

@@ -2,6 +2,8 @@ package apptest
 
 import (
 	"fmt"
+	"os"
+	"path"
 	"testing"
 	"time"
 
@@ -134,6 +136,23 @@ func (c *vmcluster) ForceFlush(t *testing.T) {
 	}
 }
 
+// MustStartVmauth is a test helper function that starts an instance of
+// vmauth and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmauth(instance string, flags []string, configFileYAML string) *Vmauth {
+	tc.t.Helper()
+
+	configFilePath := path.Join(tc.t.TempDir(), "config.yaml")
+	if err := os.WriteFile(configFilePath, []byte(configFileYAML), os.ModePerm); err != nil {
+		tc.t.Fatalf("cannot init vmauth: config file write failed: %s", err)
+	}
+	app, err := StartVmauth(instance, flags, tc.cli, configFilePath)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
 // MustStartDefaultCluster starts a typical cluster configuration with default
 // flags.
 func (tc *TestCase) MustStartDefaultCluster() PrometheusWriteQuerier {

apptest/tests/multitenant_test.go

@@ -171,4 +171,49 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
+	// Delete series from specific tenant
+	vmselect.DeleteSeries(t, "foo_bar", apptest.QueryOpts{
+		Tenant: "5:15",
+	})
+	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
+        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"},
+        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
+              ]
+     }`)
+	wantSR.Sort()
+
+	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+		Tenant: "multitenant",
+		Start:  "2022-05-10T08:03:00.000Z",
+	})
+	gotSR.Sort()
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Delete series for multitenant with tenant filter
+	vmselect.DeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+		Tenant: "multitenant",
+	})
+
+	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+		`{"data": [
+        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
+        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
+              ]
+     }`)
+	wantSR.Sort()
+
+	gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{
+		Tenant: "multitenant",
+		Start:  "2022-05-10T08:03:00.000Z",
+	})
+	gotSR.Sort()
+	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
 }

apptest/tests/vmauth_routing_test.go

@@ -0,0 +1,182 @@
+package tests
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestVMAuthRouterWithAuth(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	var authorizedRequestsCount, unauthorizedRequestsCount int
+	backendWithAuth := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+		authorizedRequestsCount++
+	}))
+	defer backendWithAuth.Close()
+	backend := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
+		unauthorizedRequestsCount++
+	}))
+	defer backend.Close()
+
+	authConfig := fmt.Sprintf(`
+users:
+- name: user1
+  username: ba-username
+  password: ba-password
+  url_prefix: %s
+unauthorized_user:
+   url_map:
+   - src_paths:
+     - /backend/health
+     - /backend/ready
+     url_prefix: %s
+  `, backendWithAuth.URL, backend.URL)
+
+	vmauth := tc.MustStartVmauth("vmauth", nil, authConfig)
+
+	makeGetRequestExpectCode := func(prepareRequest func(*http.Request), expectCode int) {
+		t.Helper()
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s", vmauth.GetHTTPListenAddr()), nil)
+		if err != nil {
+			t.Fatalf("cannot build http.Request: %s", err)
+		}
+		prepareRequest(req)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("cannot make http.Get request for target=%q: %s", req.URL, err)
+		}
+		responseText, err := io.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("cannot read response body: %s", err)
+		}
+		resp.Body.Close()
+		if resp.StatusCode != expectCode {
+			t.Fatalf("unexpected http response code: %d, want: %d, response text: %s", resp.StatusCode, expectCode, responseText)
+		}
+	}
+	assertBackendsRequestsCount := func(expectAuthorized, expectUnauthorized int) {
+		t.Helper()
+		if expectAuthorized != authorizedRequestsCount {
+			t.Fatalf("expected to have %d authorized proxied requests, got: %d", expectAuthorized, authorizedRequestsCount)
+		}
+
+		if expectUnauthorized != unauthorizedRequestsCount {
+			t.Fatalf("expected to have %d unauthorized proxied requests, got: %d", expectUnauthorized, unauthorizedRequestsCount)
+		}
+
+	}
+
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.Path = "/backend/api"
+		r.URL.User = url.UserPassword("ba-username", "ba-password")
+	}, http.StatusOK)
+	assertBackendsRequestsCount(1, 0)
+
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.Path = "/backend/health"
+	}, http.StatusOK)
+	assertBackendsRequestsCount(1, 1)
+
+	// remove unauthorized section and proxy only specified path for authorized
+	vmauth.UpdateConfiguration(t, fmt.Sprintf(`
+users:
+- name: user1
+  username: ba-username
+  password: ba-password
+  url_map:
+  - src_paths:
+    - /backend/health
+    url_prefix: %s
+`, backendWithAuth.URL))
+
+	// ensure unauthorized requests no longer served
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.Path = "/backend/health"
+	}, http.StatusUnauthorized)
+	assertBackendsRequestsCount(1, 1)
+
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.User = url.UserPassword("ba-username", "ba-password")
+		r.URL.Path = "/backend/health"
+	}, http.StatusOK)
+	assertBackendsRequestsCount(2, 1)
+
+	// url path is missing at proxy configuration
+	makeGetRequestExpectCode(func(r *http.Request) {
+		r.URL.User = url.UserPassword("ba-username", "ba-password")
+		r.URL.Path = "/backend"
+	}, http.StatusBadRequest)
+	assertBackendsRequestsCount(2, 1)
+
+}
+
+func TestVMAuthRouterWithInternalAddr(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	var proxiedRequestsCount int
+	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		proxiedRequestsCount++
+	}))
+	defer backend.Close()
+
+	authConfig := fmt.Sprintf(`
+unauthorized_user:
+   url_prefix: %s
+  `, backend.URL)
+
+	const (
+		// it's not possible to use random ports
+		// since it makes test flaky
+		listenPortPublic  = "50127"
+		listenPortPrivate = "50126"
+	)
+
+	vmauthFlags := []string{
+		fmt.Sprintf("-httpListenAddr=127.0.0.1:%s", listenPortPublic),
+		fmt.Sprintf("-httpInternalListenAddr=127.0.0.1:%s", listenPortPrivate),
+		"-flagsAuthKey=protected",
+	}
+	vmauth := tc.MustStartVmauth("vmauth", vmauthFlags, authConfig)
+
+	makeGetRequestExpectCode := func(targetURL string, expectCode int) {
+		t.Helper()
+		resp, err := http.Get(targetURL)
+		if err != nil {
+			t.Fatalf("cannot make http.Get request for target=%q: %s", targetURL, err)
+		}
+		responseText, err := io.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("cannot read response body: %s", err)
+		}
+		resp.Body.Close()
+		if resp.StatusCode != expectCode {
+			t.Fatalf("unexpected http response code: %d, want: %d, response text: %s", resp.StatusCode, expectCode, responseText)
+		}
+	}
+	assertBackendRequestsCount := func(expected int) {
+		t.Helper()
+		if proxiedRequestsCount != expected {
+			t.Fatalf("expected to have %d proxied requests, got: %d", expected, proxiedRequestsCount)
+		}
+	}
+	// built-in http server must reject request, since it protected with authKey
+	makeGetRequestExpectCode(fmt.Sprintf("http://127.0.0.1:%s/flags", listenPortPrivate), http.StatusUnauthorized)
+	assertBackendRequestsCount(0)
+
+	makeGetRequestExpectCode(fmt.Sprintf("http://127.0.0.1:%s/flags", listenPortPublic), http.StatusOK)
+	assertBackendRequestsCount(1)
+
+	// reload config and ensure that vmauth no longer proxies requests to the backend
+	vmauth.UpdateConfiguration(t, "")
+	makeGetRequestExpectCode(fmt.Sprintf("http://127.0.0.1:%s/flags", listenPortPrivate), http.StatusUnauthorized)
+	assertBackendRequestsCount(1)
+}

apptest/vmauth.go

@@ -0,0 +1,82 @@
+package apptest
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"syscall"
+	"testing"
+	"time"
+)
+
+var httpBuilitinListenAddrRE = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
+
+// Vmauth holds the state of a vmauth app and provides vmauth-specific
+// functions.
+type Vmauth struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr string
+	configFilePath string
+	cli            *Client
+}
+
+// StartVmauth starts an instance of vmauth with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
+func StartVmauth(instance string, flags []string, cli *Client, configFilePath string) (*Vmauth, error) {
+	extractREs := []*regexp.Regexp{
+		httpBuilitinListenAddrRE,
+	}
+
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmauth", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr": "127.0.0.1:0",
+			"-auth.config":    configFilePath,
+		},
+		extractREs: extractREs,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vmauth{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
+		httpListenAddr: stderrExtracts[0],
+		configFilePath: configFilePath,
+		cli:            cli,
+	}, nil
+}
+
+// UpdateConfiguration performs configuration file reload for app and waits for configuration apply
+//
+// Due to second prescision of config reload metric, config cannot be reloaded more than 1 time in a second
+func (app *Vmauth) UpdateConfiguration(t *testing.T, configFileYAML string) {
+	t.Helper()
+	ct := int(time.Now().Unix())
+	if err := os.WriteFile(app.configFilePath, []byte(configFileYAML), os.ModePerm); err != nil {
+		t.Fatalf("unexpected error at UpdateConfiguration, cannot write configFile content: %s", err)
+	}
+	if err := app.process.Signal(syscall.SIGHUP); err != nil {
+		t.Fatalf("unexpected signal error: %s", err)
+	}
+	for range 10 {
+		ts := app.GetIntMetric(t, "vmauth_config_last_reload_success_timestamp_seconds")
+		if ts < ct {
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+		return
+	}
+	t.Fatalf("timeout waiting for config reload success")
+}
+
+// GetHTTPListenAddr returns listen http addr
+func (app *Vmauth) GetHTTPListenAddr() string {
+	return app.httpListenAddr
+}

apptest/vmselect.go

@@ -117,6 +117,22 @@ func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts
 	return NewPrometheusAPIV1SeriesResponse(t, res)
 }
 
+// DeleteSeries sends a query to a /prometheus/api/v1/admin/tsdb/delete_series
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1admintsdbdelete_series
+func (app *Vmselect) DeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/delete/%s/prometheus/api/v1/admin/tsdb/delete_series", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	res := app.cli.PostForm(t, seriesURL, values, http.StatusNoContent)
+	if res != "" {
+		t.Fatalf("unexpected non-empty DeleteSeries response=%q", res)
+	}
+}
+
 // String returns the string representation of the vmselect app state.
 func (app *Vmselect) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)

dashboards/victorialogs.json

@@ -1686,7 +1686,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/victoriametrics-cluster.json

@@ -1933,7 +1933,7 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3167,7 +3167,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on many other factors such as the number of labels per time series and the length of label values. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
+          "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \\n\\nThe lower the better. \\n\\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \\n* Ingestion of completely new, not seen before time series;\\n* [Re-routing](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\\n* Not enough memory to maintain big enough caches for the current workload.\\n\\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \\n\\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/victoriametrics.json

@@ -1981,7 +1981,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3448,7 +3448,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on many other factors such as the number of labels per time series and the length of label values. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
+          "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \\n\\nThe lower the better. \\n\\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \\n* Ingestion of completely new, not seen before time series;\\n* Not enough memory to maintain big enough caches for the current workload.\\n\\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \\n\\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/victorialogs.json

@@ -1687,7 +1687,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,

dashboards/vm/victoriametrics-cluster.json

@@ -1934,7 +1934,7 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3168,7 +3168,7 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on many other factors such as the number of labels per time series and the length of label values. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
+          "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \\n\\nThe lower the better. \\n\\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \\n* Ingestion of completely new, not seen before time series;\\n* [Re-routing](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\\n* Not enough memory to maintain big enough caches for the current workload.\\n\\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \\n\\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/victoriametrics.json

@@ -1982,7 +1982,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
+              "expr": "max(\n    rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n    /\n    process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(instance)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -3449,7 +3449,7 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on many other factors such as the number of labels per time series and the length of label values. See also [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
+          "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \\n\\nThe lower the better. \\n\\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \\n* Ingestion of completely new, not seen before time series;\\n* Not enough memory to maintain big enough caches for the current workload.\\n\\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series). \\n\\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/vmalert.json

@@ -1473,7 +1473,7 @@
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "max(\n  rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n  / \n  vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
+              "expr": "max(\n  rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n  / \n  process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,