vmsingle: return an error for the /status/metric_names_stats endpoints if the feature is not enabled with the -storage.trackMetricNamesStats flag

Signed-off-by: hagen1778 <roman@victoriametrics.com>

.gitignore

@@ -27,3 +27,4 @@ _site
 coverage.txt
 cspell.json
 *~
+deployment/docker/provisioning/plugins/

Makefile

@@ -504,7 +504,7 @@ fmt:
 	gofmt -l -w -s ./apptest
 
 vet:
-	go vet ./lib/...
+	GOEXPERIMENT=synctest go vet ./lib/...
 	go vet ./app/...
 	go vet ./apptest/...
 
@@ -513,29 +513,29 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	go test ./lib/... ./app/...
+	GOEXPERIMENT=synctest go test ./lib/... ./app/...
 
 test-race:
-	go test -race ./lib/... ./app/...
+	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
 
 test-pure:
-	CGO_ENABLED=0 go test ./lib/... ./app/...
+	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test: victoria-metrics vmagent vmalert vmauth
 	go test ./apptest/... -skip="^TestCluster.*"
 
 benchmark:
-	go test -bench=. ./lib/...
+	GOEXPERIMENT=synctest go test -bench=. ./lib/...
 	go test -bench=. ./app/...
 
 benchmark-pure:
-	CGO_ENABLED=0 go test -bench=. ./lib/...
+	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
@@ -564,7 +564,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	golangci-lint run
+	GOEXPERIMENT=synctest golangci-lint run
 
 install-golangci-lint:
 	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7

app/vlinsert/insertutil/common_params.go

@@ -219,6 +219,42 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 	}
 }
 
+// InsertRowProcessor is used by native data ingestion protocol parser.
+type InsertRowProcessor interface {
+	// AddInsertRow must add r to the underlying storage.
+	AddInsertRow(r *logstorage.InsertRow)
+}
+
+// AddInsertRow adds r to lmp.
+func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
+	lmp.rowsIngestedTotal.Inc()
+	n := logstorage.EstimatedJSONRowLen(r.Fields)
+	lmp.bytesIngestedTotal.Add(n)
+
+	if len(r.Fields) > *MaxFieldsPerLine {
+		line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
+		rowsDroppedTotalTooManyFields.Inc()
+		return
+	}
+
+	lmp.mu.Lock()
+	defer lmp.mu.Unlock()
+
+	lmp.lr.MustAddInsertRow(r)
+
+	if lmp.cp.Debug {
+		s := lmp.lr.GetRowString(0)
+		lmp.lr.ResetKeepSettings()
+		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
+		rowsDroppedTotalDebug.Inc()
+		return
+	}
+	if lmp.lr.NeedFlush() {
+		lmp.flushLocked()
+	}
+}
+
 // flushLocked must be called under locked lmp.mu.
 func (lmp *logMessageProcessor) flushLocked() {
 	lmp.lastFlushTime = time.Now()

app/vlinsert/internalinsert/internalinsert.go

@@ -0,0 +1,96 @@
+package internalinsert
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+)
+
+var (
+	disableInsert  = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
+	maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
+)
+
+// RequestHandler processes /internal/insert requests.
+func RequestHandler(w http.ResponseWriter, r *http.Request) {
+	if *disableInsert {
+		httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable command-line flag")
+		return
+	}
+
+	startTime := time.Now()
+	if r.Method != "POST" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+	version := r.FormValue("version")
+	if version != netinsert.ProtocolVersion {
+		httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
+		return
+	}
+
+	requestsTotal.Inc()
+
+	cp, err := insertutil.GetCommonParams(r)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+
+	encoding := r.Header.Get("Content-Encoding")
+	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
+		lmp := cp.NewLogMessageProcessor("internalinsert", false)
+		irp := lmp.(insertutil.InsertRowProcessor)
+		err := parseData(irp, data)
+		lmp.MustClose()
+		return err
+	})
+	if err != nil {
+		errorsTotal.Inc()
+		httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
+		return
+	}
+
+	requestDuration.UpdateDuration(startTime)
+}
+
+func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
+	r := logstorage.GetInsertRow()
+	src := data
+	i := 0
+	for len(src) > 0 {
+		tail, err := r.UnmarshalInplace(src)
+		if err != nil {
+			return fmt.Errorf("cannot parse row #%d: %s", i, err)
+		}
+		src = tail
+		i++
+
+		irp.AddInsertRow(r)
+	}
+	logstorage.PutInsertRow(r)
+
+	return nil
+}
+
+var (
+	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
+	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
+
+	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
+)

app/vlinsert/main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
@@ -28,6 +29,11 @@ func Stop() {
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 
+	if path == "/internal/insert" {
+		internalinsert.RequestHandler(w, r)
+		return true
+	}
+
 	if !strings.HasPrefix(path, "/insert/") {
 		// Skip requests, which do not start with /insert/, since these aren't our requests.
 		return false

app/vlselect/internalselect/internalselect.go

@@ -0,0 +1,324 @@
+package internalselect
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+)
+
+var disableSelect = flag.Bool("internalselect.disable", false, "Whether to disable /internal/select/* HTTP endpoints")
+
+// RequestHandler processes requests to /internal/select/*
+func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if *disableSelect {
+		httpserver.Errorf(w, r, "requests to /internal/select/* are disabled with -internalselect.disable command-line flag")
+		return
+	}
+
+	startTime := time.Now()
+
+	path := r.URL.Path
+	rh := requestHandlers[path]
+	if rh == nil {
+		httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
+		return
+	}
+
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
+	if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
+		httpserver.Errorf(w, r, "%s", err)
+		// The return is skipped intentionally in order to track the duration of failed queries.
+	}
+	metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
+}
+
+var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
+	"/internal/select/query":               processQueryRequest,
+	"/internal/select/field_names":         processFieldNamesRequest,
+	"/internal/select/field_values":        processFieldValuesRequest,
+	"/internal/select/stream_field_names":  processStreamFieldNamesRequest,
+	"/internal/select/stream_field_values": processStreamFieldValuesRequest,
+	"/internal/select/streams":             processStreamsRequest,
+	"/internal/select/stream_ids":          processStreamIDsRequest,
+}
+
+func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	w.Header().Set("Content-Type", "application/octet-stream")
+
+	var wLock sync.Mutex
+	var dataLenBuf []byte
+
+	sendBuf := func(bb *bytesutil.ByteBuffer) error {
+		if len(bb.B) == 0 {
+			return nil
+		}
+
+		data := bb.B
+		if !cp.DisableCompression {
+			bufLen := len(bb.B)
+			bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
+			data = bb.B[bufLen:]
+		}
+
+		wLock.Lock()
+		dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
+		_, err := w.Write(dataLenBuf)
+		if err == nil {
+			_, err = w.Write(data)
+		}
+		wLock.Unlock()
+
+		// Reset the sent buf
+		bb.Reset()
+
+		return err
+	}
+
+	var bufs atomicutil.Slice[bytesutil.ByteBuffer]
+
+	var errGlobalLock sync.Mutex
+	var errGlobal error
+
+	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
+		if errGlobal != nil {
+			return
+		}
+
+		bb := bufs.Get(workerID)
+
+		bb.B = db.Marshal(bb.B)
+
+		if len(bb.B) < 1024*1024 {
+			// Fast path - the bb is too small to be sent to the client yet.
+			return
+		}
+
+		// Slow path - the bb must be sent to the client.
+		if err := sendBuf(bb); err != nil {
+			errGlobalLock.Lock()
+			if errGlobal != nil {
+				errGlobal = err
+			}
+			errGlobalLock.Unlock()
+		}
+	}
+
+	if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
+		return err
+	}
+	if errGlobal != nil {
+		return errGlobal
+	}
+
+	// Send the remaining data
+	for _, bb := range bufs.GetSlice() {
+		if err := sendBuf(bb); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
+	if err != nil {
+		return fmt.Errorf("cannot obtain field names: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
+}
+
+func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldName := r.FormValue("field")
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain field values: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
+}
+
+func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
+	if err != nil {
+		return fmt.Errorf("cannot obtain stream field names: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
+}
+
+func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	fieldName := r.FormValue("field")
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain stream field values: %w", err)
+	}
+
+	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
+}
+
+func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain streams: %w", err)
+	}
+
+	return writeValuesWithHits(w, streams, cp.DisableCompression)
+}
+
+func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
+	cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
+	if err != nil {
+		return err
+	}
+
+	limit, err := getInt64FromRequest(r, "limit")
+	if err != nil {
+		return err
+	}
+
+	streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
+	if err != nil {
+		return fmt.Errorf("cannot obtain streams: %w", err)
+	}
+
+	return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
+}
+
+type commonParams struct {
+	TenantIDs []logstorage.TenantID
+	Query     *logstorage.Query
+
+	DisableCompression bool
+}
+
+func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
+	version := r.FormValue("version")
+	if version != expectedProtocolVersion {
+		return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
+	}
+
+	tenantIDsStr := r.FormValue("tenant_ids")
+	tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
+	if err != nil {
+		return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
+	}
+
+	timestamp, err := getInt64FromRequest(r, "timestamp")
+	if err != nil {
+		return nil, err
+	}
+
+	qStr := r.FormValue("query")
+	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
+	if err != nil {
+		return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
+	}
+
+	s := r.FormValue("disable_compression")
+	disableCompression, err := strconv.ParseBool(s)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
+	}
+
+	cp := &commonParams{
+		TenantIDs: tenantIDs,
+		Query:     q,
+
+		DisableCompression: disableCompression,
+	}
+	return cp, nil
+}
+
+func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
+	var b []byte
+	for i := range vhs {
+		b = vhs[i].Marshal(b)
+	}
+
+	if !disableCompression {
+		b = zstd.CompressLevel(nil, b, 1)
+	}
+
+	w.Header().Set("Content-Type", "application/octet-stream")
+
+	if _, err := w.Write(b); err != nil {
+		return fmt.Errorf("cannot send response to the client: %w", err)
+	}
+
+	return nil
+}
+
+func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
+	s := r.FormValue(argName)
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
+	}
+	return n, nil
+}

app/vlselect/logsql/buffered_writer.go

@@ -1,47 +0,0 @@
-package logsql
-
-import (
-	"bufio"
-	"io"
-	"sync"
-)
-
-func getBufferedWriter(w io.Writer) *bufferedWriter {
-	v := bufferedWriterPool.Get()
-	if v == nil {
-		return &bufferedWriter{
-			bw: bufio.NewWriter(w),
-		}
-	}
-	bw := v.(*bufferedWriter)
-	bw.bw.Reset(w)
-	return bw
-}
-
-func putBufferedWriter(bw *bufferedWriter) {
-	bw.reset()
-	bufferedWriterPool.Put(bw)
-}
-
-var bufferedWriterPool sync.Pool
-
-type bufferedWriter struct {
-	mu sync.Mutex
-	bw *bufio.Writer
-}
-
-func (bw *bufferedWriter) reset() {
-	// nothing to do
-}
-
-func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
-	bw.mu.Lock()
-	_, _ = bw.bw.Write(p)
-	bw.mu.Unlock()
-}
-
-func (bw *bufferedWriter) FlushIgnoreErrors() {
-	bw.mu.Lock()
-	_ = bw.bw.Flush()
-	bw.mu.Unlock()
-}

app/vlselect/logsql/logsql.go

@@ -3,6 +3,7 @@ package logsql
 import (
 	"context"
 	"fmt"
+	"io"
 	"math"
 	"net/http"
 	"regexp"
@@ -11,12 +12,14 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
@@ -57,10 +60,13 @@ func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 
 	var mLock sync.Mutex
 	m := make(map[string][]facetEntry)
-	writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		if rowsCount == 0 {
 			return
 		}
+
+		columns := db.Columns
 		if len(columns) != 3 {
 			logger.Panicf("BUG: expecting 3 columns; got %d columns", len(columns))
 		}
@@ -156,17 +162,19 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		if rowsCount == 0 {
 			return
 		}
 
+		columns := db.Columns
 		timestampValues := columns[0].Values
 		hitsValues := columns[len(columns)-1].Values
 		columns = columns[1 : len(columns)-1]
 
 		bb := blockResultPool.Get()
-		for i := range timestamps {
+		for i := 0; i < rowsCount; i++ {
 			timestampStr := strings.Clone(timestampValues[i])
 			hitsStr := strings.Clone(hitsValues[i])
 			hits, err := strconv.ParseUint(hitsStr, 10, 64)
@@ -205,6 +213,8 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	WriteHitsSeries(w, m)
 }
 
+var blockResultPool bytesutil.ByteBufferPool
+
 func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
 	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
 		return m
@@ -536,7 +546,7 @@ var liveTailRequests = metrics.NewCounter(`vl_live_tailing_requests`)
 const tailOffsetNsecs = 5e9
 
 type logRow struct {
-	timestamp int64
+	timestamp string
 	fields    []logstorage.Field
 }
 
@@ -552,7 +562,7 @@ type tailProcessor struct {
 	mu sync.Mutex
 
 	perStreamRows  map[string][]logRow
-	lastTimestamps map[string]int64
+	lastTimestamps map[string]string
 
 	err error
 }
@@ -562,12 +572,12 @@ func newTailProcessor(cancel func()) *tailProcessor {
 		cancel: cancel,
 
 		perStreamRows:  make(map[string][]logRow),
-		lastTimestamps: make(map[string]int64),
+		lastTimestamps: make(map[string]string),
 	}
 }
 
-func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-	if len(timestamps) == 0 {
+func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
+	if db.RowsCount() == 0 {
 		return
 	}
 
@@ -579,14 +589,8 @@ func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logsto
 	}
 
 	// Make sure columns contain _time field, since it is needed for proper tail work.
-	hasTime := false
-	for _, c := range columns {
-		if c.Name == "_time" {
-			hasTime = true
-			break
-		}
-	}
-	if !hasTime {
+	timestamps, ok := db.GetTimestamps()
+	if !ok {
 		tp.err = fmt.Errorf("missing _time field")
 		tp.cancel()
 		return
@@ -595,8 +599,8 @@ func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logsto
 	// Copy block rows to tp.perStreamRows
 	for i, timestamp := range timestamps {
 		streamID := ""
-		fields := make([]logstorage.Field, len(columns))
-		for j, c := range columns {
+		fields := make([]logstorage.Field, len(db.Columns))
+		for j, c := range db.Columns {
 			name := strings.Clone(c.Name)
 			value := strings.Clone(c.Values[i])
 
@@ -688,12 +692,15 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	m := make(map[string]*statsSeries)
 	var mLock sync.Mutex
 
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+
+		columns := db.Columns
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := range timestamps {
+		for i := 0; i < rowsCount; i++ {
 			// Do not move q.GetTimestamp() outside writeBlock, since ts
 			// must be initialized to query timestamp for every processed log row.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8312
@@ -802,12 +809,14 @@ func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *htt
 	var rowsLock sync.Mutex
 
 	timestamp := q.GetTimestamp()
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		columns := db.Columns
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := range timestamps {
+		for i := 0; i < rowsCount; i++ {
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if slices.Contains(byFields, c.Name) {
@@ -869,11 +878,21 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bw := getBufferedWriter(w)
+	sw := &syncWriter{
+		w: w,
+	}
+
+	var bwShards atomicutil.Slice[bufferedWriter]
+	bwShards.Init = func(shard *bufferedWriter) {
+		shard.sw = sw
+	}
 	defer func() {
-		bw.FlushIgnoreErrors()
-		putBufferedWriter(bw)
+		shards := bwShards.GetSlice()
+		for _, shard := range shards {
+			shard.FlushIgnoreErrors()
+		}
 	}()
+
 	w.Header().Set("Content-Type", "application/stream+json")
 
 	if limit > 0 {
@@ -883,32 +902,34 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 				httpserver.Errorf(w, r, "%s", err)
 				return
 			}
-			bb := blockResultPool.Get()
-			b := bb.B
+			bw := bwShards.Get(0)
 			for i := range rows {
-				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
-				b = append(b, '\n')
-				bw.WriteIgnoreErrors(b)
+				bw.buf = logstorage.MarshalFieldsToJSON(bw.buf, rows[i].fields)
+				bw.buf = append(bw.buf, '\n')
+				if len(bw.buf) > 16*1024 {
+					bw.FlushIgnoreErrors()
+				}
 			}
-			bb.B = b
-			blockResultPool.Put(bb)
 			return
 		}
 
 		q.AddPipeLimit(uint64(limit))
 	}
 
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
-		if len(columns) == 0 || len(columns[0].Values) == 0 {
+	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
+		rowsCount := db.RowsCount()
+		if rowsCount == 0 {
 			return
 		}
+		columns := db.Columns
 
-		bb := blockResultPool.Get()
-		for i := range timestamps {
-			WriteJSONRow(bb, columns, i)
+		bw := bwShards.Get(workerID)
+		for i := 0; i < rowsCount; i++ {
+			WriteJSONRow(bw, columns, i)
+			if len(bw.buf) > 16*1024 {
+				bw.FlushIgnoreErrors()
+			}
 		}
-		bw.WriteIgnoreErrors(bb.B)
-		blockResultPool.Put(bb)
 	}
 
 	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
@@ -917,14 +938,37 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 	}
 }
 
-var blockResultPool bytesutil.ByteBufferPool
-
-type row struct {
-	timestamp int64
-	fields    []logstorage.Field
+type syncWriter struct {
+	mu sync.Mutex
+	w  io.Writer
 }
 
-func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+func (sw *syncWriter) Write(p []byte) (int, error) {
+	sw.mu.Lock()
+	n, err := sw.w.Write(p)
+	sw.mu.Unlock()
+	return n, err
+}
+
+type bufferedWriter struct {
+	buf []byte
+	sw  *syncWriter
+}
+
+func (bw *bufferedWriter) Write(p []byte) (int, error) {
+	bw.buf = append(bw.buf, p...)
+
+	// Do not send bw.buf to bw.sw here, since the data at bw.buf may be incomplete (it must end with '\n')
+
+	return len(p), nil
+}
+
+func (bw *bufferedWriter) FlushIgnoreErrors() {
+	_, _ = bw.sw.Write(bw.buf)
+	bw.buf = bw.buf[:0]
+}
+
+func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
 
@@ -993,7 +1037,7 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 	}
 }
 
-func getLastNRows(rows []row, limit int) []row {
+func getLastNRows(rows []logRow, limit int) []logRow {
 	sort.Slice(rows, func(i, j int) bool {
 		return rows[i].timestamp < rows[j].timestamp
 	})
@@ -1003,18 +1047,31 @@ func getLastNRows(rows []row, limit int) []row {
 	return rows
 }
 
-func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
+func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
 	ctxWithCancel, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	var rows []row
+	var missingTimeColumn atomic.Bool
+	var rows []logRow
 	var rowsLock sync.Mutex
-	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	writeBlock := func(_ uint, db *logstorage.DataBlock) {
+		if missingTimeColumn.Load() {
+			return
+		}
+
+		columns := db.Columns
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
 
+		timestamps, ok := db.GetTimestamps()
+		if !ok {
+			missingTimeColumn.Store(true)
+			cancel()
+			return
+		}
+
 		for i, timestamp := range timestamps {
 			fields := make([]logstorage.Field, len(columns))
 			for j := range columns {
@@ -1024,7 +1081,7 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			}
 
 			rowsLock.Lock()
-			rows = append(rows, row{
+			rows = append(rows, logRow{
 				timestamp: timestamp,
 				fields:    fields,
 			})
@@ -1035,11 +1092,13 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			cancel()
 		}
 	}
-	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
-		return nil, err
+	err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock)
+
+	if missingTimeColumn.Load() {
+		return nil, fmt.Errorf("missing _time column in the result for the query [%s]", q)
 	}
 
-	return rows, nil
+	return rows, err
 }
 
 func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {

app/vlselect/main.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/internalselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -72,7 +73,7 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 
-	if !strings.HasPrefix(path, "/select/") {
+	if !strings.HasPrefix(path, "/select/") && !strings.HasPrefix(path, "/internal/select/") {
 		// Skip requests, which do not start with /select/, since these aren't our requests.
 		return false
 	}
@@ -119,12 +120,24 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	defer decRequestConcurrency()
 
+	if strings.HasPrefix(path, "/internal/select/") {
+		// Process internal request from vlselect without timeout (e.g. use ctx instead of ctxWithTimeout),
+		// since the timeout must be controlled by the vlselect.
+		internalselect.RequestHandler(ctx, w, r)
+		return true
+	}
+
 	ok := processSelectRequest(ctxWithTimeout, w, r, path)
 	if !ok {
 		return false
 	}
 
-	err := ctxWithTimeout.Err()
+	logRequestErrorIfNeeded(ctxWithTimeout, w, r, startTime)
+	return true
+}
+
+func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http.Request, startTime time.Time) {
+	err := ctx.Err()
 	switch err {
 	case nil:
 		// nothing to do
@@ -140,8 +153,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	default:
 		httpserver.Errorf(w, r, "unexpected error: %s", err)
 	}
-
-	return true
 }
 
 func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

app/vlselect/vmui/index.html

@@ -35,10 +35,10 @@
   <meta property="og:title" content="UI for VictoriaLogs">
   <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
   <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-BgdvCSTM.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
+  <script type="module" crossorigin src="./assets/index-BMcUMlYJ.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-BSp13qCn.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-u4IOGr0E.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-sXHL6qTd.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vlstorage/main.go

@@ -10,11 +10,14 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
@@ -39,14 +42,55 @@ var (
 		"the storage stops accepting new data")
 
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
+
+	storageNodeAddrs = flagutil.NewArrayString("storageNode", "Comma-separated list of TCP addresses for storage nodes to route the ingested logs to and to send select queries to. "+
+		"If the list is empty, then the ingested logs are stored and queried locally from -storageDataPath")
+	insertConcurrency        = flag.Int("insert.concurrency", 2, "The average number of concurrent data ingestion requests, which can be sent to every -storageNode")
+	insertDisableCompression = flag.Bool("insert.disableCompression", false, "Whether to disable compression when sending the ingested data to -storageNode nodes. "+
+		"Disabled compression reduces CPU usage at the cost of higher network usage")
+	selectDisableCompression = flag.Bool("select.disableCompression", false, "Whether to disable compression for select query responses received from -storageNode nodes. "+
+		"Disabled compression reduces CPU usage at the cost of higher network usage")
+
+	storageNodeUsername     = flagutil.NewArrayString("storageNode.username", "Optional basic auth username to use for the corresponding -storageNode")
+	storageNodePassword     = flagutil.NewArrayString("storageNode.password", "Optional basic auth password to use for the corresponding -storageNode")
+	storageNodePasswordFile = flagutil.NewArrayString("storageNode.passwordFile", "Optional path to basic auth password to use for the corresponding -storageNode. "+
+		"The file is re-read every second")
+	storageNodeBearerToken     = flagutil.NewArrayString("storageNode.bearerToken", "Optional bearer auth token to use for the corresponding -storageNode")
+	storageNodeBearerTokenFile = flagutil.NewArrayString("storageNode.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -storageNode. "+
+		"The token is re-read from the file every second")
+
+	storageNodeTLS = flagutil.NewArrayBool("storageNode.tls", "Whether to use TLS (HTTPS) protocol for communicating with the corresponding -storageNode. "+
+		"By default communication is performed via HTTP")
+	storageNodeTLSCAFile = flagutil.NewArrayString("storageNode.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -storageNode. "+
+		"By default, system CA is used")
+	storageNodeTLSCertFile = flagutil.NewArrayString("storageNode.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+		"to the corresponding -storageNode")
+	storageNodeTLSKeyFile    = flagutil.NewArrayString("storageNode.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -storageNode")
+	storageNodeTLSServerName = flagutil.NewArrayString("storageNode.tlsServerName", "Optional TLS server name to use for connections to the corresponding -storageNode. "+
+		"By default, the server name from -storageNode is used")
+	storageNodeTLSInsecureSkipVerify = flagutil.NewArrayBool("storageNode.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -storageNode")
 )
 
+var localStorage *logstorage.Storage
+var localStorageMetrics *metrics.Set
+
+var netstorageInsert *netinsert.Storage
+var netstorageSelect *netselect.Storage
+
 // Init initializes vlstorage.
 //
 // Stop must be called when vlstorage is no longer needed
 func Init() {
-	if strg != nil {
-		logger.Panicf("BUG: Init() has been already called")
+	if len(*storageNodeAddrs) == 0 {
+		initLocalStorage()
+	} else {
+		initNetworkStorage()
+	}
+}
+
+func initLocalStorage() {
+	if localStorage != nil {
+		logger.Panicf("BUG: initLocalStorage() has been already called")
 	}
 
 	if retentionPeriod.Duration() < 24*time.Hour {
@@ -63,60 +107,139 @@ func Init() {
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
-	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
+	localStorage = logstorage.MustOpenStorage(*storageDataPath, cfg)
 
 	var ss logstorage.StorageStats
-	strg.UpdateStats(&ss)
+	localStorage.UpdateStats(&ss)
 	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
 		"smallPartSize: %d bytes; bigPartSize: %d bytes",
 		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
 		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
 
-	// register storage metrics
-	storageMetrics = metrics.NewSet()
-	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, strg)
+	// register local storage metrics
+	localStorageMetrics = metrics.NewSet()
+	localStorageMetrics.RegisterMetricsWriter(func(w io.Writer) {
+		writeStorageMetrics(w, localStorage)
 	})
-	metrics.RegisterSet(storageMetrics)
+	metrics.RegisterSet(localStorageMetrics)
+}
+
+func initNetworkStorage() {
+	if netstorageInsert != nil || netstorageSelect != nil {
+		logger.Panicf("BUG: initNetworkStorage() has been already called")
+	}
+
+	authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
+	isTLSs := make([]bool, len(*storageNodeAddrs))
+	for i := range authCfgs {
+		authCfgs[i] = newAuthConfigForStorageNode(i)
+		isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
+	}
+
+	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
+	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
+
+	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
+	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
+
+	logger.Infof("initialized all the network services")
+}
+
+func newAuthConfigForStorageNode(argIdx int) *promauth.Config {
+	username := storageNodeUsername.GetOptionalArg(argIdx)
+	password := storageNodePassword.GetOptionalArg(argIdx)
+	passwordFile := storageNodePasswordFile.GetOptionalArg(argIdx)
+	var basicAuthCfg *promauth.BasicAuthConfig
+	if username != "" || password != "" || passwordFile != "" {
+		basicAuthCfg = &promauth.BasicAuthConfig{
+			Username:     username,
+			Password:     promauth.NewSecret(password),
+			PasswordFile: passwordFile,
+		}
+	}
+
+	token := storageNodeBearerToken.GetOptionalArg(argIdx)
+	tokenFile := storageNodeBearerTokenFile.GetOptionalArg(argIdx)
+
+	tlsCfg := &promauth.TLSConfig{
+		CAFile:             storageNodeTLSCAFile.GetOptionalArg(argIdx),
+		CertFile:           storageNodeTLSCertFile.GetOptionalArg(argIdx),
+		KeyFile:            storageNodeTLSKeyFile.GetOptionalArg(argIdx),
+		ServerName:         storageNodeTLSServerName.GetOptionalArg(argIdx),
+		InsecureSkipVerify: storageNodeTLSInsecureSkipVerify.GetOptionalArg(argIdx),
+	}
+
+	opts := &promauth.Options{
+		BasicAuth:       basicAuthCfg,
+		BearerToken:     token,
+		BearerTokenFile: tokenFile,
+		TLSConfig:       tlsCfg,
+	}
+	ac, err := opts.NewConfig()
+	if err != nil {
+		logger.Panicf("FATAL: cannot populate auth config for storage node #%d: %s", argIdx, err)
+	}
+
+	return ac
 }
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics, true)
-	storageMetrics = nil
+	if localStorage != nil {
+		metrics.UnregisterSet(localStorageMetrics, true)
+		localStorageMetrics = nil
 
-	strg.MustClose()
-	strg = nil
+		localStorage.MustClose()
+		localStorage = nil
+	} else {
+		netstorageInsert.MustStop()
+		netstorageInsert = nil
+
+		netstorageSelect.MustStop()
+		netstorageSelect = nil
+	}
 }
 
 // RequestHandler is a storage request handler.
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 	if path == "/internal/force_merge" {
-		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-			return true
-		}
-		// Run force merge in background
-		partitionNamePrefix := r.FormValue("partition_prefix")
-		go func() {
-			activeForceMerges.Inc()
-			defer activeForceMerges.Dec()
-			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-			startTime := time.Now()
-			strg.MustForceMerge(partitionNamePrefix)
-			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-		}()
-		return true
+		return processForceMerge(w, r)
 	}
 	return false
 }
 
-var strg *logstorage.Storage
-var storageMetrics *metrics.Set
+func processForceMerge(w http.ResponseWriter, r *http.Request) bool {
+	if localStorage == nil {
+		// Force merge isn't supported by non-local storage
+		return false
+	}
 
-// CanWriteData returns non-nil error if it cannot write data to vlstorage.
+	if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
+		return true
+	}
+
+	// Run force merge in background
+	partitionNamePrefix := r.FormValue("partition_prefix")
+	go func() {
+		activeForceMerges.Inc()
+		defer activeForceMerges.Dec()
+		logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
+		startTime := time.Now()
+		localStorage.MustForceMerge(partitionNamePrefix)
+		logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
+	}()
+	return true
+}
+
+// CanWriteData returns non-nil error if it cannot write data to vlstorage
 func CanWriteData() error {
-	if strg.IsReadOnly() {
+	if localStorage == nil {
+		// The data can be always written in non-local mode.
+		return nil
+	}
+
+	if localStorage.IsReadOnly() {
 		return &httpserver.ErrorWithStatusCode{
 			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
 				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
@@ -130,50 +253,77 @@ func CanWriteData() error {
 //
 // It is advised to call CanWriteData() before calling MustAddRows()
 func MustAddRows(lr *logstorage.LogRows) {
-	strg.MustAddRows(lr)
+	if localStorage != nil {
+		// Store lr in the local storage.
+		localStorage.MustAddRows(lr)
+	} else {
+		// Store lr across the remote storage nodes.
+		lr.ForEachRow(netstorageInsert.AddRow)
+	}
 }
 
 // RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
-	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
+func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+	if localStorage != nil {
+		return localStorage.RunQuery(ctx, tenantIDs, q, writeBlock)
+	}
+	return netstorageSelect.RunQuery(ctx, tenantIDs, q, writeBlock)
 }
 
 // GetFieldNames executes q and returns field names seen in results.
 func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return strg.GetFieldNames(ctx, tenantIDs, q)
+	if localStorage != nil {
+		return localStorage.GetFieldNames(ctx, tenantIDs, q)
+	}
+	return netstorageSelect.GetFieldNames(ctx, tenantIDs, q)
 }
 
 // GetFieldValues executes q and returns unique values for the fieldName seen in results.
 //
 // If limit > 0, then up to limit unique values are returned.
 func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	if localStorage != nil {
+		return localStorage.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	}
+	return netstorageSelect.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreamFieldNames executes q and returns stream field names seen in results.
 func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
+	if localStorage != nil {
+		return localStorage.GetStreamFieldNames(ctx, tenantIDs, q)
+	}
+	return netstorageSelect.GetStreamFieldNames(ctx, tenantIDs, q)
 }
 
 // GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
 //
 // If limit > 0, then up to limit unique stream field values are returned.
 func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	if localStorage != nil {
+		return localStorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	}
+	return netstorageSelect.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreams executes q and returns streams seen in query results.
 //
 // If limit > 0, then up to limit unique streams are returned.
 func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreams(ctx, tenantIDs, q, limit)
+	if localStorage != nil {
+		return localStorage.GetStreams(ctx, tenantIDs, q, limit)
+	}
+	return netstorageSelect.GetStreams(ctx, tenantIDs, q, limit)
 }
 
 // GetStreamIDs executes q and returns streamIDs seen in query results.
 //
 // If limit > 0, then up to limit unique streamIDs are returned.
 func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
+	if localStorage != nil {
+		return localStorage.GetStreamIDs(ctx, tenantIDs, q, limit)
+	}
+	return netstorageSelect.GetStreamIDs(ctx, tenantIDs, q, limit)
 }
 
 func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {

app/vlstorage/netinsert/netinsert.go

@@ -0,0 +1,369 @@
+package netinsert
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/valyala/fastrand"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+)
+
+// the maximum size of a single data block sent to storage node.
+const maxInsertBlockSize = 2 * 1024 * 1024
+
+// ProtocolVersion is the version of the data ingestion protocol.
+//
+// It must be changed every time the data encoding at /internal/insert HTTP endpoint is changed.
+const ProtocolVersion = "v1"
+
+// Storage is a network storage for sending data to remote storage nodes in the cluster.
+type Storage struct {
+	sns []*storageNode
+
+	disableCompression bool
+
+	srt *streamRowsTracker
+
+	pendingDataBuffers chan *bytesutil.ByteBuffer
+
+	stopCh chan struct{}
+	wg     sync.WaitGroup
+}
+
+type storageNode struct {
+	// scheme is http or https scheme to communicate with addr
+	scheme string
+
+	// addr is TCP address of storage node to send the ingested data to
+	addr string
+
+	// s is a storage, which holds the given storageNode
+	s *Storage
+
+	// c is an http client used for sending data blocks to addr.
+	c *http.Client
+
+	// ac is auth config used for setting request headers such as Authorization and Host.
+	ac *promauth.Config
+
+	// pendingData contains pending data, which must be sent to the storage node at the addr.
+	pendingDataMu        sync.Mutex
+	pendingData          *bytesutil.ByteBuffer
+	pendingDataLastFlush time.Time
+
+	// the unix timestamp until the storageNode is disabled for data writing.
+	disabledUntil atomic.Uint64
+}
+
+func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
+	tr := httputil.NewTransport(false, "vlinsert_backend")
+	tr.TLSHandshakeTimeout = 20 * time.Second
+	tr.DisableCompression = true
+
+	scheme := "http"
+	if isTLS {
+		scheme = "https"
+	}
+
+	sn := &storageNode{
+		scheme: scheme,
+		addr:   addr,
+		s:      s,
+		c: &http.Client{
+			Transport: ac.NewRoundTripper(tr),
+		},
+		ac: ac,
+
+		pendingData: &bytesutil.ByteBuffer{},
+	}
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		sn.backgroundFlusher()
+	}()
+
+	return sn
+}
+
+func (sn *storageNode) backgroundFlusher() {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-sn.s.stopCh:
+			return
+		case <-t.C:
+			sn.flushPendingData()
+		}
+	}
+}
+
+func (sn *storageNode) flushPendingData() {
+	sn.pendingDataMu.Lock()
+	if time.Since(sn.pendingDataLastFlush) < time.Second {
+		// nothing to flush
+		sn.pendingDataMu.Unlock()
+		return
+	}
+
+	pendingData := sn.grabPendingDataForFlushLocked()
+	sn.pendingDataMu.Unlock()
+
+	sn.mustSendInsertRequest(pendingData)
+}
+
+func (sn *storageNode) addRow(r *logstorage.InsertRow) {
+	bb := bbPool.Get()
+	b := bb.B
+
+	b = r.Marshal(b)
+
+	if len(b) > maxInsertBlockSize {
+		logger.Warnf("skipping too long log entry, since its length exceeds %d bytes; the actual log entry length is %d bytes; log entry contents: %s", maxInsertBlockSize, len(b), b)
+		return
+	}
+
+	var pendingData *bytesutil.ByteBuffer
+	sn.pendingDataMu.Lock()
+	if sn.pendingData.Len()+len(b) > maxInsertBlockSize {
+		pendingData = sn.grabPendingDataForFlushLocked()
+	}
+	sn.pendingData.MustWrite(b)
+	sn.pendingDataMu.Unlock()
+
+	bb.B = b
+	bbPool.Put(bb)
+
+	if pendingData != nil {
+		sn.mustSendInsertRequest(pendingData)
+	}
+}
+
+var bbPool bytesutil.ByteBufferPool
+
+func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
+	sn.pendingDataLastFlush = time.Now()
+	pendingData := sn.pendingData
+	sn.pendingData = <-sn.s.pendingDataBuffers
+
+	return pendingData
+}
+
+func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
+	defer func() {
+		pendingData.Reset()
+		sn.s.pendingDataBuffers <- pendingData
+	}()
+
+	err := sn.sendInsertRequest(pendingData)
+	if err == nil {
+		return
+	}
+
+	if !errors.Is(err, errTemporarilyDisabled) {
+		logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
+	}
+	for !sn.s.sendInsertRequestToAnyNode(pendingData) {
+		logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
+
+		t := timerpool.Get(time.Second)
+		select {
+		case <-sn.s.stopCh:
+			timerpool.Put(t)
+			logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
+			return
+		case <-t.C:
+			timerpool.Put(t)
+		}
+	}
+}
+
+func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
+	dataLen := pendingData.Len()
+	if dataLen == 0 {
+		// Nothing to send.
+		return nil
+	}
+
+	if sn.disabledUntil.Load() > fasttime.UnixTimestamp() {
+		return errTemporarilyDisabled
+	}
+
+	ctx, cancel := contextutil.NewStopChanContext(sn.s.stopCh)
+	defer cancel()
+
+	var body io.Reader
+	if !sn.s.disableCompression {
+		bb := zstdBufPool.Get()
+		defer zstdBufPool.Put(bb)
+
+		bb.B = zstd.CompressLevel(bb.B[:0], pendingData.B, 1)
+		body = bb.NewReader()
+	} else {
+		body = pendingData.NewReader()
+	}
+
+	reqURL := sn.getRequestURL("/internal/insert")
+	req, err := http.NewRequestWithContext(ctx, "POST", reqURL, body)
+	if err != nil {
+		logger.Panicf("BUG: unexpected error when creating an http request: %s", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	if !sn.s.disableCompression {
+		req.Header.Set("Content-Encoding", "zstd")
+	}
+	if err := sn.ac.SetHeaders(req, true); err != nil {
+		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
+	}
+
+	resp, err := sn.c.Do(req)
+	if err != nil {
+		// Disable sn for data writing for 10 seconds.
+		sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
+
+		return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode/100 == 2 {
+		return nil
+	}
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		respBody = []byte(fmt.Sprintf("%s", err))
+	}
+
+	// Disable sn for data writing for 10 seconds.
+	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
+
+	return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
+}
+
+func (sn *storageNode) getRequestURL(path string) string {
+	return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
+}
+
+var zstdBufPool bytesutil.ByteBufferPool
+
+// NewStorage returns new Storage for the given addrs with the given authCfgs.
+//
+// The concurrency is the average number of concurrent connections per every addr.
+//
+// If disableCompression is set, then the data is sent uncompressed to the remote storage.
+//
+// Call MustStop on the returned storage when it is no longer needed.
+func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
+	pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
+	for i := 0; i < cap(pendingDataBuffers); i++ {
+		pendingDataBuffers <- &bytesutil.ByteBuffer{}
+	}
+
+	s := &Storage{
+		disableCompression: disableCompression,
+		pendingDataBuffers: pendingDataBuffers,
+		stopCh:             make(chan struct{}),
+	}
+
+	sns := make([]*storageNode, len(addrs))
+	for i, addr := range addrs {
+		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
+	}
+	s.sns = sns
+
+	s.srt = newStreamRowsTracker(len(sns))
+
+	return s
+}
+
+// MustStop stops the s.
+func (s *Storage) MustStop() {
+	close(s.stopCh)
+	s.wg.Wait()
+	s.sns = nil
+}
+
+// AddRow adds the given log row into s.
+func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
+	idx := s.srt.getNodeIdx(streamHash)
+	sn := s.sns[idx]
+	sn.addRow(r)
+}
+
+func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
+	startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
+	for i := range s.sns {
+		idx := (startIdx + i) % len(s.sns)
+		sn := s.sns[idx]
+		err := sn.sendInsertRequest(pendingData)
+		if err == nil {
+			return true
+		}
+		if !errors.Is(err, errTemporarilyDisabled) {
+			logger.Warnf("cannot send pending data to the storage node %q: %s; trying to send it to another storage node", sn.addr, err)
+		}
+	}
+	return false
+}
+
+var errTemporarilyDisabled = fmt.Errorf("writing to the node is temporarily disabled")
+
+type streamRowsTracker struct {
+	mu sync.Mutex
+
+	nodesCount    int64
+	rowsPerStream map[uint64]uint64
+}
+
+func newStreamRowsTracker(nodesCount int) *streamRowsTracker {
+	return &streamRowsTracker{
+		nodesCount:    int64(nodesCount),
+		rowsPerStream: make(map[uint64]uint64),
+	}
+}
+
+func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
+	if srt.nodesCount == 1 {
+		// Fast path for a single node.
+		return 0
+	}
+
+	srt.mu.Lock()
+	defer srt.mu.Unlock()
+
+	streamRows := srt.rowsPerStream[streamHash] + 1
+	srt.rowsPerStream[streamHash] = streamRows
+
+	if streamRows <= 1000 {
+		// Write the initial rows for the stream to a single storage node for better locality.
+		// This should work great for log streams containing small number of logs, since will be distributed
+		// evenly among available storage nodes because they have different streamHash.
+		return streamHash % uint64(srt.nodesCount)
+	}
+
+	// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
+	// in order to improve query performance over this stream (the data for the log stream
+	// can be processed in parallel on all the storage nodes).
+	//
+	// The random distribution is preferred over round-robin distribution in order to avoid possible
+	// dependency between the order of the ingested logs and the number of storage nodes,
+	// which may lead to non-uniform distribution of logs among storage nodes.
+	return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
+}

app/vlstorage/netinsert/netinsert_test.go

@@ -0,0 +1,57 @@
+package netinsert
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"testing"
+
+	"github.com/cespare/xxhash/v2"
+)
+
+func TestStreamRowsTracker(t *testing.T) {
+	f := func(rowsCount, streamsCount, nodesCount int) {
+		t.Helper()
+
+		// generate stream hashes
+		streamHashes := make([]uint64, streamsCount)
+		for i := range streamHashes {
+			streamHashes[i] = xxhash.Sum64([]byte(fmt.Sprintf("stream %d.", i)))
+		}
+
+		srt := newStreamRowsTracker(nodesCount)
+
+		rng := rand.New(rand.NewSource(0))
+		rowsPerNode := make([]uint64, nodesCount)
+		for i := 0; i < rowsCount; i++ {
+			streamIdx := rng.Intn(streamsCount)
+			h := streamHashes[streamIdx]
+			nodeIdx := srt.getNodeIdx(h)
+			rowsPerNode[nodeIdx]++
+		}
+
+		// Verify that rows are uniformly distributed among nodes.
+		expectedRowsPerNode := float64(rowsCount) / float64(nodesCount)
+		for nodeIdx, nodeRows := range rowsPerNode {
+			if math.Abs(float64(nodeRows)-expectedRowsPerNode)/expectedRowsPerNode > 0.15 {
+				t.Fatalf("non-uniform distribution of rows among nodes; node %d has %d rows, while it must have %v rows; rowsPerNode=%d",
+					nodeIdx, nodeRows, expectedRowsPerNode, rowsPerNode)
+			}
+		}
+	}
+
+	rowsCount := 10000
+	streamsCount := 9
+	nodesCount := 2
+	f(rowsCount, streamsCount, nodesCount)
+
+	rowsCount = 10000
+	streamsCount = 100
+	nodesCount = 2
+	f(rowsCount, streamsCount, nodesCount)
+
+	rowsCount = 100000
+	streamsCount = 1000
+	nodesCount = 9
+	f(rowsCount, streamsCount, nodesCount)
+}

app/vlstorage/netselect/netselect.go

@@ -0,0 +1,469 @@
+package netselect
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+)
+
+const (
+	// FieldNamesProtocolVersion is the version of the protocol used for /internal/select/field_names HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	FieldNamesProtocolVersion = "v1"
+
+	// FieldValuesProtocolVersion is the version of the protocol used for /internal/select/field_values HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	FieldValuesProtocolVersion = "v1"
+
+	// StreamFieldNamesProtocolVersion is the version of the protocol used for /internal/select/stream_field_names HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamFieldNamesProtocolVersion = "v1"
+
+	// StreamFieldValuesProtocolVersion is the version of the protocol used for /internal/select/stream_field_values HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamFieldValuesProtocolVersion = "v1"
+
+	// StreamsProtocolVersion is the version of the protocol used for /internal/select/streams HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamsProtocolVersion = "v1"
+
+	// StreamIDsProtocolVersion is the version of the protocol used for /internal/select/stream_ids HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	StreamIDsProtocolVersion = "v1"
+
+	// QueryProtocolVersion is the version of the protocol used for /internal/select/query HTTP endpoint.
+	//
+	// It must be updated every time the protocol changes.
+	QueryProtocolVersion = "v1"
+)
+
+// Storage is a network storage for querying remote storage nodes in the cluster.
+type Storage struct {
+	sns []*storageNode
+
+	disableCompression bool
+}
+
+type storageNode struct {
+	// scheme is http or https scheme to communicate with addr
+	scheme string
+
+	// addr is TCP address of the storage node to query
+	addr string
+
+	// s is a storage, which holds the given storageNode
+	s *Storage
+
+	// c is an http client used for querying storage node at addr.
+	c *http.Client
+
+	// ac is auth config used for setting request headers such as Authorization and Host.
+	ac *promauth.Config
+}
+
+func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
+	tr := httputil.NewTransport(false, "vlselect_backend")
+	tr.TLSHandshakeTimeout = 20 * time.Second
+	tr.DisableCompression = true
+
+	scheme := "http"
+	if isTLS {
+		scheme = "https"
+	}
+
+	sn := &storageNode{
+		scheme: scheme,
+		addr:   addr,
+		s:      s,
+		c: &http.Client{
+			Transport: ac.NewRoundTripper(tr),
+		},
+		ac: ac,
+	}
+	return sn
+}
+
+func (sn *storageNode) runQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, processBlock func(db *logstorage.DataBlock)) error {
+	args := sn.getCommonArgs(QueryProtocolVersion, tenantIDs, q)
+
+	reqURL := sn.getRequestURL("/internal/select/query", args)
+	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
+	if err != nil {
+		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
+	}
+	if err := sn.ac.SetHeaders(req, true); err != nil {
+		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
+	}
+
+	// send the request to the storage node
+	resp, err := sn.c.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		responseBody, err := io.ReadAll(resp.Body)
+		if err != nil {
+			responseBody = []byte(err.Error())
+		}
+		return fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
+	}
+
+	// read the response
+	var dataLenBuf [8]byte
+	var buf []byte
+	var db logstorage.DataBlock
+	var valuesBuf []string
+	for {
+		if _, err := io.ReadFull(resp.Body, dataLenBuf[:]); err != nil {
+			if errors.Is(err, io.EOF) {
+				// The end of response stream
+				return nil
+			}
+			return fmt.Errorf("cannot read block size from %q: %w", reqURL, err)
+		}
+		blockLen := encoding.UnmarshalUint64(dataLenBuf[:])
+		if blockLen > math.MaxInt {
+			return fmt.Errorf("too big data block: %d bytes; mustn't exceed %v bytes", blockLen, math.MaxInt)
+		}
+
+		buf = slicesutil.SetLength(buf, int(blockLen))
+		if _, err := io.ReadFull(resp.Body, buf); err != nil {
+			return fmt.Errorf("cannot read block with size of %d bytes from %q: %w", blockLen, reqURL, err)
+		}
+
+		src := buf
+		if !sn.s.disableCompression {
+			bufLen := len(buf)
+			var err error
+			buf, err = zstd.Decompress(buf, buf)
+			if err != nil {
+				return fmt.Errorf("cannot decompress data block: %w", err)
+			}
+			src = buf[bufLen:]
+		}
+
+		for len(src) > 0 {
+			tail, vb, err := db.UnmarshalInplace(src, valuesBuf[:0])
+			if err != nil {
+				return fmt.Errorf("cannot unmarshal data block received from %q: %w", reqURL, err)
+			}
+			valuesBuf = vb
+			src = tail
+
+			processBlock(&db)
+
+			clear(valuesBuf)
+		}
+	}
+}
+
+func (sn *storageNode) getFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(FieldNamesProtocolVersion, tenantIDs, q)
+
+	return sn.getValuesWithHits(ctx, "/internal/select/field_names", args)
+}
+
+func (sn *storageNode) getFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(FieldValuesProtocolVersion, tenantIDs, q)
+	args.Set("field", fieldName)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/field_values", args)
+}
+
+func (sn *storageNode) getStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamFieldNamesProtocolVersion, tenantIDs, q)
+
+	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_names", args)
+}
+
+func (sn *storageNode) getStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamFieldValuesProtocolVersion, tenantIDs, q)
+	args.Set("field", fieldName)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_values", args)
+}
+
+func (sn *storageNode) getStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamsProtocolVersion, tenantIDs, q)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/streams", args)
+}
+
+func (sn *storageNode) getStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	args := sn.getCommonArgs(StreamIDsProtocolVersion, tenantIDs, q)
+	args.Set("limit", fmt.Sprintf("%d", limit))
+
+	return sn.getValuesWithHits(ctx, "/internal/select/stream_ids", args)
+}
+
+func (sn *storageNode) getCommonArgs(version string, tenantIDs []logstorage.TenantID, q *logstorage.Query) url.Values {
+	args := url.Values{}
+	args.Set("version", version)
+	args.Set("tenant_ids", string(logstorage.MarshalTenantIDs(nil, tenantIDs)))
+	args.Set("query", q.String())
+	args.Set("timestamp", fmt.Sprintf("%d", q.GetTimestamp()))
+	args.Set("disable_compression", fmt.Sprintf("%v", sn.s.disableCompression))
+	return args
+}
+
+func (sn *storageNode) getValuesWithHits(ctx context.Context, path string, args url.Values) ([]logstorage.ValueWithHits, error) {
+	data, err := sn.executeRequestAt(ctx, path, args)
+	if err != nil {
+		return nil, err
+	}
+	return unmarshalValuesWithHits(data)
+}
+
+func (sn *storageNode) executeRequestAt(ctx context.Context, path string, args url.Values) ([]byte, error) {
+	reqURL := sn.getRequestURL(path, args)
+	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
+	if err != nil {
+		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
+	}
+
+	// send the request to the storage node
+	resp, err := sn.c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		responseBody, err := io.ReadAll(resp.Body)
+		if err != nil {
+			responseBody = []byte(err.Error())
+		}
+		return nil, fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
+	}
+
+	// read the response
+	var bb bytesutil.ByteBuffer
+	if _, err := bb.ReadFrom(resp.Body); err != nil {
+		return nil, fmt.Errorf("cannot read response from %q: %w", reqURL, err)
+	}
+
+	if sn.s.disableCompression {
+		return bb.B, nil
+	}
+
+	bbLen := len(bb.B)
+	bb.B, err = zstd.Decompress(bb.B, bb.B)
+	if err != nil {
+		return nil, err
+	}
+	return bb.B[bbLen:], nil
+}
+
+func (sn *storageNode) getRequestURL(path string, args url.Values) string {
+	return fmt.Sprintf("%s://%s%s?%s", sn.scheme, sn.addr, path, args.Encode())
+}
+
+// NewStorage returns new Storage for the given addrs and the given authCfgs.
+//
+// If disableCompression is set, then uncompressed responses are received from storage nodes.
+//
+// Call MustStop on the returned storage when it is no longer needed.
+func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, disableCompression bool) *Storage {
+	s := &Storage{
+		disableCompression: disableCompression,
+	}
+
+	sns := make([]*storageNode, len(addrs))
+	for i, addr := range addrs {
+		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
+	}
+	s.sns = sns
+
+	return s
+}
+
+// MustStop stops the s.
+func (s *Storage) MustStop() {
+	s.sns = nil
+}
+
+// RunQuery runs the given q and calls writeBlock for the returned data blocks
+func (s *Storage) RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+	nqr, err := logstorage.NewNetQueryRunner(ctx, tenantIDs, q, s.RunQuery, writeBlock)
+	if err != nil {
+		return err
+	}
+
+	search := func(stopCh <-chan struct{}, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+		return s.runQuery(stopCh, tenantIDs, q, writeBlock)
+	}
+
+	concurrency := q.GetConcurrency()
+	return nqr.Run(ctx, concurrency, search)
+}
+
+func (s *Storage) runQuery(stopCh <-chan struct{}, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
+	ctxWithCancel, cancel := contextutil.NewStopChanContext(stopCh)
+	defer cancel()
+
+	errs := make([]error, len(s.sns))
+
+	var wg sync.WaitGroup
+	for i := range s.sns {
+		wg.Add(1)
+		go func(nodeIdx int) {
+			defer wg.Done()
+			sn := s.sns[nodeIdx]
+			err := sn.runQuery(ctxWithCancel, tenantIDs, q, func(db *logstorage.DataBlock) {
+				writeBlock(uint(nodeIdx), db)
+			})
+			if err != nil {
+				// Cancel the remaining parallel queries
+				cancel()
+			}
+
+			errs[nodeIdx] = err
+		}(i)
+	}
+	wg.Wait()
+
+	return getFirstNonCancelError(errs)
+}
+
+// GetFieldNames executes q and returns field names seen in results.
+func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getFieldNames(ctx, tenantIDs, q)
+	})
+}
+
+// GetFieldValues executes q and returns unique values for the fieldName seen in results.
+//
+// If limit > 0, then up to limit unique values are returned.
+func (s *Storage) GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	})
+}
+
+// GetStreamFieldNames executes q and returns stream field names seen in results.
+func (s *Storage) GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreamFieldNames(ctx, tenantIDs, q)
+	})
+}
+
+// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
+//
+// If limit > 0, then up to limit unique stream field values are returned.
+func (s *Storage) GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	})
+}
+
+// GetStreams executes q and returns streams seen in query results.
+//
+// If limit > 0, then up to limit unique streams are returned.
+func (s *Storage) GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreams(ctx, tenantIDs, q, limit)
+	})
+}
+
+// GetStreamIDs executes q and returns streamIDs seen in query results.
+//
+// If limit > 0, then up to limit unique streamIDs are returned.
+func (s *Storage) GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
+	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
+		return sn.getStreamIDs(ctx, tenantIDs, q, limit)
+	})
+}
+
+func (s *Storage) getValuesWithHits(ctx context.Context, limit uint64, resetHitsOnLimitExceeded bool,
+	callback func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error)) ([]logstorage.ValueWithHits, error) {
+
+	ctxWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	results := make([][]logstorage.ValueWithHits, len(s.sns))
+	errs := make([]error, len(s.sns))
+
+	var wg sync.WaitGroup
+	for i := range s.sns {
+		wg.Add(1)
+		go func(nodeIdx int) {
+			defer wg.Done()
+
+			sn := s.sns[nodeIdx]
+			vhs, err := callback(ctxWithCancel, sn)
+			results[nodeIdx] = vhs
+			errs[nodeIdx] = err
+
+			if err != nil {
+				// Cancel the remaining parallel requests
+				cancel()
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	if err := getFirstNonCancelError(errs); err != nil {
+		return nil, err
+	}
+
+	vhs := logstorage.MergeValuesWithHits(results, limit, resetHitsOnLimitExceeded)
+
+	return vhs, nil
+}
+
+func getFirstNonCancelError(errs []error) error {
+	for _, err := range errs {
+		if err != nil && !errors.Is(err, context.Canceled) {
+			return err
+		}
+	}
+	return nil
+}
+
+func unmarshalValuesWithHits(src []byte) ([]logstorage.ValueWithHits, error) {
+	var vhs []logstorage.ValueWithHits
+	for len(src) > 0 {
+		var vh logstorage.ValueWithHits
+		tail, err := vh.UnmarshalInplace(src)
+		if err != nil {
+			return nil, fmt.Errorf("cannot unmarshal ValueWithHits #%d: %w", len(vhs), err)
+		}
+		src = tail
+
+		// Clone vh.Value, since it points to src.
+		vh.Value = strings.Clone(vh.Value)
+
+		vhs = append(vhs, vh)
+	}
+
+	return vhs, nil
+}

app/vmagent/remotewrite/client.go

@@ -10,20 +10,22 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
 )
 
 var (
@@ -88,7 +90,8 @@ type client struct {
 	remoteWriteURL string
 
 	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
-	useVMProto bool
+	useVMProto          atomic.Bool
+	canDowngradeVMProto atomic.Bool
 
 	fq *persistentqueue.FastQueue
 	hc *http.Client
@@ -167,17 +170,11 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
 	}
 	if !useVMProto && !usePromProto {
-		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
-		doRequest := func(url string) (*http.Response, error) {
-			return c.doRequest(url, nil)
-		}
-		useVMProto = protoparserutil.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
-		if !useVMProto {
-			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
-				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", sanitizedURL)
-		}
+		// The VM protocol could be downgraded later at runtime if unsupported media type response status is received.
+		useVMProto = true
+		c.canDowngradeVMProto.Store(true)
 	}
-	c.useVMProto = useVMProto
+	c.useVMProto.Store(useVMProto)
 
 	return c
 }
@@ -434,6 +431,7 @@ again:
 		c.retriesCount.Inc()
 		goto again
 	}
+
 	statusCode := resp.StatusCode
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
@@ -442,24 +440,46 @@ again:
 		c.blocksSent.Inc()
 		return true
 	}
+
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
-	if statusCode == 409 || statusCode == 400 {
-		body, err := io.ReadAll(resp.Body)
-		_ = resp.Body.Close()
-		if err != nil {
-			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
-				"failed to read response body: %s",
-				len(block), c.sanitizedURL, statusCode, err)
-		} else {
-			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
-				len(block), c.sanitizedURL, statusCode, string(body))
-		}
-		// Just drop block on 409 and 400 status codes like Prometheus does.
+	if statusCode == 409 {
+		logBlockRejected(block, c.sanitizedURL, resp)
+
+		// Just drop block on 409 status code like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
 		_ = resp.Body.Close()
 		c.packetsDropped.Inc()
 		return true
+		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
+		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
+		// - Real-world implementations of v1 use both 400 and 415 status codes.
+		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
+	} else if statusCode == 415 || statusCode == 400 {
+		if c.canDowngradeVMProto.Swap(false) {
+			logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
+				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
+			c.useVMProto.Store(false)
+		}
+
+		if encoding.IsZstd(block) {
+			logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
+				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
+
+			block = mustRepackBlockFromZstdToSnappy(block)
+
+			c.retriesCount.Inc()
+			_ = resp.Body.Close()
+			goto again
+		}
+
+		// Just drop snappy blocks on 400 or 415 status codes like Prometheus does.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
+		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
+		logBlockRejected(block, c.sanitizedURL, resp)
+		_ = resp.Body.Close()
+		c.packetsDropped.Inc()
+		return true
 	}
 
 	// Unexpected status code returned
@@ -511,6 +531,28 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 	return retryDuration
 }
 
+func mustRepackBlockFromZstdToSnappy(zstdBlock []byte) []byte {
+	plainBlock := make([]byte, 0, len(zstdBlock)*2)
+	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
+	if err != nil {
+		logger.Panicf("FATAL: cannot re-pack block with size %d bytes from Zstd to Snappy: %s", len(zstdBlock), err)
+	}
+
+	return snappy.Encode(nil, plainBlock)
+}
+
+func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+			"failed to read response body: %s",
+			len(block), sanitizedURL, resp.StatusCode, err)
+	} else {
+		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+			len(block), sanitizedURL, resp.StatusCode, string(body))
+	}
+}
+
 // parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
 // retryAfterString should be in either HTTP-date or a number of seconds.
 // It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.

app/vmagent/remotewrite/client_test.go

@@ -5,6 +5,9 @@ import (
 	"net/http"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/golang/snappy"
 )
 
 func TestCalculateRetryDuration(t *testing.T) {
@@ -97,3 +100,19 @@ func helper(d time.Duration) time.Duration {
 
 	return d + dv
 }
+
+func TestRepackBlockFromZstdToSnappy(t *testing.T) {
+	expectedPlainBlock := []byte(`foobar`)
+
+	zstdBlock := encoding.CompressZSTDLevel(nil, expectedPlainBlock, 1)
+	snappyBlock := mustRepackBlockFromZstdToSnappy(zstdBlock)
+
+	actualPlainBlock, err := snappy.Decode(nil, snappyBlock)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if string(actualPlainBlock) != string(expectedPlainBlock) {
+		t.Fatalf("unexpected plain block; got %q; want %q", actualPlainBlock, expectedPlainBlock)
+	}
+}

app/vmagent/remotewrite/pendingseries.go

@@ -40,7 +40,7 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
 	ps.wr.fq = fq
 	ps.wr.isVMRemoteWrite = isVMRemoteWrite
@@ -100,7 +100,7 @@ type writeRequest struct {
 	fq *persistentqueue.FastQueue
 
 	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	isVMRemoteWrite bool
+	isVMRemoteWrite *atomic.Bool
 
 	// How many significant figures must be left before sending the writeRequest to fq.
 	significantFigures int
@@ -138,7 +138,7 @@ func (wr *writeRequest) reset() {
 // This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
 func (wr *writeRequest) mustFlushOnStop() {
 	wr.wr.Timeseries = wr.tss
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
+	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
 		logger.Panicf("BUG: final flush must always return true")
 	}
 	wr.reset()
@@ -152,7 +152,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
 	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
+	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
 		return false
 	}
 	wr.reset()

app/vmagent/remotewrite/remotewrite.go

@@ -807,7 +807,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
+		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{

app/vmagent/remotewrite/remotewrite_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -68,7 +69,9 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		allRelabelConfigs.Store(rcs)
 
 		pss := make([]*pendingSeries, 1)
-		pss[0] = newPendingSeries(nil, true, 0, 100)
+		isVMProto := &atomic.Bool{}
+		isVMProto.Store(true)
+		pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
 		rwctx := &remoteWriteCtx{
 			idx:                    0,
 			streamAggrKeepInput:    keepInput,

app/vmselect/netstorage/netstorage.go

@@ -1379,6 +1379,5 @@ func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern str
 func ResetMetricNamesStats(qt *querytracer.Tracer) error {
 	qt = qt.NewChild("reset metric names usage stats")
 	defer qt.Done()
-	vmstorage.ResetMetricNamesStats(qt)
-	return nil
+	return vmstorage.ResetMetricNamesStats(qt)
 }

app/vmselect/prometheus/prometheus.go

@@ -806,7 +806,6 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	} else {
 		queryOffset = 0
 	}
-	qs := &promql.QueryStats{}
 	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 start,
@@ -822,9 +821,10 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
-
-		QueryStats: qs,
 	}
+	qs := promql.NewQueryStats(query, nil, ec)
+	ec.QueryStats = qs
+
 	result, err := promql.Exec(qt, ec, query, true)
 	if err != nil {
 		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %w", query, start, step, err)
@@ -853,6 +853,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot flush query response to remote client: %w", err)
 	}
+
 	return nil
 }
 
@@ -914,7 +915,6 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		start, end = promql.AdjustStartEnd(start, end, step)
 	}
 
-	qs := &promql.QueryStats{}
 	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 end,
@@ -930,9 +930,10 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
-
-		QueryStats: qs,
 	}
+	qs := promql.NewQueryStats(query, nil, ec)
+	ec.QueryStats = qs
+
 	result, err := promql.Exec(qt, ec, query, false)
 	if err != nil {
 		return err
@@ -961,6 +962,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot send query range response to remote client: %w", err)
 	}
+
 	return nil
 }
 

app/vmselect/prometheus/query_range_response.qtpl

@@ -34,7 +34,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 			// It cannot be converted to int without breaking backwards compatibility at vmalert :(
 		%}
 		"seriesFetched": "{%dl qs.SeriesFetched.Load() %}",
-		"executionTimeMsec": {%dl qs.ExecutionTimeMsec.Load() %}
+		"executionTimeMsec": {%dl qs.ExecutionDuration.Load().Milliseconds() %}
 	}
 	{% code
 		qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)

app/vmselect/prometheus/query_range_response.qtpl.go

@@ -71,9 +71,9 @@ func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result,
 	qw422016.N().DL(qs.SeriesFetched.Load())
 //line app/vmselect/prometheus/query_range_response.qtpl:36
 	qw422016.N().S(`","executionTimeMsec":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:37
-	qw422016.N().DL(qs.ExecutionTimeMsec.Load())
-//line app/vmselect/prometheus/query_range_response.qtpl:37
+//line app/vmselect/prometheus/query_range_response.qtpl:38
+	qw422016.N().DL(qs.ExecutionDuration.Load().Milliseconds())
+//line app/vmselect/prometheus/query_range_response.qtpl:38
 	qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_range_response.qtpl:40
 	qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)

app/vmselect/prometheus/query_response.qtpl

@@ -36,7 +36,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 			// It cannot be converted to int without breaking backwards compatibility at vmalert :(
 		%}
 		"seriesFetched": "{%dl qs.SeriesFetched.Load() %}",
-		"executionTimeMsec": {%dl qs.ExecutionTimeMsec.Load() %}
+		"executionTimeMsec": {%dl qs.ExecutionDuration.Load().Milliseconds() %}
 	}
 	{% code
 		qt.Printf("generate /api/v1/query response for series=%d", seriesCount)

app/vmselect/prometheus/query_response.qtpl.go

@@ -81,9 +81,9 @@ func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *
 	qw422016.N().DL(qs.SeriesFetched.Load())
 //line app/vmselect/prometheus/query_response.qtpl:38
 	qw422016.N().S(`","executionTimeMsec":`)
-//line app/vmselect/prometheus/query_response.qtpl:39
-	qw422016.N().DL(qs.ExecutionTimeMsec.Load())
-//line app/vmselect/prometheus/query_response.qtpl:39
+//line app/vmselect/prometheus/query_response.qtpl:40
+	qw422016.N().DL(qs.ExecutionDuration.Load().Milliseconds())
+//line app/vmselect/prometheus/query_response.qtpl:40
 	qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_response.qtpl:42
 	qt.Printf("generate /api/v1/query response for series=%d", seriesCount)

app/vmselect/prometheus/tsdb_status_response.qtpl

@@ -11,7 +11,7 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb .
 	"data":{
 		"totalSeries": {%dul= status.TotalSeries %},
 		"totalLabelValuePairs": {%dul= status.TotalLabelValuePairs %},
-		"seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %},
+		"seriesCountByMetricName":{%= tsdbStatusMetricNameEntries(status.SeriesCountByMetricName,status.SeriesQueryStatsByMetricName) %},
 		"seriesCountByLabelName":{%= tsdbStatusEntries(status.SeriesCountByLabelName) %},
 		"seriesCountByFocusLabelValue":{%= tsdbStatusEntries(status.SeriesCountByFocusLabelValue) %},
 		"seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %},
@@ -34,4 +34,32 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb .
 ]
 {% endfunc %}
 
+{% func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) %}
+{% code
+  queryStatsByMetricName := make(map[string]storage.MetricNamesStatsRecord,len(queryStats))
+  for _, record := range queryStats{
+      queryStatsByMetricName[record.MetricName] = record
+    }
+%}
+[
+	{% for i, e := range a %}
+		{
+      {% code
+      entry, ok := queryStatsByMetricName[e.Name]
+      %}
+			"name":{%q= e.Name %},
+      {% if !ok %}
+			"value":{%d= int(e.Count) %}
+      {% else %}
+			"value":{%d= int(e.Count) %},
+			"requestsCount":{%d= int(entry.RequestsCount) %},
+			"lastRequestTimestamp":{%d= int(entry.LastRequestTs) %}
+      {% endif %}
+		}
+		{% if i+1 < len(a) %},{% endif %}
+	{% endfor %}
+]
+{% endfunc %}
+
+
 {% endstripspace %}

app/vmselect/prometheus/tsdb_status_response.qtpl.go

@@ -37,9 +37,9 @@ func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBSta
 	qw422016.N().DUL(status.TotalLabelValuePairs)
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:13
 	qw422016.N().S(`,"seriesCountByMetricName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
-	streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+	streamtsdbStatusMetricNameEntries(qw422016, status.SeriesCountByMetricName, status.SeriesQueryStatsByMetricName)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
 	qw422016.N().S(`,"seriesCountByLabelName":`)
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:15
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelName)
@@ -147,3 +147,89 @@ func tsdbStatusEntries(a []storage.TopHeapEntry) string {
 	return qs422016
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 }
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:38
+func streamtsdbStatusMetricNameEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:40
+	queryStatsByMetricName := make(map[string]storage.MetricNamesStatsRecord, len(queryStats))
+	for _, record := range queryStats {
+		queryStatsByMetricName[record.MetricName] = record
+	}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:44
+	qw422016.N().S(`[`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:46
+	for i, e := range a {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:46
+		qw422016.N().S(`{`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:49
+		entry, ok := queryStatsByMetricName[e.Name]
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:50
+		qw422016.N().S(`"name":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
+		qw422016.N().Q(e.Name)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
+		qw422016.N().S(`,`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:52
+		if !ok {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:52
+			qw422016.N().S(`"value":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:53
+			qw422016.N().D(int(e.Count))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:54
+		} else {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:54
+			qw422016.N().S(`"value":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
+			qw422016.N().D(int(e.Count))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
+			qw422016.N().S(`,"requestsCount":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:56
+			qw422016.N().D(int(entry.RequestsCount))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:56
+			qw422016.N().S(`,"lastRequestTimestamp":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:57
+			qw422016.N().D(int(entry.LastRequestTs))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:58
+		}
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:58
+		qw422016.N().S(`}`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
+		if i+1 < len(a) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
+		}
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:61
+	}
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:61
+	qw422016.N().S(`]`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+func writetsdbStatusMetricNameEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	streamtsdbStatusMetricNameEntries(qw422016, a, queryStats)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+}
+
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) string {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	writetsdbStatusMetricNameEntries(qb422016, a, queryStats)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+	return qs422016
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+}

app/vmselect/promql/eval.go

@@ -172,30 +172,6 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	return &ec
 }
 
-// QueryStats contains various stats for the query.
-type QueryStats struct {
-	// SeriesFetched contains the number of series fetched from storage during the query evaluation.
-	SeriesFetched atomic.Int64
-
-	// ExecutionTimeMsec contains the number of milliseconds the query took to execute.
-	ExecutionTimeMsec atomic.Int64
-}
-
-func (qs *QueryStats) addSeriesFetched(n int) {
-	if qs == nil {
-		return
-	}
-	qs.SeriesFetched.Add(int64(n))
-}
-
-func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
-	if qs == nil {
-		return
-	}
-	d := time.Since(startTime).Milliseconds()
-	qs.ExecutionTimeMsec.Add(d)
-}
-
 func (ec *EvalConfig) validate() {
 	if ec.Start > ec.End {
 		logger.Panicf("BUG: start cannot exceed end; got %d vs %d", ec.Start, ec.End)
@@ -1721,12 +1697,13 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	if err != nil {
 		return nil, err
 	}
+	qs := ec.QueryStats
 	rssLen := rss.Len()
 	if rssLen == 0 {
 		rss.Cancel()
 		return nil, nil
 	}
-	ec.QueryStats.addSeriesFetched(rssLen)
+	qs.addSeriesFetched(rssLen)
 
 	// Verify timeseries fit available memory during rollup calculations.
 	timeseriesLen := rssLen

app/vmselect/promql/query_stats.go

@@ -0,0 +1,55 @@
+package promql
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+)
+
+// QueryStats contains various stats of the query evaluation.
+type QueryStats struct {
+	// ExecutionDuration contains the time duration the query took to execute.
+	ExecutionDuration atomic.Pointer[time.Duration]
+	// SeriesFetched contains the number of series fetched from storage or cache.
+	SeriesFetched atomic.Int64
+
+	at *auth.Token
+
+	query     string
+	queryType string
+	start     int64
+	end       int64
+	step      int64
+}
+
+// NewQueryStats creates a new QueryStats object.
+func NewQueryStats(query string, at *auth.Token, ec *EvalConfig) *QueryStats {
+	qs := &QueryStats{
+		at:        at,
+		query:     query,
+		step:      ec.Step,
+		start:     ec.Start,
+		end:       ec.End,
+		queryType: "range",
+	}
+	if qs.start == qs.end {
+		qs.queryType = "instant"
+	}
+	return qs
+}
+
+func (qs *QueryStats) addSeriesFetched(n int) {
+	if qs == nil {
+		return
+	}
+	qs.SeriesFetched.Add(int64(n))
+}
+
+func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
+	if qs == nil {
+		return
+	}
+	d := time.Since(startTime)
+	qs.ExecutionDuration.Store(&d)
+}

app/vmselect/vmui/index.html

@@ -36,10 +36,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-Clv2OTUl.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-PQqNLyna.js">
+  <script type="module" crossorigin src="./assets/index-BF2w5kzJ.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-BSp13qCn.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-u4IOGr0E.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-sXHL6qTd.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -201,6 +201,9 @@ func DeleteSeries(qt *querytracer.Tracer, tfss []*storage.TagFilters, maxMetrics
 
 // GetMetricNamesStats returns metric names usage stats with give limit and lte predicate
 func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (storage.MetricNamesStatsResponse, error) {
+	if !*trackMetricNamesStats {
+		return storage.MetricNamesStatsResponse{}, fmt.Errorf("metrics usage feature must be enabled specifically using the `-storage.trackMetricNamesStats` flag")
+	}
 	WG.Add(1)
 	r := Storage.GetMetricNamesStats(qt, limit, le, matchPattern)
 	WG.Done()
@@ -208,10 +211,14 @@ func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern str
 }
 
 // ResetMetricNamesStats resets state for metric names usage tracker
-func ResetMetricNamesStats(qt *querytracer.Tracer) {
+func ResetMetricNamesStats(qt *querytracer.Tracer) error {
+	if !*trackMetricNamesStats {
+		return fmt.Errorf("metrics usage feature must be enabled specifically using the `-storage.trackMetricNamesStats` flag")
+	}
 	WG.Add(1)
 	Storage.ResetMetricNamesStats(qt)
 	WG.Done()
+	return nil
 }
 
 // SearchMetricNames returns metric names for the given tfss on the given tr.

app/vmui/packages/vmui/package.json

@@ -8,21 +8,21 @@
     "@types/lodash.debounce": "^4.0.9",
     "@types/lodash.get": "^4.4.9",
     "@types/qs": "^6.9.18",
-    "@types/react": "^19.0.12",
+    "@types/react": "^19.1.2",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
     "classnames": "^2.5.1",
     "dayjs": "^1.11.13",
     "lodash.debounce": "^4.0.8",
     "lodash.get": "^4.4.2",
-    "marked": "^15.0.7",
+    "marked": "^15.0.8",
     "marked-emoji": "^2.0.0",
-    "preact": "^10.26.4",
+    "preact": "^10.26.5",
     "qs": "^6.14.0",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.4.0",
+    "react-router-dom": "^7.5.0",
     "uplot": "^1.6.32",
-    "vite": "^6.2.3",
+    "vite": "^6.2.6",
     "web-vitals": "^4.2.4"
   },
   "scripts": {
@@ -55,25 +55,25 @@
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6",
     "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
     "@eslint/eslintrc": "^3.3.1",
-    "@eslint/js": "^9.23.0",
+    "@eslint/js": "^9.24.0",
     "@preact/preset-vite": "^2.10.1",
     "@testing-library/jest-dom": "^6.6.3",
     "@testing-library/preact": "^3.2.4",
-    "@types/node": "^22.13.13",
-    "@typescript-eslint/eslint-plugin": "^8.28.0",
-    "@typescript-eslint/parser": "^8.28.0",
+    "@types/node": "^22.14.1",
+    "@typescript-eslint/eslint-plugin": "^8.30.1",
+    "@typescript-eslint/parser": "^8.30.1",
     "cross-env": "^7.0.3",
-    "eslint": "^9.23.0",
-    "eslint-plugin-react": "^7.37.4",
+    "eslint": "^9.24.0",
+    "eslint-plugin-react": "^7.37.5",
     "globals": "^16.0.0",
-    "http-proxy-middleware": "^3.0.3",
-    "jsdom": "^26.0.0",
+    "http-proxy-middleware": "^3.0.5",
+    "jsdom": "^26.1.0",
     "postcss": "^8.5.3",
     "rollup-plugin-visualizer": "^5.14.0",
-    "sass": "^1.86.0",
-    "sass-embedded": "^1.86.0",
-    "typescript": "^5.8.2",
-    "vitest": "^3.0.9",
-    "webpack": "^5.98.0"
+    "sass": "^1.86.3",
+    "sass-embedded": "^1.86.3",
+    "typescript": "^5.8.3",
+    "vitest": "^3.1.1",
+    "webpack": "^5.99.5"
   }
 }

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -1,7 +1,6 @@
 import React, { FC, useState, useEffect, useMemo, useCallback } from "preact/compat";
 import Autocomplete from "../../Main/Autocomplete/Autocomplete";
 import { useFetchQueryOptions } from "../../../hooks/useFetchQueryOptions";
-import { escapeRegexp, hasUnclosedQuotes } from "../../../utils/regexp";
 import useGetMetricsQL from "../../../hooks/useGetMetricsQL";
 import { QueryContextType } from "../../../types";
 import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete";

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/autocompleteUtils.ts

@@ -85,7 +85,7 @@ export function getContext(
   );
   const endOfClosedQuotes =
     !hasUnclosedQuotes(valueBeforeCursor) &&
-    ["`", "'", '"'].some((char) => valueBeforeCursor.endsWith(char));
+    ["`", "'", "\""].some((char) => valueBeforeCursor.endsWith(char));
   if (
     !valueBeforeCursor ||
     endOfClosedBrackets ||

app/vmui/packages/vmui/src/components/DownloadButton/DownloadButton.tsx

@@ -0,0 +1,81 @@
+import React, { FC, useCallback, useState } from "preact/compat";
+import Tooltip from "../Main/Tooltip/Tooltip";
+import Button from "../Main/Button/Button";
+import { DownloadIcon } from "../Main/Icons";
+import Popper from "../Main/Popper/Popper";
+import { useRef } from "react";
+import "./style.scss";
+import useBoolean from "../../hooks/useBoolean";
+
+interface DownloadButtonProps {
+  title: string;
+  downloadFormatOptions?: string[];
+  onDownload: (format?: string) => void;
+}
+
+const DownloadButton: FC<DownloadButtonProps> = ({ title, downloadFormatOptions, onDownload }) => {
+  const {
+    value: isPopupOpen,
+    setTrue: onOpenPopup,
+    setFalse: onClosePopup,
+  } = useBoolean(false);
+  const downloadButtonRef = useRef<HTMLDivElement>(null);
+  const onDownloadClick = useCallback(() => {
+    if (isPopupOpen) {
+      onClosePopup();
+      return;
+    }
+
+    if (downloadFormatOptions && downloadFormatOptions.length > 0) {
+      onOpenPopup();
+    } else {
+      onDownload();
+      onClosePopup();
+    }
+  }, [onDownload, onClosePopup, isPopupOpen, onOpenPopup]);
+
+  const onDownloadFormatClick = useCallback((event: Event) => {
+    const button = event.currentTarget as HTMLButtonElement;
+    onDownload(button.textContent ?? undefined);
+  }, [onDownload]);
+
+  return (
+    <>
+      <div ref={downloadButtonRef}>
+        <Tooltip
+          title={title}
+        >
+          <Button
+            variant="text"
+            startIcon={<DownloadIcon/>}
+            onClick={onDownloadClick}
+            ariaLabel={title}
+          />
+        </Tooltip>
+      </div>
+      {downloadFormatOptions && downloadFormatOptions.length > 0 && (
+        <Popper
+          open={isPopupOpen}
+          onClose={onClosePopup}
+          buttonRef={downloadButtonRef}
+          placement={"bottom-right"}
+        >
+          {downloadFormatOptions.map((option) =>
+            <div
+              key={option}
+              className={"vm-download-button__format-option"}
+            >
+              <Button
+                variant="text"
+                onClick={onDownloadFormatClick}
+                className={"vm-download-button__format-option-button"}
+              >
+                {option}
+              </Button>
+            </div>)}
+        </Popper>)}
+    </>
+  );
+};
+
+export default DownloadButton;

app/vmui/packages/vmui/src/components/DownloadButton/style.scss

@@ -0,0 +1,15 @@
+@use "src/styles/variables" as *;
+
+.vm-download-button {
+  &__format-option {
+    padding: 4px;
+    &:first-child {
+      padding-bottom: 0;
+    }
+
+    &-button {
+      width: 100%;
+      justify-content: flex-start;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/LogsConfigurators/GroupLogsConfigurators/style.scss

@@ -4,7 +4,7 @@
   display: grid;
   gap: calc($padding-large * 2);
   padding: $padding-global 0;
-  width: 600px;
+  max-width: 600px;
 
   &-item {
     display: grid;

app/vmui/packages/vmui/src/components/QueryHistory/QueryHistory.tsx

@@ -1,22 +1,23 @@
 import React, { FC, useEffect, useMemo, useState } from "preact/compat";
-import Button from "../../../components/Main/Button/Button";
-import { ClockIcon, DeleteIcon } from "../../../components/Main/Icons";
-import Tooltip from "../../../components/Main/Tooltip/Tooltip";
-import useBoolean from "../../../hooks/useBoolean";
-import Modal from "../../../components/Main/Modal/Modal";
-import Tabs from "../../../components/Main/Tabs/Tabs";
-import useDeviceDetect from "../../../hooks/useDeviceDetect";
-import useEventListener from "../../../hooks/useEventListener";
-import { useQueryState } from "../../../state/query/QueryStateContext";
-import { getQueriesFromStorage } from "./utils";
+import Button from "../Main/Button/Button";
+import { ClockIcon, DeleteIcon } from "../Main/Icons";
+import Tooltip from "../Main/Tooltip/Tooltip";
+import useBoolean from "../../hooks/useBoolean";
+import Modal from "../Main/Modal/Modal";
+import Tabs from "../Main/Tabs/Tabs";
+import useDeviceDetect from "../../hooks/useDeviceDetect";
+import useEventListener from "../../hooks/useEventListener";
+import { useQueryState } from "../../state/query/QueryStateContext";
+import { clearQueryHistoryStorage, getQueriesFromStorage, setFavoriteQueriesToStorage } from "./utils";
 import QueryHistoryItem from "./QueryHistoryItem";
 import classNames from "classnames";
 import "./style.scss";
-import { saveToStorage } from "../../../utils/storage";
-import { arrayEquals } from "../../../utils/array";
+import { saveToStorage, StorageKeys } from "../../utils/storage";
+import { arrayEquals } from "../../utils/array";
 
 interface Props {
   handleSelectQuery: (query: string, index: number) => void
+  historyKey: Extract<StorageKeys, "LOGS_QUERY_HISTORY" | "METRICS_QUERY_HISTORY">;
 }
 
 export const HistoryTabTypes = {
@@ -31,7 +32,7 @@ export const historyTabs = [
   { label: "Favorite queries", value: HistoryTabTypes.favorite },
 ];
 
-const QueryHistory: FC<Props> = ({ handleSelectQuery }) => {
+const QueryHistory: FC<Props> = ({ handleSelectQuery, historyKey }) => {
   const { queryHistory: historyState } = useQueryState();
   const { isMobile } = useDeviceDetect();
 
@@ -42,8 +43,8 @@ const QueryHistory: FC<Props> = ({ handleSelectQuery }) => {
   } = useBoolean(false);
 
   const [activeTab, setActiveTab] = useState(historyTabs[0].value);
-  const [historyStorage, setHistoryStorage] = useState(getQueriesFromStorage("QUERY_HISTORY"));
-  const [historyFavorites, setHistoryFavorites] = useState(getQueriesFromStorage("QUERY_FAVORITES"));
+  const [historyStorage, setHistoryStorage] = useState(getQueriesFromStorage(historyKey, "QUERY_HISTORY"));
+  const [historyFavorites, setHistoryFavorites] = useState(getQueriesFromStorage(historyKey, "QUERY_FAVORITES"));
 
   const historySession = useMemo(() => {
     return historyState.map((h) => h.values.filter(q => q).reverse());
@@ -86,20 +87,20 @@ const QueryHistory: FC<Props> = ({ handleSelectQuery }) => {
   };
 
   const updateStageHistory = () => {
-    setHistoryStorage(getQueriesFromStorage("QUERY_HISTORY"));
-    setHistoryFavorites(getQueriesFromStorage("QUERY_FAVORITES"));
+    setHistoryStorage(getQueriesFromStorage(historyKey, "QUERY_HISTORY"));
+    setHistoryFavorites(getQueriesFromStorage(historyKey, "QUERY_FAVORITES"));
   };
 
   const handleClearStorage = () => {
-    saveToStorage("QUERY_HISTORY", "");
+    clearQueryHistoryStorage(historyKey, "QUERY_HISTORY");
   };
 
   useEffect(() => {
     const nextValue = historyFavorites[0] || [];
-    const prevValue = getQueriesFromStorage("QUERY_FAVORITES")[0] || [];
+    const prevValue = getQueriesFromStorage(historyKey, "QUERY_FAVORITES")[0] || [];
     const isEqual = arrayEquals(nextValue, prevValue);
     if (isEqual) return;
-    saveToStorage("QUERY_FAVORITES", JSON.stringify(historyFavorites));
+    setFavoriteQueriesToStorage(historyKey, historyFavorites);
   }, [historyFavorites]);
 
   useEventListener("storage", updateStageHistory);
@@ -174,7 +175,7 @@ const QueryHistory: FC<Props> = ({ handleSelectQuery }) => {
                     startIcon={<DeleteIcon/>}
                     onClick={handleClearStorage}
                   >
-                      clear history
+                    clear history
                   </Button>
                 </div>
               )}

app/vmui/packages/vmui/src/components/QueryHistory/QueryHistoryItem.tsx

@@ -1,8 +1,8 @@
 import React, { FC, useMemo } from "preact/compat";
-import Button from "../../../components/Main/Button/Button";
-import { CopyIcon, PlayCircleOutlineIcon, StarBorderIcon, StarIcon } from "../../../components/Main/Icons";
-import Tooltip from "../../../components/Main/Tooltip/Tooltip";
-import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
+import Button from "../Main/Button/Button";
+import { CopyIcon, PlayCircleOutlineIcon, StarBorderIcon, StarIcon } from "../Main/Icons";
+import Tooltip from "../Main/Tooltip/Tooltip";
+import useCopyToClipboard from "../../hooks/useCopyToClipboard";
 import "./style.scss";
 
 interface Props {

app/vmui/packages/vmui/src/components/QueryHistory/utils.test.ts

@@ -0,0 +1,121 @@
+import { describe, it, expect, vi, Mock, afterEach } from "vitest";
+import { getFromStorage, saveToStorage } from "../../utils/storage";
+import { getUpdatedHistory, setQueriesToStorage } from "./utils";
+import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../constants/graph";
+
+vi.mock("../../utils/storage", () => ({
+  getFromStorage: vi.fn(),
+  saveToStorage: vi.fn(),
+}));
+
+describe("utils", () => {
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+  describe("setQueriesToStorage", () => {
+    it("should not change QUERY_HISTORY ", () => {
+      const getFromStorageMock = getFromStorage as Mock;
+      const saveToStorageMock = saveToStorage as Mock;
+      getFromStorageMock.mockReturnValue(JSON.stringify({
+        "QUERY_HISTORY": [],
+      }));
+
+      setQueriesToStorage("LOGS_QUERY_HISTORY", []);
+      expect(saveToStorageMock).toHaveBeenCalledWith(
+        "LOGS_QUERY_HISTORY",
+        "{\"QUERY_HISTORY\":[[]]}"
+      );
+    });
+
+    it("should not change QUERY_HISTORY cause add the same query", () => {
+      const getFromStorageMock = getFromStorage as Mock;
+      const saveToStorageMock = saveToStorage as Mock;
+      getFromStorageMock.mockReturnValue(JSON.stringify({
+        "QUERY_HISTORY": [["first_query"]],
+      }));
+
+      setQueriesToStorage("LOGS_QUERY_HISTORY", [{ index: 0, values: ["first_query"] }]);
+      expect(saveToStorageMock).toHaveBeenCalledWith(
+        "LOGS_QUERY_HISTORY",
+        "{\"QUERY_HISTORY\":[[\"first_query\"]]}"
+      );
+    });
+
+    it("should add new query to the first position to QUERY_HISTORY", () => {
+      const getFromStorageMock = getFromStorage as Mock;
+      const saveToStorageMock = saveToStorage as Mock;
+      getFromStorageMock.mockReturnValue(JSON.stringify({
+        "QUERY_HISTORY": [["first_query"]],
+      }));
+
+      setQueriesToStorage("LOGS_QUERY_HISTORY", [{ index: 0, values: ["new_query"] }]);
+      expect(saveToStorageMock).toHaveBeenCalledWith(
+        "LOGS_QUERY_HISTORY",
+        "{\"QUERY_HISTORY\":[[\"new_query\",\"first_query\"]]}"
+      );
+    });
+
+    it("should limit the QUERY_HISTORY if add extra query", () => {
+      const getFromStorageMock = getFromStorage as Mock;
+      const saveToStorageMock = saveToStorage as Mock;
+      const maxQueries = MAX_QUERIES_HISTORY * MAX_QUERY_FIELDS;
+      const currentHistory = (new Array(maxQueries)).fill(1).map((_, i) => `${i}_query`);
+      getFromStorageMock.mockReturnValue(JSON.stringify({
+        "QUERY_HISTORY": [currentHistory],
+      }));
+
+      setQueriesToStorage("LOGS_QUERY_HISTORY", [{ index: 0, values: ["extra_query"] }]);
+
+      const calls = saveToStorageMock.mock.calls;
+      const firstCallArgs = calls[0];
+      expect(firstCallArgs[0]).toStrictEqual("LOGS_QUERY_HISTORY");
+      const savedQueries = JSON.parse(firstCallArgs[1]);
+      expect(savedQueries["QUERY_HISTORY"][0][0]).toStrictEqual("extra_query");
+      expect(savedQueries["QUERY_HISTORY"][0].length).toStrictEqual(maxQueries);
+    });
+  });
+
+  describe("getUpdatedHistory", () => {
+    it("should add new query to the end of array", () => {
+      const updatedHistory = getUpdatedHistory("new_query", {
+        index: 2,
+        values: ["first_query", "second_query"]
+      });
+      expect(updatedHistory).toStrictEqual({
+        index: 2,
+        values: [
+          "first_query",
+          "second_query",
+          "new_query",
+        ],
+      });
+    });
+
+    it("should not add new query if the last query is the same", () => {
+      const updatedHistory = getUpdatedHistory("new_query", {
+        index: 2,
+        values: ["first_query", "new_query"]
+      });
+      expect(updatedHistory).toStrictEqual({
+        index: 1,
+        values: [
+          "first_query",
+          "new_query",
+        ],
+      });
+    });
+
+    it("should remove the first query if the maximum number of query is reached", () => {
+      const maxQueries = MAX_QUERIES_HISTORY * MAX_QUERY_FIELDS;
+      const values = (new Array(maxQueries)).fill(1).map((_, i) => `${i}_query`);
+      const updatedHistory = getUpdatedHistory("new_query", {
+        index: 2,
+        values: values
+      });
+      expect(updatedHistory.index).toStrictEqual(maxQueries);
+      expect(updatedHistory.values.length).toStrictEqual(maxQueries);
+      expect(updatedHistory.values[0]).toStrictEqual("1_query");
+      expect(updatedHistory.values[updatedHistory.values.length - 1]).toStrictEqual("new_query");
+    });
+  });
+});

app/vmui/packages/vmui/src/components/QueryHistory/utils.ts

@@ -0,0 +1,89 @@
+import { getFromStorage, removeFromStorage, saveToStorage, StorageKeys } from "../../utils/storage";
+import { QueryHistoryType } from "../../state/query/reducer";
+import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../constants/graph";
+
+export type HistoryKey = Extract<StorageKeys, "LOGS_QUERY_HISTORY" | "METRICS_QUERY_HISTORY">;
+export type HistoryType = "QUERY_HISTORY" | "QUERY_FAVORITES";
+
+const getHistoryFromStorage = (key: HistoryKey) => {
+  const list = getFromStorage(key) as string;
+  const history: Record<HistoryType, string[][]> = list ? JSON.parse(list) : {};
+  return history;
+};
+
+const saveHistoryToStorage = (key: HistoryKey, historyType: HistoryType, history: string[][]) => {
+  const storageHistory = getHistoryFromStorage(key);
+  saveToStorage(key, JSON.stringify({
+    ...storageHistory,
+    [historyType]: history
+  }));
+};
+
+export const getQueriesFromStorage = (key: HistoryKey, historyType: HistoryType) => {
+  return getHistoryFromStorage(key)[historyType] || [];
+};
+
+export const setQueriesToStorage = (key: HistoryKey, history: QueryHistoryType[]) => {
+  // For localStorage, avoid splitting into query fields because when working from multiple tabs can cause confusion.
+  // For convenience, we maintain the original structure of `string[][]`
+  const lastValues = history.map(h => h.values[h.index]);
+  const storageHistory = getHistoryFromStorage(key);
+  const storageValues = storageHistory["QUERY_HISTORY"] || [];
+  if (!storageValues[0]) storageValues[0] = [];
+
+  const values = storageValues[0];
+  const TOTAL_LIMIT = MAX_QUERIES_HISTORY * MAX_QUERY_FIELDS;
+
+  lastValues.forEach((v) => {
+    const already = values.includes(v);
+    if (!already && v) values.unshift(v);
+    if (values.length > TOTAL_LIMIT) values.pop();
+  });
+
+  const newStorageHistory = {
+    ...storageHistory,
+    QUERY_HISTORY: [values]
+  };
+
+  saveToStorage(key, JSON.stringify(newStorageHistory));
+};
+
+export const setFavoriteQueriesToStorage = (key: HistoryKey, favoriteQueries: string[][]) => {
+  saveHistoryToStorage(key, "QUERY_FAVORITES", favoriteQueries);
+};
+
+export const clearQueryHistoryStorage = (key: HistoryKey, historyType: HistoryType) => {
+  const history = getHistoryFromStorage(key);
+  saveToStorage(key, JSON.stringify({
+    ...history,
+    [historyType]: [],
+  }));
+};
+
+export const getUpdatedHistory = (query: string, queryHistory?: QueryHistoryType): QueryHistoryType => {
+  const h = queryHistory || { values: [] };
+  const queryEqual = query === h.values[h.values.length - 1];
+  const newValues = !queryEqual && query ? [...h.values, query] : h.values;
+
+  // limit the history
+  if (newValues.length > MAX_QUERIES_HISTORY) newValues.shift();
+
+  return {
+    index: h.values.length - Number(queryEqual),
+    values: newValues
+  };
+};
+
+const migrateMetricsQueryHistoryToHistoryByKey = () => {
+  const migrateHistory = (type: HistoryType) => {
+    const queryList = getFromStorage(type) as string;
+    if (queryList) {
+      const queryHistory: string[][] = JSON.parse(queryList);
+      saveHistoryToStorage("METRICS_QUERY_HISTORY", type, queryHistory);
+      removeFromStorage([type]);
+    }
+  };
+  migrateHistory("QUERY_HISTORY");
+  migrateHistory("QUERY_FAVORITES");
+};
+migrateMetricsQueryHistoryToHistoryByKey();

app/vmui/packages/vmui/src/components/Table/TableSettings/style.scss

@@ -37,6 +37,10 @@
         overflow: auto;
         margin-bottom: $padding-global;
 
+        @media(max-width: 500px) {
+          width: 100vw;
+        }
+
         &__item {
           width: 100%;
           font-size: $font-size;

app/vmui/packages/vmui/src/components/Table/helpers.test.ts

@@ -0,0 +1,48 @@
+import { describe, it, expect } from "vitest";
+import { descendingComparator } from "./helpers";
+import { getNanoTimestamp } from "../../utils/time"; // используем реальную реализацию
+
+describe("descendingComparator", () => {
+  it("returns 0 for equal numbers", () => {
+    const result = descendingComparator({ value: 42 }, { value: 42 }, "value");
+    expect(result).toBe(0);
+  });
+
+  it("sorts numbers descending", () => {
+    const result = descendingComparator({ value: 100 }, { value: 50 }, "value");
+    expect(result).toBeLessThan(0);
+  });
+
+  it("sorts null below any value", () => {
+    expect(descendingComparator({ value: null }, { value: 10 }, "value")).toBe(1);
+    expect(descendingComparator({ value: 10 }, { value: null }, "value")).toBe(-1);
+    expect(descendingComparator({ value: null }, { value: null }, "value")).toBe(0);
+  });
+
+  it("sorts strings descending", () => {
+    const result = descendingComparator({ name: "zzz" }, { name: "aaa" }, "name");
+    expect(result).toBe(-1);
+  });
+
+  it("sorts numeric strings as numbers when possible", () => {
+    const result = descendingComparator({ value: "200" }, { value: "50" }, "value");
+    expect(result).toBeLessThan(0);
+  });
+
+  it("sorts date strings via getNanoTimestamp", () => {
+    const a = { timestamp: "2024-01-01T00:00:00.200Z" };
+    const b = { timestamp: "2023-01-01T00:00:00.100Z" };
+
+    const nanoA = getNanoTimestamp(a.timestamp);
+    const nanoB = getNanoTimestamp(b.timestamp);
+    expect(nanoA).toBeGreaterThan(nanoB);
+
+    const result = descendingComparator(a, b, "timestamp");
+    expect(result).toBe(-1);
+  });
+
+  it("handles booleans and undefined safely", () => {
+    expect(descendingComparator({ value: true }, { value: false }, "value")).toBe(-1);
+    expect(descendingComparator({ value: undefined }, { value: false }, "value")).toBe(1);
+  });
+});

app/vmui/packages/vmui/src/components/Table/helpers.ts

@@ -6,11 +6,38 @@ const dateColumns = ["date", "timestamp", "time"];
 export function descendingComparator<T>(a: T, b: T, orderBy: keyof T) {
   const valueA = a[orderBy];
   const valueB = b[orderBy];
-  const parsedValueA = dateColumns.includes(String(orderBy)) ? getNanoTimestamp(`${valueA}`) : valueA;
-  const parsedValueB = dateColumns.includes(String(orderBy)) ? getNanoTimestamp(`${valueB}`) : valueB;
 
-  if (parsedValueB < parsedValueA) return -1;
-  if (parsedValueB > parsedValueA) return 1;
+  // null/undefined
+  if (valueA == null && valueB == null) return 0;
+  if (valueA == null) return 1;
+  if (valueB == null) return -1;
+
+  const strA = String(valueA);
+  const strB = String(valueB);
+
+  // Dates
+  const isDate = dateColumns.includes(String(orderBy));
+  if (isDate) {
+    const timeA = getNanoTimestamp(strA);
+    const timeB = getNanoTimestamp(strB);
+
+    if (timeB < timeA) return -1;
+    if (timeB > timeA) return 1;
+    return 0;
+  }
+
+  // Numbers
+  const numA = Number(strA);
+  const numB = Number(strB);
+  const isNumeric = !isNaN(numA) && !isNaN(numB);
+
+  if (isNumeric) {
+    return numB - numA;
+  }
+
+  // Strings
+  if (strB < strA) return -1;
+  if (strB > strA) return 1;
   return 0;
 }
 

app/vmui/packages/vmui/src/components/TraceQuery/TracingsView.tsx

@@ -10,6 +10,7 @@ import Modal from "../Main/Modal/Modal";
 import JsonForm from "../../pages/TracePage/JsonForm/JsonForm";
 import classNames from "classnames";
 import useDeviceDetect from "../../hooks/useDeviceDetect";
+import { downloadJSON } from "../../utils/file";
 
 interface TraceViewProps {
   traces: Trace[];
@@ -54,17 +55,7 @@ const TracingsView: FC<TraceViewProps> = ({ traces, jsonEditor = false, onDelete
   };
 
   const handleSaveToFile = (tracingData: Trace) => () => {
-    const blob = new Blob([tracingData.originalJSON], { type: "application/json" });
-    const href = URL.createObjectURL(blob);
-
-    const link = document.createElement("a");
-    link.href = href;
-    link.download = `vmui_trace_${tracingData.queryValue}.json`;
-    document.body.appendChild(link);
-    link.click();
-
-    document.body.removeChild(link);
-    URL.revokeObjectURL(href);
+    downloadJSON(tracingData.originalJSON, `vmui_trace_${tracingData.queryValue}.json`);
   };
 
   const handleExpandAll = (tracingData: Trace) => () => {

app/vmui/packages/vmui/src/layouts/Header/style.scss

@@ -58,8 +58,8 @@
     }
 
     &_mobile {
-      max-width: 65px;
-      min-width: 65px;
+      max-width: 75px;
+      min-width: 75px;
       margin: 0 auto;
     }
   }

app/vmui/packages/vmui/src/pages/CustomPanel/DownloadReport/DownloadReport.tsx

@@ -20,6 +20,7 @@ import { parseLineToJSON } from "../../../utils/json";
 import { ExportMetricResult, ReportMetaData } from "../../../api/types";
 import { getApiEndpoint } from "../../../utils/url";
 import MarkdownEditor from "../../../components/Main/MarkdownEditor/MarkdownEditor";
+import { downloadJSON } from "../../../utils/file";
 
 export enum ReportType {
   QUERY_DATA,
@@ -100,17 +101,7 @@ const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DAT
 
   const generateFile = useCallback((data: unknown) => {
     const json = JSON.stringify(data, null, 2);
-    const blob = new Blob([json], { type: "application/json" });
-    const href = URL.createObjectURL(blob);
-
-    const link = document.createElement("a");
-    link.href = href;
-    link.download = `${filename || defaultFilename}.json`;
-    document.body.appendChild(link);
-    link.click();
-
-    document.body.removeChild(link);
-    URL.revokeObjectURL(href);
+    downloadJSON(json, `${filename || defaultFilename}.json`);
     handleClose();
   }, [filename]);
 

app/vmui/packages/vmui/src/pages/CustomPanel/QueryConfigurator/QueryConfigurator.tsx

@@ -23,9 +23,10 @@ import { arrayEquals } from "../../../utils/array";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import { QueryStats } from "../../../api/types";
 import { usePrettifyQuery } from "./hooks/usePrettifyQuery";
-import QueryHistory from "../QueryHistory/QueryHistory";
+import QueryHistory from "../../../components/QueryHistory/QueryHistory";
 import AnomalyConfig from "../../../components/ExploreAnomaly/AnomalyConfig";
 import QueryEditorAutocomplete from "../../../components/Configurators/QueryEditor/QueryEditorAutocomplete";
+import { getUpdatedHistory } from "../../../components/QueryHistory/utils";
 
 export interface QueryConfiguratorProps {
   queryErrors: string[];
@@ -79,19 +80,10 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
   const updateHistory = () => {
     queryDispatch({
       type: "SET_QUERY_HISTORY",
-      payload: stateQuery.map((q, i) => {
-        const h = queryHistory[i] || { values: [] };
-        const queryEqual = q === h.values[h.values.length - 1];
-        const newValues = !queryEqual && q ? [...h.values, q] : h.values;
-
-        // limit the history
-        if (newValues.length > MAX_QUERIES_HISTORY)  newValues.shift();
-
-        return {
-          index: h.values.length - Number(queryEqual),
-          values: newValues
-        };
-      })
+      payload: {
+        key: "METRICS_QUERY_HISTORY",
+        history: stateQuery.map((q, i) => getUpdatedHistory(q, queryHistory[i]))
+      }
     });
   };
 
@@ -275,7 +267,10 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
     <div className="vm-query-configurator-settings">
       <AdditionalSettings hideButtons={hideButtons}/>
       <div className="vm-query-configurator-settings__buttons">
-        <QueryHistory handleSelectQuery={handleSelectHistory}/>
+        <QueryHistory
+          handleSelectQuery={handleSelectHistory}
+          historyKey={"METRICS_QUERY_HISTORY"}
+        />
         {hideButtons?.anomalyConfig && <AnomalyConfig/>}
         {!hideButtons?.addQuery && stateQuery.length < MAX_QUERY_FIELDS && (
           <Button

app/vmui/packages/vmui/src/pages/CustomPanel/QueryHistory/utils.ts

@@ -1,27 +0,0 @@
-import { getFromStorage, saveToStorage, StorageKeys } from "../../../utils/storage";
-import { QueryHistoryType } from "../../../state/query/reducer";
-import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../../constants/graph";
-
-export const getQueriesFromStorage = (key: StorageKeys) => {
-  const list = getFromStorage(key) as string;
-  return list ? JSON.parse(list) as string[][] : [];
-};
-
-export const setQueriesToStorage = (history: QueryHistoryType[]) => {
-  // For localStorage, avoid splitting into query fields because when working from multiple tabs can cause confusion.
-  // For convenience, we maintain the original structure of `string[][]`
-  const lastValues = history.map(h => h.values[h.index]);
-  const storageValues = getQueriesFromStorage("QUERY_HISTORY");
-  if (!storageValues[0]) storageValues[0] = [];
-
-  const values = storageValues[0];
-  const TOTAL_LIMIT = MAX_QUERIES_HISTORY * MAX_QUERY_FIELDS;
-
-  lastValues.forEach((v) => {
-    const already = values.includes(v);
-    if (!already && v) values.unshift(v);
-    if (values.length > TOTAL_LIMIT) values.shift();
-  });
-
-  saveToStorage("QUERY_HISTORY", JSON.stringify(storageValues));
-};

app/vmui/packages/vmui/src/pages/ExploreLogs/DownloadLogsButton/DownloadLogsButton.tsx

@@ -0,0 +1,56 @@
+import React, { FC, useMemo } from "preact/compat";
+import { useCallback } from "react";
+import dayjs from "dayjs";
+import DownloadButton from "../../../components/DownloadButton/DownloadButton";
+import { DATE_FILENAME_FORMAT } from "../../../constants/date";
+import { downloadCSV, downloadJSON } from "../../../utils/file";
+import { Logs } from "../../../api/types";
+
+interface DownloadLogsButtonProps {
+  logs: Logs[];
+}
+
+const DownloadLogsButton: FC<DownloadLogsButtonProps> = ({ logs }) => {
+  const { fileExtensions, getDownloaderByExtension } = useMemo(() => {
+    const downloadFileOptions: {
+      extension: string;
+      downloader: (data: Record<string,string>[], filename: string) => void;
+    }[] = [
+      { extension: "csv", downloader: downloadCSV },
+      {
+        extension: "json",
+        downloader: (data: Record<string,string>[], filename: string) => {
+          const json = JSON.stringify(data, null, 2);
+          downloadJSON(json, filename);
+        }
+      }
+    ];
+
+    const getDownloaderByExtension = (extension: string) => {
+      return downloadFileOptions.find(({ extension: optionExtension }) => optionExtension === extension)?.downloader;
+    };
+    const fileExtensions = downloadFileOptions.map(({ extension }) => extension);
+
+    return { fileExtensions, getDownloaderByExtension };
+  }, []);
+
+  const onDownload = useCallback((fileExtension?: string) => {
+    if (!fileExtension){
+      return;
+    }
+
+    const downloader = getDownloaderByExtension(fileExtension);
+    if (downloader){
+      const timestamp = dayjs().utc().format(DATE_FILENAME_FORMAT);
+      downloader(logs, `vmui_logs_${timestamp}.${fileExtension}`);
+    }
+  }, [logs]);
+
+  return <DownloadButton
+    title={"Download logs"}
+    onDownload={onDownload}
+    downloadFormatOptions={fileExtensions}
+  />;
+};
+
+export default DownloadLogsButton;

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx

@@ -15,12 +15,16 @@ import { useFetchLogHits } from "./hooks/useFetchLogHits";
 import { LOGS_ENTRIES_LIMIT } from "../../constants/logs";
 import { getTimeperiodForDuration, relativeTimeOptions } from "../../utils/time";
 import { useSearchParams } from "react-router-dom";
+import { useQueryDispatch, useQueryState } from "../../state/query/QueryStateContext";
+import { getUpdatedHistory } from "../../components/QueryHistory/utils";
 
 const storageLimit = Number(getFromStorage("LOGS_LIMIT"));
 const defaultLimit = isNaN(storageLimit) ? LOGS_ENTRIES_LIMIT : storageLimit;
 
 const ExploreLogs: FC = () => {
   const { serverUrl } = useAppState();
+  const { queryHistory } = useQueryState();
+  const queryDispatch = useQueryDispatch();
   const { duration, relativeTime, period: periodState } = useTimeState();
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
   const [searchParams] = useSearchParams();
@@ -28,6 +32,18 @@ const ExploreLogs: FC = () => {
 
   const [limit, setLimit] = useStateSearchParams(defaultLimit, "limit");
   const [query, setQuery] = useStateSearchParams("*", "query");
+
+  const updateHistory = () => {
+    const history = getUpdatedHistory(query, queryHistory[0]);
+    queryDispatch({
+      type: "SET_QUERY_HISTORY",
+      payload: {
+        key: "LOGS_QUERY_HISTORY",
+        history: [history],
+      }
+    });
+  };
+  
   const [isUpdatingQuery, setIsUpdatingQuery] = useState(false);
   const [period, setPeriod] = useState<TimeParams>(periodState);
   const [queryError, setQueryError] = useState<ErrorTypes | string>("");
@@ -60,6 +76,7 @@ const ExploreLogs: FC = () => {
       "g0.end_input": newPeriod.date,
       "g0.relative_time": relativeTime || "none",
     });
+    updateHistory();
   };
 
   const handleChangeLimit = (limit: number) => {

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx

@@ -14,6 +14,7 @@ import GroupLogs from "../GroupLogs/GroupLogs";
 import JsonView from "../../../components/Views/JsonView/JsonView";
 import LineLoader from "../../../components/Main/LineLoader/LineLoader";
 import SelectLimit from "../../../components/Main/Pagination/SelectLimit/SelectLimit";
+import DownloadLogsButton from "../DownloadLogsButton/DownloadLogsButton";
 
 const MemoizedTableLogs = React.memo(TableLogs);
 const MemoizedGroupLogs = React.memo(GroupLogs);
@@ -83,7 +84,12 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, isLoading }) => {
           "vm-explore-logs-body-header_mobile": isMobile,
         })}
       >
-        <div className="vm-section-header__tabs">
+        <div
+          className={classNames({
+            "vm-section-header__tabs": true,
+            "vm-explore-logs-body-header__tabs_mobile": isMobile,
+          })}
+        >
           <Tabs
             activeItem={String(activeTab)}
             items={tabs}
@@ -99,20 +105,28 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, isLoading }) => {
               limit={rowsPerPage}
               onChange={handleSetRowsPerPage}
             />
-            <TableSettings
-              columns={columns}
-              selectedColumns={displayColumns}
-              onChangeColumns={setDisplayColumns}
-              tableCompact={tableCompact}
-              toggleTableCompact={toggleTableCompact}
-            />
+            <div className="vm-explore-logs-body-header__table-settings">
+              {data.length > 0 && <DownloadLogsButton logs={data} />}
+              <TableSettings
+                columns={columns}
+                selectedColumns={displayColumns}
+                onChangeColumns={setDisplayColumns}
+                tableCompact={tableCompact}
+                toggleTableCompact={toggleTableCompact}
+              />
+            </div>
           </div>
         )}
         {activeTab === DisplayType.group && (
-          <div
-            className="vm-explore-logs-body-header__settings"
-            ref={groupSettingsRef}
-          />
+          <>
+            <div
+              className="vm-explore-logs-body-header__settings"
+              ref={groupSettingsRef}
+            />
+          </>
+        )}
+        {activeTab === DisplayType.json && data.length > 0 && (
+          <DownloadLogsButton logs={data} />
         )}
       </div>
 

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/style.scss

@@ -8,12 +8,20 @@
 
     &_mobile {
       margin: -$padding-global 0-$padding-global 0;
+      display: block;
+      border-bottom: none;
     }
 
     &__settings {
       display: flex;
       align-items: center;
       gap: $padding-small;
+      justify-content: flex-end;
+    }
+
+    &__table-settings {
+      display: flex;
+      flex-direction: row;
     }
 
     &__log-info {
@@ -23,6 +31,12 @@
       color: $color-text-secondary;
       font-size: $font-size-small;
     }
+
+    &__tabs {
+      &_mobile {
+        border-bottom: var(--border-divider);
+      }
+    }
   }
 
   &__empty {
@@ -45,6 +59,7 @@
 
     &_mobile {
       width: calc(100vw - ($padding-global * 2) - var(--scrollbar-width));
+      padding-top: $padding-large;
     }
 
     .vm-table {

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsHeader/ExploreLogsHeader.tsx

@@ -9,6 +9,8 @@ import TextField from "../../../components/Main/TextField/TextField";
 import LogsQueryEditorAutocomplete from "../../../components/Configurators/QueryEditor/LogsQL/LogsQueryEditorAutocomplete";
 import { useQueryDispatch, useQueryState } from "../../../state/query/QueryStateContext";
 import Switch from "../../../components/Main/Switch/Switch";
+import QueryHistory from "../../../components/QueryHistory/QueryHistory";
+import useBoolean from "../../../hooks/useBoolean";
 
 export interface ExploreLogHeaderProps {
   query: string;
@@ -30,11 +32,12 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
   onRun,
 }) => {
   const { isMobile } = useDeviceDetect();
-  const { autocomplete } = useQueryState();
+  const { autocomplete, queryHistory } = useQueryState();
   const queryDispatch = useQueryDispatch();
 
   const [errorLimit, setErrorLimit] = useState("");
   const [limitInput, setLimitInput] = useState(limit);
+  const { value: awaitQuery, setValue: setAwaitQuery } = useBoolean(false);
 
   const handleChangeLimit = (val: string) => {
     const number = +val;
@@ -55,6 +58,33 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
     setLimitInput(limit);
   }, [limit]);
 
+  const handleHistoryChange = (step: number) => {
+    const { values, index } = queryHistory[0];
+    const newIndexHistory = index + step;
+    if (newIndexHistory < 0 || newIndexHistory >= values.length) return;
+    onChange(values[newIndexHistory] || "");
+    queryDispatch({
+      type: "SET_QUERY_HISTORY_BY_INDEX",
+      payload: { value: { values, index: newIndexHistory }, queryNumber: 0 }
+    });
+  };
+
+  const handleSelectHistory = (value: string) => {
+    onChange(value);
+    setAwaitQuery(true);
+  };
+
+  const createHandlerArrow = (step: number) => () => {
+    handleHistoryChange(step);
+  };
+
+  useEffect(() => {
+    if (awaitQuery) {
+      onRun();
+      setAwaitQuery(false);
+    }
+  }, [query, awaitQuery]);
+
   return (
     <div
       className={classNames({
@@ -68,8 +98,8 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
           value={query}
           autocomplete={autocomplete}
           autocompleteEl={LogsQueryEditorAutocomplete}
-          onArrowUp={() => null}
-          onArrowDown={() => null}
+          onArrowUp={createHandlerArrow(-1)}
+          onArrowDown={createHandlerArrow(1)}
           onEnter={onRun}
           onChange={onChange}
           label={"Log query"}
@@ -113,6 +143,10 @@ const ExploreLogsHeader: FC<ExploreLogHeaderProps> = ({
             Documentation
           </a>
         </div>
+        <QueryHistory
+          handleSelectQuery={handleSelectHistory}
+          historyKey={"LOGS_QUERY_HISTORY"}
+        />
         <div className="vm-explore-logs-header-bottom-execute">
           <Button
             startIcon={isLoading ? <SpinnerIcon/> : <PlayIcon/>}

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx

@@ -18,6 +18,8 @@ import SelectLimit from "../../../components/Main/Pagination/SelectLimit/SelectL
 import { usePaginateGroups } from "../hooks/usePaginateGroups";
 import { GroupLogsType } from "../../../types";
 import { getNanoTimestamp } from "../../../utils/time";
+import useDeviceDetect from "../../../hooks/useDeviceDetect";
+import DownloadLogsButton from "../DownloadLogsButton/DownloadLogsButton";
 
 interface Props {
   logs: Logs[];
@@ -25,6 +27,7 @@ interface Props {
 }
 
 const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
+  const { isMobile } = useDeviceDetect();
   const [searchParams, setSearchParams] = useSearchParams();
 
   const [page, setPage] = useState(1);
@@ -94,7 +97,7 @@ const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
   };
 
   useEffect(() => {
-    setExpandGroups(new Array(groupData.length).fill(true));
+    setExpandGroups(new Array(groupData.length).fill(!isMobile));
   }, [groupData]);
 
   useEffect(() => {
@@ -159,6 +162,7 @@ const GroupLogs: FC<Props> = ({ logs, settingsRef }) => {
               ariaLabel={expandAll ? "Collapse All" : "Expand All"}
             />
           </Tooltip>
+          <DownloadLogsButton logs={logs} />
           <GroupLogsConfigurators logs={logs}/>
         </div>
       ), settingsRef.current)}

app/vmui/packages/vmui/src/state/query/reducer.ts

@@ -1,6 +1,6 @@
 import { getFromStorage, saveToStorage } from "../../utils/storage";
 import { getQueryArray } from "../../utils/query-string";
-import { setQueriesToStorage } from "../../pages/CustomPanel/QueryHistory/utils";
+import { HistoryKey, setQueriesToStorage } from "../../components/QueryHistory/utils";
 import {
   QueryAutocompleteCache,
   QueryAutocompleteCacheItem
@@ -24,7 +24,7 @@ export interface QueryState {
 export type QueryAction =
   | { type: "SET_QUERY", payload: string[] }
   | { type: "SET_QUERY_HISTORY_BY_INDEX", payload: { value: QueryHistoryType, queryNumber: number } }
-  | { type: "SET_QUERY_HISTORY", payload: QueryHistoryType[] }
+  | { type: "SET_QUERY_HISTORY", payload: { key: HistoryKey, history: QueryHistoryType[] } }
   | { type: "TOGGLE_AUTOCOMPLETE" }
   | { type: "SET_AUTOCOMPLETE_QUICK", payload: boolean }
   | { type: "SET_AUTOCOMPLETE_CACHE", payload: { key: QueryAutocompleteCacheItem, value: string[] } }
@@ -48,10 +48,10 @@ export function reducer(state: QueryState, action: QueryAction): QueryState {
         query: action.payload.map(q => q)
       };
     case "SET_QUERY_HISTORY":
-      setQueriesToStorage(action.payload);
+      setQueriesToStorage(action.payload.key, action.payload.history);
       return {
         ...state,
-        queryHistory: action.payload
+        queryHistory: action.payload.history
       };
     case "SET_QUERY_HISTORY_BY_INDEX":
       state.queryHistory.splice(action.payload.queryNumber, 1, action.payload.value);

app/vmui/packages/vmui/src/utils/file.ts

@@ -0,0 +1,48 @@
+export const downloadFile = (data: Blob, filename: string) => {
+  const link = document.createElement("a");
+  const url = URL.createObjectURL(data);
+
+  link.setAttribute("href", url);
+  link.setAttribute("download", filename);
+
+  document.body.appendChild(link);
+  link.click();
+  document.body.removeChild(link);
+  URL.revokeObjectURL(url);
+};
+
+export const downloadCSV = (data: Record<string, string>[], filename: string) => {
+  const getHeader = (data: Record<string, string>[]) => {
+    const headersObj = data.reduce<Record<string, boolean>>((headers, row) => {
+      Object.keys(row).forEach((key) => {
+        if(key && !headers[key]){
+          headers[key] = true;
+        }
+      });
+      return headers;
+    }, {});
+    return Object.keys(headersObj);
+  };
+
+  const formatValueToCSV= (value: string) =>
+    (value.includes(",") || value.includes("\n") || value.includes("\""))
+      ? "\"" + value.replace(/"/g, "\"\"") + "\""
+      : value;
+
+  const convertToCSV = (data: Record<string, string>[]): string => {
+    const header = getHeader(data);
+    const rows = data.map(item =>
+      header.map(fieldName => item[fieldName] ? formatValueToCSV(item[fieldName]): "").join(",")
+    );
+    return [header.map(formatValueToCSV).join(","), ...rows].join("\r\n");
+  };
+
+  const csvContent = convertToCSV(data);
+  const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" });
+  downloadFile(blob, filename);
+};
+
+export const downloadJSON = (data: string, filename: string) => {
+  const blob = new Blob([data], { type: "application/json" });
+  downloadFile(blob, filename);
+};

app/vmui/packages/vmui/src/utils/storage.ts

@@ -1,18 +1,26 @@
+/**
+ * Do not use this type in local storage type
+ * @deprecated
+ * */
+type DeprecatedStorageKeys = "QUERY_HISTORY" | "QUERY_FAVORITES";
+
 export type StorageKeys = "AUTOCOMPLETE"
-    | "NO_CACHE"
-    | "QUERY_TRACING"
-    | "SERIES_LIMITS"
-    | "TABLE_COMPACT"
-    | "TIMEZONE"
-    | "DISABLED_DEFAULT_TIMEZONE"
-    | "THEME"
-    | "LOGS_LIMIT"
-    | "LOGS_MARKDOWN"
-    | "LOGS_DISABLED_HOVERS"
-    | "EXPLORE_METRICS_TIPS"
-    | "QUERY_HISTORY"
-    | "QUERY_FAVORITES"
-    | "SERVER_URL"
+  | "NO_CACHE"
+  | "QUERY_TRACING"
+  | "SERIES_LIMITS"
+  | "TABLE_COMPACT"
+  | "TIMEZONE"
+  | "DISABLED_DEFAULT_TIMEZONE"
+  | "THEME"
+  | "LOGS_LIMIT"
+  | "LOGS_MARKDOWN"
+  | "LOGS_DISABLED_HOVERS"
+  | "EXPLORE_METRICS_TIPS"
+  | "LOGS_QUERY_HISTORY"
+  | "METRICS_QUERY_HISTORY"
+  | "SERVER_URL"
+  | DeprecatedStorageKeys;
+
 
 export const saveToStorage = (key: StorageKeys, value: string | boolean | Record<string, unknown>): void => {
   if (value) {

apptest/model.go

@@ -344,3 +344,53 @@ type SnapshotDeleteResponse struct {
 type SnapshotDeleteAllResponse struct {
 	Status string
 }
+
+// TSDBStatusResponse is an in-memory reprensentation of the json response
+// returned by the /prometheus/api/v1/status/tsdb endpoint.
+type TSDBStatusResponse struct {
+	IsPartial bool
+	Data      TSDBStatusResponseData
+}
+
+// Sort performs sorting of stats entries
+func (tsr *TSDBStatusResponse) Sort() {
+	sortTSDBStatusResponseEntries(tsr.Data.SeriesCountByLabelName)
+	sortTSDBStatusResponseEntries(tsr.Data.SeriesCountByFocusLabelValue)
+	sortTSDBStatusResponseEntries(tsr.Data.SeriesCountByLabelValuePair)
+	sortTSDBStatusResponseEntries(tsr.Data.LabelValueCountByLabelName)
+}
+
+// TSDBStatusResponseData is a part of TSDBStatusResponse
+type TSDBStatusResponseData struct {
+	TotalSeries                  int
+	TotalLabelValuePairs         int
+	SeriesCountByMetricName      []TSDBStatusResponseMetricNameEntry
+	SeriesCountByLabelName       []TSDBStatusResponseEntry
+	SeriesCountByFocusLabelValue []TSDBStatusResponseEntry
+	SeriesCountByLabelValuePair  []TSDBStatusResponseEntry
+	LabelValueCountByLabelName   []TSDBStatusResponseEntry
+}
+
+// TSDBStatusResponseEntry defines stats entry for TSDBStatusResponseData
+type TSDBStatusResponseEntry struct {
+	Name  string
+	Count int
+}
+
+// TSDBStatusResponseMetricNameEntry defines metric names stats entry for TSDBStatusResponseData
+type TSDBStatusResponseMetricNameEntry struct {
+	Name                 string
+	Count                int
+	RequestsCount        int
+	LastRequestTimestamp int
+}
+
+func sortTSDBStatusResponseEntries(entries []TSDBStatusResponseEntry) {
+	sort.Slice(entries, func(i, j int) bool {
+		left, right := entries[i], entries[j]
+		if left.Count == right.Count {
+			return left.Name < right.Name
+		}
+		return left.Count < right.Count
+	})
+}

apptest/tests/metric_names_stats_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
@@ -19,6 +20,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 
 	const ingestDateTime = `2024-02-05T08:57:36.700Z`
 	const ingestTimestamp = ` 1707123456700`
+	const date = `2024-02-05`
 	dataSet := []string{
 		`metric_name_1{label="foo"} 10`,
 		`metric_name_1{label="bar"} 10`,
@@ -29,6 +31,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 	for idx := range dataSet {
 		dataSet[idx] += ingestTimestamp
 	}
+	tsdbMetricNameEntryCmpOpts := cmpopts.IgnoreFields(apptest.TSDBStatusResponseMetricNameEntry{}, "LastRequestTimestamp")
 
 	sut.PrometheusAPIV1ImportPrometheus(t, dataSet, at.QueryOpts{})
 	sut.ForceFlush(t)
@@ -60,6 +63,31 @@ func TestSingleMetricNamesStats(t *testing.T) {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
+	expectedStatsResponse := apptest.TSDBStatusResponse{
+		Data: at.TSDBStatusResponseData{
+			TotalSeries:          5,
+			TotalLabelValuePairs: 10,
+			SeriesCountByMetricName: []apptest.TSDBStatusResponseMetricNameEntry{
+				{Name: "metric_name_1", RequestsCount: 3},
+				{Name: "metric_name_2", RequestsCount: 1},
+				{Name: "metric_name_3", RequestsCount: 1},
+			},
+			SeriesCountByLabelName:       []apptest.TSDBStatusResponseEntry{{Name: "__name__"}, {Name: "label"}},
+			SeriesCountByFocusLabelValue: []apptest.TSDBStatusResponseEntry{},
+			SeriesCountByLabelValuePair: []apptest.TSDBStatusResponseEntry{
+				{Name: "__name__=metric_name_1"}, {Name: "label=baz"},
+				{Name: "__name__=metric_name_2"}, {Name: "__name__=metric_name_3"},
+				{Name: "label=bar"}, {Name: "label=foo"},
+			},
+			LabelValueCountByLabelName: []apptest.TSDBStatusResponseEntry{{Name: "__name__"}, {Name: "label"}},
+		},
+	}
+	expectedStatsResponse.Sort()
+	gotStatus := sut.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{})
+	if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
+		t.Errorf("unexpected APIV1StatusTSDB response (-want, +got):\n%s", diff)
+	}
+
 	// perform query request for single metric and check counter increase
 	sut.PrometheusAPIV1Query(t, `metric_name_2`, at.QueryOpts{Time: ingestDateTime})
 	expected = apptest.MetricNamesStatsResponse{
@@ -129,6 +157,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 
 	const ingestDateTime = `2024-02-05T08:57:36.700Z`
 	const ingestTimestamp = ` 1707123456700`
+	const date = `2024-02-05`
 	dataSet := []string{
 		`metric_name_1{label="foo"} 10`,
 		`metric_name_1{label="bar"} 10`,
@@ -140,6 +169,8 @@ func TestClusterMetricNamesStats(t *testing.T) {
 		dataSet[idx] += ingestTimestamp
 	}
 
+	tsdbMetricNameEntryCmpOpts := cmpopts.IgnoreFields(apptest.TSDBStatusResponseMetricNameEntry{}, "LastRequestTimestamp")
+
 	// ingest per tenant data and verify it with search
 	tenantIDs := []string{"1:1", "1:15", "15:15"}
 	for _, tenantID := range tenantIDs {
@@ -176,6 +207,31 @@ func TestClusterMetricNamesStats(t *testing.T) {
 		if diff := cmp.Diff(expected, gotStats); diff != "" {
 			t.Errorf("unexpected response tenant: %s (-want, +got):\n%s", tenantID, diff)
 		}
+
+		expectedStatsResponse := apptest.TSDBStatusResponse{
+			Data: at.TSDBStatusResponseData{
+				TotalSeries:          5,
+				TotalLabelValuePairs: 10,
+				SeriesCountByMetricName: []apptest.TSDBStatusResponseMetricNameEntry{
+					{Name: "metric_name_1", RequestsCount: 3},
+					{Name: "metric_name_2", RequestsCount: 1},
+					{Name: "metric_name_3", RequestsCount: 1},
+				},
+				SeriesCountByLabelName:       []apptest.TSDBStatusResponseEntry{{Name: "__name__"}, {Name: "label"}},
+				SeriesCountByFocusLabelValue: []apptest.TSDBStatusResponseEntry{},
+				SeriesCountByLabelValuePair: []apptest.TSDBStatusResponseEntry{
+					{Name: "__name__=metric_name_1"}, {Name: "label=baz"},
+					{Name: "__name__=metric_name_2"}, {Name: "__name__=metric_name_3"},
+					{Name: "label=bar"}, {Name: "label=foo"},
+				},
+				LabelValueCountByLabelName: []apptest.TSDBStatusResponseEntry{{Name: "__name__"}, {Name: "label"}},
+			},
+		}
+		expectedStatsResponse.Sort()
+		gotStatus := vmselect.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{Tenant: tenantID})
+		if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
+			t.Errorf("unexpected APIV1StatusTSDB response tenant: %s (-want, +got):\n%s", tenantID, diff)
+		}
 	}
 
 	// verify multitenant stats

apptest/tests/vmagent_remotewrite_test.go

@@ -2,6 +2,9 @@ package tests
 
 import (
 	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
@@ -30,6 +33,7 @@ func testSingleVMAgentRemoteWrite(t *testing.T, forcePromProto bool) {
 		`-remoteWrite.flushInterval=50ms`,
 		fmt.Sprintf(`-remoteWrite.forcePromProto=%v`, forcePromProto),
 		fmt.Sprintf(`-remoteWrite.url=http://%s/api/v1/write`, vmsingle.HTTPAddr()),
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
 	}, ``)
 
 	vmagent.APIV1ImportPrometheus(t, []string{
@@ -52,3 +56,123 @@ func testSingleVMAgentRemoteWrite(t *testing.T, forcePromProto bool) {
 		},
 	})
 }
+
+// TestSingleVMAgentUnsupportedMediaTypeDropIfSnappy verifies that the remote write process:
+// - Starts with Prometheus remote write protocol using `snappy`.
+// - Does not retry `snappy`-encoded requests if they fail; instead, they are dropped.
+func TestSingleVMAgentUnsupportedMediaTypeDropIfSnappy(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	var remoteWriteContentEncodingsMux sync.Mutex
+	var remoteWriteContentEncodings []string
+	// remoteWriteSrv is a stub HTTP server simulate a remote write endpoint with the following behavior:
+	// - Fail all requests with `415 Unsupported Media Type`.
+	// - Records received `Content-Encoding` header.
+	remoteWriteSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		remoteWriteContentEncodingsMux.Lock()
+		remoteWriteContentEncodings = append(remoteWriteContentEncodings, r.Header.Get(`Content-Encoding`))
+		remoteWriteContentEncodingsMux.Unlock()
+
+		w.WriteHeader(http.StatusUnsupportedMediaType)
+	}))
+	defer remoteWriteSrv.Close()
+
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
+		`-remoteWrite.forcePromProto=true`,
+		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
+	}, ``)
+
+	vmagent.APIV1ImportPrometheusNoWaitFlush(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	vmagent.APIV1ImportPrometheusNoWaitFlush(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	tc.Assert(&at.AssertOptions{
+		Msg: `unexpected content encoding headers sent to remote write server; expected zstd`,
+		Got: func() any {
+			remoteWriteContentEncodingsMux.Lock()
+			defer remoteWriteContentEncodingsMux.Unlock()
+
+			return append([]string(nil), remoteWriteContentEncodings...)
+		},
+		Want: []string{`snappy`, `snappy`},
+	})
+
+	expectedRetriesCount := 0
+	if actualRetriesCount := vmagent.RemoteWriteRequestsRetriesCountTotal(t); actualRetriesCount != expectedRetriesCount {
+		t.Fatalf("unexpected number of retries; got %d, want %d", actualRetriesCount, expectedRetriesCount)
+	}
+	expectedPacketsDroppedTotal := 2
+	if actualPacketsDroppedCount := vmagent.RemoteWritePacketsDroppedTotal(t); actualPacketsDroppedCount != expectedPacketsDroppedTotal {
+		t.Fatalf("unexpected number of dropped packets; got %d, want %d", actualPacketsDroppedCount, expectedPacketsDroppedTotal)
+	}
+}
+
+// TestSingleVMAgentDowngradeRemoteWriteProtocol verifies that the remote write process:
+// - Starts with VictoriaMetrics remote write protocol using `zstd`.
+// - Upon receiving `415 Unsupported Media Type`, downgrades to Prometheus remote write with `snappy`.
+// - Re-packs and retries failed requests.
+// - Sends all subsequent requests using `snappy`.
+func TestSingleVMAgentDowngradeRemoteWriteProtocol(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	var remoteWriteContentEncodings []string
+	// remoteWriteSrv is a stub HTTP server that simulates a remote write endpoint with the following behavior:
+	// - Rejects requests with `zstd` encoding by responding with `415 Unsupported Media Type`.
+	// - Accepts requests with `snappy` encoding.
+	// - Records the `Content-Encoding` header of incoming requests.
+	remoteWriteSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		remoteWriteContentEncodings = append(remoteWriteContentEncodings, r.Header.Get(`Content-Encoding`))
+
+		if r.Header.Get(`Content-Encoding`) == `zstd` {
+			w.WriteHeader(http.StatusUnsupportedMediaType)
+			_, _ = w.Write([]byte(`zstd not supported`))
+
+			return
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer remoteWriteSrv.Close()
+
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
+		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
+	}, ``)
+
+	// Send request encoded with `zstd`; it fails, gets repacked as `snappy`, and retries successfully.
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	// Send request encoded with `snappy` immediately; it succeeds without retries.
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	tc.Assert(&at.AssertOptions{
+		Msg: `unexpected content encoding headers sent to remote write server`,
+		Got: func() any {
+			return remoteWriteContentEncodings
+		},
+		Want:       []string{`zstd`, `snappy`, `snappy`},
+		DoNotRetry: true,
+	})
+
+	expectedRetriesCount := 1
+	if actualRetriesCount := vmagent.RemoteWriteRequestsRetriesCountTotal(t); actualRetriesCount != expectedRetriesCount {
+		t.Fatalf("unexpected number of retries; got %d, want %d", actualRetriesCount, expectedRetriesCount)
+	}
+	expectedPacketsDroppedTotal := 0
+	if actualPacketsDroppedCount := vmagent.RemoteWritePacketsDroppedTotal(t); actualPacketsDroppedCount != expectedPacketsDroppedTotal {
+		t.Fatalf("unexpected number of dropped packets; got %d, want %d", actualPacketsDroppedCount, expectedPacketsDroppedTotal)
+	}
+}

apptest/vmagent.go

@@ -3,6 +3,7 @@ package apptest
 import (
 	"fmt"
 	"net/http"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -28,8 +29,9 @@ func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfig
 
 	app, stderrExtracts, err := startApp(instance, "../../bin/vmagent", flags, &appOptions{
 		defaultFlags: map[string]string{
-			"-httpListenAddr":    "127.0.0.1:0",
-			"-promscrape.config": promScrapeConfigFilePath,
+			"-httpListenAddr":          "127.0.0.1:0",
+			"-promscrape.config":       promScrapeConfigFilePath,
+			"-remoteWrite.tmpDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
 		},
 		extractREs: extractREs,
 	})
@@ -55,16 +57,48 @@ func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfig
 // The call is blocked until the data is flushed to vmstorage or the timeout is reached.
 //
 // See https://docs.victoriametrics.com/url-examples/#apiv1importprometheus
-func (app *Vmagent) APIV1ImportPrometheus(t *testing.T, records []string, _ QueryOpts) {
+func (app *Vmagent) APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	app.sendBlocking(t, len(records), func() {
+		app.APIV1ImportPrometheusNoWaitFlush(t, records, opts)
+	})
+}
+
+// APIV1ImportPrometheusNoWaitFlush is a test helper function that inserts a
+// collection of records in Prometheus text exposition format for the given
+// tenant by sending a HTTP POST request to /api/v1/import/prometheus vmagent endpoint.
+//
+// The call accepts the records but does not guarantee successful flush to vmstorage.
+// Flushing may still be in progress on the function return.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1importprometheus
+func (app *Vmagent) APIV1ImportPrometheusNoWaitFlush(t *testing.T, records []string, _ QueryOpts) {
 	t.Helper()
 
 	data := []byte(strings.Join(records, "\n"))
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, app.apiV1ImportPrometheusURL, "text/plain", data)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
+	_, statusCode := app.cli.Post(t, app.apiV1ImportPrometheusURL, "text/plain", data)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+	}
+}
+
+// RemoteWriteRequestsRetriesCountTotal sums up the total retries for remote write requests.
+func (app *Vmagent) RemoteWriteRequestsRetriesCountTotal(t *testing.T) int {
+	total := 0.0
+	for _, v := range app.GetMetricsByPrefix(t, "vmagent_remotewrite_retries_count_total") {
+		total += v
+	}
+	return int(total)
+}
+
+// RemoteWritePacketsDroppedTotal sums up the total number of dropped remote write packets.
+func (app *Vmagent) RemoteWritePacketsDroppedTotal(t *testing.T) int {
+	total := 0.0
+	for _, v := range app.GetMetricsByPrefix(t, "vmagent_remotewrite_packets_dropped_total") {
+		total += v
+	}
+	return int(total)
 }
 
 // sendBlocking sends the data to vmstorage by executing `send` function and

apptest/vmselect.go

@@ -174,6 +174,37 @@ func (app *Vmselect) MetricNamesStatsReset(t *testing.T, opts QueryOpts) {
 	}
 }
 
+// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
+// //
+// See https://docs.victoriametrics.com/#tsdb-stats
+func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/status/tsdb", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	addNonEmpty := func(name, value string) {
+		if len(value) == 0 {
+			return
+		}
+		values.Add(name, value)
+	}
+	addNonEmpty("match[]", matchQuery)
+	addNonEmpty("topN", topN)
+	addNonEmpty("date", date)
+
+	res, statusCode := app.cli.PostForm(t, seriesURL, values)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var status TSDBStatusResponse
+	if err := json.Unmarshal([]byte(res), &status); err != nil {
+		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
+	}
+	status.Sort()
+	return status
+}
+
 // String returns the string representation of the vmselect app state.
 func (app *Vmselect) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)

apptest/vmsingle.go

@@ -350,6 +350,37 @@ func (app *Vmsingle) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse
 	return &res
 }
 
+// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
+// //
+// See https://docs.victoriametrics.com/#tsdb-stats
+func (app *Vmsingle) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/prometheus/api/v1/status/tsdb", app.httpListenAddr)
+	values := opts.asURLValues()
+	addNonEmpty := func(name, value string) {
+		if len(value) == 0 {
+			return
+		}
+		values.Add(name, value)
+	}
+	addNonEmpty("match[]", matchQuery)
+	addNonEmpty("topN", topN)
+	addNonEmpty("date", date)
+
+	res, statusCode := app.cli.PostForm(t, seriesURL, values)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var status TSDBStatusResponse
+	if err := json.Unmarshal([]byte(res), &status); err != nil {
+		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
+	}
+	status.Sort()
+	return status
+}
+
 // HTTPAddr returns the address at which the vmstorage process is listening
 // for http connections.
 func (app *Vmsingle) HTTPAddr() string {

dashboards/Makefile

@@ -19,3 +19,4 @@ dashboards-sync:
 	SRC=backupmanager.json D_UID=gF-lxRdVz TITLE="VictoriaMetrics - backupmanager" $(MAKE) dashboard-copy
 	SRC=clusterbytenant.json D_UID=IZFqd3lMz TITLE="VictoriaMetrics Cluster Per Tenant Statistic" $(MAKE) dashboard-copy
 	SRC=victorialogs.json D_UID=OqPIZTX4z TITLE="VictoriaLogs" $(MAKE) dashboard-copy
+	SRC=victorialogs-cluster.json D_UID=XqCOFEX4z TITLE="VictoriaLogs - cluster" $(MAKE) dashboard-copy

deployment/docker/Makefile

@@ -208,32 +208,40 @@ package-via-docker-386:
 remove-docker-images:
 	docker image ls --format '{{.ID}}' | xargs docker image rm -f
 
-docker-single-up:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml up -d
+# VM single
+docker-vm-single-up:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vm-single.yml up -d
 
-docker-single-down:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml down -v
+docker-vm-single-down:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vm-single.yml down -v
 
-docker-single-vm-datasource-up:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml -f deployment/docker/vm-datasource/docker-compose.yml up -d
+# VM cluster
+docker-vm-cluster-up:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vm-cluster.yml up -d
 
-docker-single-vm-datasource-down:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml -f deployment/docker/vm-datasource/docker-compose.yml down -v
+docker-vm-cluster-down:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vm-cluster.yml down -v
 
-docker-cluster-up:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml up -d
+# VL single
+docker-vl-single-up:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vl-single.yml up -d
 
-docker-cluster-down:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml down -v
+docker-vl-single-down:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vl-single.yml down -v
 
-docker-cluster-vm-datasource-up:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml -f deployment/docker/vm-datasource/docker-compose-cluster.yml up -d
+# VL cluster
+docker-vl-cluster-up:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vl-cluster.yml up -d
 
-docker-cluster-vm-datasource-down:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml -f deployment/docker/vm-datasource/docker-compose-cluster.yml down -v
+docker-vl-cluster-down:
+	$(DOCKER_COMPOSE) -f deployment/docker/compose-vl-cluster.yml down -v
 
-docker-victorialogs-up:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-victorialogs.yml up -d
+# Command aliases to keep backward-compatibility, as they could have been mentioned on the Internet before the rename.
+docker-single-up: docker-vm-single-up
+docker-single-down: docker-vm-single-down
 
-docker-victorialogs-down:
-	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-victorialogs.yml down -v
+docker-cluster-up: docker-vm-cluster-up
+docker-cluster-down: docker-vm-cluster-down
+
+docker-victorialogs-up: docker-vl-single-up
+docker-victorialogs-down: docker-vl-single-down

deployment/docker/README.md

@@ -1,41 +1,44 @@
 # Docker compose environment for VictoriaMetrics
 
-Docker compose environment for VictoriaMetrics includes VictoriaMetrics components,
+Docker compose environment for VictoriaMetrics includes VictoriaMetrics and VictoriaLogs components,
 [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) 
 and [Grafana](https://grafana.com/).
 
-For starting the docker-compose environment ensure you have docker installed and running and access to the Internet.
-**All commands should be executed from the root directory of [the repo](https://github.com/VictoriaMetrics/VictoriaMetrics).**
+For starting the docker-compose environment ensure that you have docker installed and running, and that you have access
+to the Internet.
+**All commands should be executed from the root directory of [the VictoriaMetrics repo](https://github.com/VictoriaMetrics/VictoriaMetrics).**
 
-* [VictoriaMetrics single server](#victoriametrics-single-server)
-* [VictoriaMetrics cluster](#victoriametrics-cluster)
-* [vmagent](#vmagent)
-* [vmauth](#vmauth)
-* [vmalert](#vmalert)
-* [alertmanager](#alertmanager)
+* Metrics:
+  * [VictoriaMetrics single server](#victoriametrics-single-server)
+  * [VictoriaMetrics cluster](#victoriametrics-cluster)
+  * [vmagent](#vmagent)
+* Logs:
+  * [VictoriaLogs single server](#victoriaLogs-server)
+  * [VictoriaLogs cluster](#victoriaLogs-cluster)
+* [Common](#common-components)
+  * [vmauth](#vmauth)
+  * [vmalert](#vmalert)
+  * [alertmanager](#alertmanager)
+  * [Grafana](#grafana)
 * [Alerts](#alerts)
-* [Grafana](#grafana)
-* [VictoriaLogs](#victoriaLogs-server)
-
 
 ## VictoriaMetrics single server
 
 To spin-up environment with VictoriaMetrics single server run the following command:
 ```
-make docker-single-up 
+make docker-vm-single-up
 ```
 
 VictoriaMetrics will be accessible on the following ports:
-
 * `--graphiteListenAddr=:2003`
 * `--opentsdbListenAddr=:4242`
 * `--httpListenAddr=:8428`
 
 The communication scheme between components is the following:
-* [vmagent](#vmagent) sends scraped metrics to `single server VictoriaMetrics`;
-* [grafana](#grafana) is configured with datasource pointing to `single server VictoriaMetrics`;
-* [vmalert](#vmalert) is configured to query `single server VictoriaMetrics` and send alerts state
-  and recording rules back to it;
+* [vmagent](#vmagent) sends scraped metrics to `VictoriaMetrics single-node`;
+* [grafana](#grafana) is configured with datasource pointing to `VictoriaMetrics single-node`;
+* [vmalert](#vmalert) is configured to query `VictoriaMetrics single-node`, and send alerts state
+  and recording rules results back to `vmagent`;
 * [alertmanager](#alertmanager) is configured to receive notifications from `vmalert`.
 
 <img alt="VictoriaMetrics single-server deployment" width="500" src="assets/vm-single-server.png">
@@ -47,31 +50,30 @@ use link [http://localhost:8428/vmui](http://localhost:8428/vmui).
 
 To access `vmalert` use link [http://localhost:8428/vmalert](http://localhost:8428/vmalert/).
 
-To shutdown environment execute the following command:
+To shutdown environment run:
 ```
-make docker-single-down
+make docker-vm-single-down
 ```
 
-
 ## VictoriaMetrics cluster
 
 To spin-up environment with VictoriaMetrics cluster run the following command:
 ```
-make docker-cluster-up
+make docker-vm-cluster-up
 ```
 
 VictoriaMetrics cluster environment consists of `vminsert`, `vmstorage` and `vmselect` components.
-`vminsert` has exposed port `:8480`, access to `vmselect` components goes through `vmauth` on port `:8427`,
+`vminsert` exposes port `:8480` for ingestion. Access to `vmselect` for reads goes through `vmauth` on port `:8427`,
 and the rest of components are available only inside the environment.
 
 The communication scheme between components is the following:
 * [vmagent](#vmagent) sends scraped metrics to `vminsert`;
-* `vminsert` forwards data to `vmstorage`;
+* `vminsert` shards and forwards data to `vmstorage`;
 * `vmselect`s are connected to `vmstorage` for querying data;
 * [vmauth](#vmauth) balances incoming read requests among `vmselect`s;
 * [grafana](#grafana) is configured with datasource pointing to `vmauth`;
 * [vmalert](#vmalert) is configured to query `vmselect`s via `vmauth` and send alerts state
-  and recording rules to `vminsert`;
+  and recording rules to `vmagent`;
 * [alertmanager](#alertmanager) is configured to receive notifications from `vmalert`.
 
 <img alt="VictoriaMetrics cluster deployment" width="500" src="assets/vm-cluster.png">
@@ -85,118 +87,89 @@ To access `vmalert` use link [http://localhost:8427/select/0/prometheus/vmalert/
 
 To shutdown environment execute the following command:
 ```
-make docker-cluster-down
+make docker-vm-cluster-down
 ```
 
 ## vmagent
 
 vmagent is used for scraping and pushing time series to VictoriaMetrics instance. 
-It accepts Prometheus-compatible configuration [prometheus.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus.yml)
-with listed targets for scraping.
+It accepts Prometheus-compatible configuration with listed targets for scraping:
+* [scraping VictoriaMetrics single-node](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-single.yml) services;
+* [scraping VictoriaMetrics cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-cluster.yml) services;
+* [scraping VictoriaLogs single-node](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vl-single.yml) services;
+* [scraping VictoriaLogs cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vl-cluster.yml) services;
 
-[Web interface link](http://localhost:8429/).
-
-## vmauth
-
-[vmauth](https://docs.victoriametrics.com/vmauth/) acts as a [balancer](https://docs.victoriametrics.com/vmauth/#load-balancing)
-to spread the load across `vmselect`'s. [Grafana](#grafana) and [vmalert](#vmalert) use vmauth for read queries.
-vmauth config is available [here](ttps://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/auth-cluster.yml)
-
-
-## vmalert
-
-vmalert evaluates alerting rules [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts.yml)
-to track VictoriaMetrics health state. It is connected with AlertManager for firing alerts,
-and with VictoriaMetrics for executing queries and storing alert's state.
-
-[Web interface link](http://localhost:8880/).
-
-## alertmanager
-
-AlertManager accepts notifications from `vmalert` and fires alerts.
-All notifications are blackholed according to [alertmanager.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alertmanager.yml) config.
-
-[Web interface link](http://localhost:9093/).
-
-## Grafana
-
-To access service open following [link](http://localhost:3000).
-
-Default credential:
-
-* login - `admin`
-* password - `admin`
-
-Grafana is provisioned by default with following entities:
-
-* `VictoriaMetrics` datasource
-* `VictoriaMetrics - cluster` datasource
-* `VictoriaMetrics overview` dashboard
-* `VictoriaMetrics - cluster` dashboard
-* `VictoriaMetrics - vmagent` dashboard
-* `VictoriaMetrics - vmalert` dashboard
-
-Remember to pick `VictoriaMetrics - cluster` datasource when viewing `VictoriaMetrics - cluster` dashboard.
-
-Optionally, environment with [VictoriaMetrics Grafana datasource](https://github.com/VictoriaMetrics/victoriametrics-datasource)
-can be started with the following commands:
-```
-make docker-single-vm-datasource-up    # start single server
-make docker-single-vm-datasource-down  # shut down single server
-
-make docker-cluster-vm-datasource-up   # start cluster
-make docker-cluster-vm-datasource-down # shutdown cluster
-```
-
-## Alerts
-
-See below a list of recommended alerting rules for various VictoriaMetrics components for running in production. 
-Some alerting rules thresholds are just recommendations and could require an adjustment. 
-The list of alerting rules is the following:
-* [alerts-health.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml):
-  alerting rules related to all VictoriaMetrics components for tracking their "health" state; 
-* [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts.yml):
-  alerting rules related to [single-server VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) installation;
-* [alerts-cluster.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml):
-  alerting rules related to [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/cluster-victoriametrics/);
-* [alerts-vmagent.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmagent.yml):
-  alerting rules related to [vmagent](https://docs.victoriametrics.com/vmagent/) component;
-* [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml):
-  alerting rules related to [vmalert](https://docs.victoriametrics.com/vmalert/) component;
-* [alerts-vmauth.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml):
-  alerting rules related to [vmauth](https://docs.victoriametrics.com/vmauth/) component;
-* [alerts-vlogs.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vlogs.yml):
-    alerting rules related to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/);
-* [alerts-vmanomaly.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmanomaly.yml):
-    alerting rules related to [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/);
-
-Please, also see [how to monitor](https://docs.victoriametrics.com/single-server-victoriametrics/#monitoring) 
-VictoriaMetrics installations.
+Web interface link is [http://localhost:8429/](http://localhost:8429/).
 
 ## VictoriaLogs server
 
 To spin-up environment with VictoriaLogs run the following command:
 ```
-make docker-victorialogs-up
+make docker-vl-single-up
 ```
 
 VictoriaLogs will be accessible on the `--httpListenAddr=:9428` port.
 In addition to VictoriaLogs server, the docker compose contains the following components:
 * [vector](https://vector.dev/guides/) service for collecting docker logs and sending them to VictoriaLogs;
-* VictoriaMetrics single server to collect metrics from `VictoriaLogs` and `vector`;
-* [grafana](#grafana) is configured with [VictoriaLogs datasource](https://github.com/VictoriaMetrics/victorialogs-datasource).
+* `VictoriaMetrics single-node` to collect metrics from all the components;
+* [Grafana](#grafana) is configured with [VictoriaLogs datasource](https://github.com/VictoriaMetrics/victorialogs-datasource).
+* [vmalert](#vmalert) is configured to query `VictoriaLogs single-node`, and send alerts state
+  and recording rules results to `VictoriaMetrics single-node`;
+* [alertmanager](#alertmanager) is configured to receive notifications from `vmalert`.
+
+<img alt="VictoriaLogs single-server deployment" width="500" src="assets/vl-single-server.png">
 
 To access Grafana use link [http://localhost:3000](http://localhost:3000).
 
 To access [VictoriaLogs UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui)
-use link [http://localhost:9428/select/vmui](http://localhost:9428/select/vmui).
+use link [http://localhost:8427/select/vmui](http://localhost:8427/select/vmui).
 
 Please, also see [how to monitor](https://docs.victoriametrics.com/victorialogs/#monitoring) 
 VictoriaLogs installations.
 
 To shutdown environment execute the following command:
 ```
-make docker-victorialogs-down
+make docker-vl-single-down
+```
+
+## VictoriaLogs cluster
+
+To spin-up environment with VictoriaLogs cluster run the following command:
+```
+make docker-vl-cluster-up
+```
+
+VictoriaLogs cluster environment consists of `vlinsert`, `vlstorage` and `vlselect` components.
+`vlinsert` and `vlselect` are available through `vmauth` on port `:8427`.
+For example, `vector` pushes logs via `http://vmauth:8427/insert/elasticsearch/` path,
+and Grafana queries `http://vmauth:8427` for datasource queries.
+
+The rest of components are available only inside the environment.
+
+In addition to VictoriaLogs cluster, the docker compose contains the following components:
+* [vector](https://vector.dev/guides/) service for collecting docker logs and sending them to `vlinsert`;
+* [Grafana](#grafana) is configured with [VictoriaLogs datasource](https://github.com/VictoriaMetrics/victorialogs-datasource) and pointing to `vmauth`.
+* `VictoriaMetrics single-node` to collect metrics from all the components;
+* `vlinsert` forwards ingested data to `vlstorage`
+* `vlselect`s are connected to `vlstorage` for querying data;
+* [vmauth](#vmauth) balances incoming read and write requests among `vlselect`s and `vlinsert`s;
+* [vmalert](#vmalert) is configured to query `vlselect`s, and send alerts state
+  and recording rules results to `VictoriaMetrics single-node`;
+* [alertmanager](#alertmanager) is configured to receive notifications from `vmalert`.
+
+<img alt="VictoriaLogs cluster deployment" width="500" src="assets/vl-cluster.png">
+
+To access Grafana use link [http://localhost:3000](http://localhost:3000).
+
+To access [VictoriaLogs UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui)
+use link [http://localhost:8427/select/vmui](http://localhost:8427/select/vmui).
+
+Please, also see [how to monitor](https://docs.victoriametrics.com/victorialogs/#monitoring)
+VictoriaLogs installations.
+
+To shutdown environment execute the following command:
+```
+make docker-vl-cluster-down
 ```
 
 Please see more examples on integration of VictoriaLogs with other log shippers below:
@@ -211,3 +184,64 @@ Please see more examples on integration of VictoriaLogs with other log shippers
 * [telegraf](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf)
 * [fluentd](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentd)
 * [datadog-serverless](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/datadog-serverless)
+
+# Common components
+
+## vmauth
+
+[vmauth](https://docs.victoriametrics.com/vmauth/) acts as a [load balancer](https://docs.victoriametrics.com/vmauth/#load-balancing)
+to spread the load across `vmselect`'s or `vlselect`'s. [Grafana](#grafana) and [vmalert](#vmalert) use vmauth for read queries.
+vmauth routes read queries to VictoriaMetrics or VictoriaLogs depending on requested path.
+vmauth config is available here for [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/auth-vm-cluster.yml),
+[VictoriaLogs single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/auth-vl-single.yml),
+[VictoriaLogs cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/auth-vl-cluster.yml).
+
+
+## vmalert
+
+vmalert evaluates various [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules).
+It is connected with AlertManager for firing alerts, and with VictoriaMetrics or VictoriaLogs for executing queries and storing alert's state.
+
+Web interface link [http://localhost:8880/](http://localhost:8880/).
+
+## alertmanager
+
+AlertManager accepts notifications from `vmalert` and fires alerts.
+All notifications are blackholed according to [alertmanager.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/alertmanager.yml) config.
+
+Web interface link [http://localhost:9093/](http://localhost:9093/).
+
+## Grafana
+
+Web interface link [http://localhost:3000](http://localhost:3000).
+
+Default credentials:
+* login: `admin`
+* password: `admin`
+
+Grafana is provisioned with default dashboards and datasources.
+
+## Alerts
+
+See below a list of recommended alerting rules for various VictoriaMetrics components for running in production.
+Some alerting rules thresholds are just recommendations and could require an adjustment.
+The list of alerting rules is the following:
+* [alerts-health.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml):
+  alerting rules related to all VictoriaMetrics components for tracking their "health" state;
+* [alerts.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts.yml):
+  alerting rules related to [single-server VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) installation;
+* [alerts-cluster.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml):
+  alerting rules related to [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/cluster-victoriametrics/);
+* [alerts-vmagent.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmagent.yml):
+  alerting rules related to [vmagent](https://docs.victoriametrics.com/vmagent/) component;
+* [alerts-vmalert.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml):
+  alerting rules related to [vmalert](https://docs.victoriametrics.com/vmalert/) component;
+* [alerts-vmauth.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml):
+  alerting rules related to [vmauth](https://docs.victoriametrics.com/vmauth/) component;
+* [alerts-vlogs.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vlogs.yml):
+  alerting rules related to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/);
+* [alerts-vmanomaly.yml](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmanomaly.yml):
+  alerting rules related to [VictoriaMetrics Anomaly Detection](https://docs.victoriametrics.com/anomaly-detection/);
+
+Please, also see [how to monitor VictoriaMetrics installations](https://docs.victoriametrics.com/single-server-victoriametrics/#monitoring) 
+and [how to monitor VictoriaLogs installations](https://docs.victoriametrics.com/victorialogs/#monitoring).

deployment/docker/auth-mixed-datasource.yml

@@ -1,9 +0,0 @@
-# route requests between VictoriaMetrics and VictoriaLogs
-unauthorized_user:
-  url_map:
-  - src_paths:
-    - "/api/v1/query.*"
-    url_prefix: "http://victoriametrics:8428"
-  - src_paths:
-    - "/select/logsql/.*"
-    url_prefix: "http://victorialogs:9428"

deployment/docker/auth-vl-cluster.yml

@@ -0,0 +1,15 @@
+# route requests between VictoriaMetrics and VictoriaLogs
+unauthorized_user:
+  url_map:
+  - src_paths:
+    - "/api/v1/.*"
+    url_prefix: http://victoriametrics:8428
+  - src_paths:
+    - "/select/.*"
+    url_prefix:
+    - http://vlselect-1:9428
+    - http://vlselect-2:9428
+  - src_paths:
+    - "/insert/.*"
+    url_prefix:
+    - http://vlinsert:9428

deployment/docker/auth-vl-single.yml

@@ -0,0 +1,10 @@
+# route requests between VictoriaMetrics and VictoriaLogs
+unauthorized_user:
+  url_map:
+  - src_paths:
+    - "/api/v1/.*"
+    url_prefix: http://victoriametrics:8428
+  - src_paths:
+    - "/select/.*"
+    url_prefix:
+    - http://victorialogs:9428

deployment/docker/auth-vm-cluster.yml

@@ -1,6 +1,14 @@
 # balance load among vmselects
 # see https://docs.victoriametrics.com/vmauth/#load-balancing
 unauthorized_user:
-  url_prefix:
+  url_map:
+  - src_paths:
+    - "/select/.*"
+    url_prefix:
     - http://vmselect-1:8481
     - http://vmselect-2:8481
+  - src_paths:
+    - "/insert/.*"
+    url_prefix:
+    - http://vminsert-1:8480
+    - http://vminsert-2:8480

deployment/docker/compose-vl-cluster.yml

@@ -0,0 +1,141 @@
+services:
+  # Grafana instance configured with VictoriaLogs as datasource
+  grafana:
+    image: grafana/grafana:11.5.0
+    depends_on:
+      - "victoriametrics"
+      - "vmauth"
+    ports:
+      - 3000:3000
+    volumes:
+      - grafanadata:/var/lib/grafana
+      - ./provisioning/datasources/victoriametrics-logs-datasource/cluster.yml:/etc/grafana/provisioning/datasources/cluster.yml
+      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./provisioning/plugins/:/var/lib/grafana/plugins
+      - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
+      - ./../../dashboards/victorialogs-cluster.json:/var/lib/grafana/dashboards/vl.json
+    environment:
+      - "GF_INSTALL_PLUGINS=victoriametrics-logs-datasource"
+    restart: always
+
+  # vector is logs collector. It collects logs according to vector.yml
+  # and forwards them to VictoriaLogs
+  vector:
+    image: docker.io/timberio/vector:0.46.X-distroless-libc
+    volumes:
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+      - type: bind
+        source: /var/lib/docker
+        target: /var/lib/docker
+      - ./vector-vl-cluster.yml:/etc/vector/vector.yaml:ro
+    depends_on: [vmauth]
+    ports:
+      - "8686:8686"
+    user: root
+
+  vlinsert:
+    image: victoriametrics/victoria-logs:v1.19.0-victorialogs
+    command:
+      - "--storageNode=vlstorage-1:9428"
+      - "--storageNode=vlstorage-2:9428"
+
+  vlselect-1:
+    image: victoriametrics/victoria-logs:v1.19.0-victorialogs
+    command:
+      - "--storageNode=vlstorage-1:9428"
+      - "--storageNode=vlstorage-2:9428"
+  vlselect-2:
+    image: victoriametrics/victoria-logs:v1.19.0-victorialogs
+    command:
+      - "--storageNode=vlstorage-1:9428"
+      - "--storageNode=vlstorage-2:9428"
+
+  vlstorage-1:
+    image: victoriametrics/victoria-logs:v1.19.0-victorialogs
+    command:
+      - "--storageDataPath=/vlogs"
+    volumes:
+      - vldata-1:/vlogs
+  vlstorage-2:
+    image: victoriametrics/victoria-logs:v1.19.0-victorialogs
+    command:
+      - "--storageDataPath=/vlogs"
+    volumes:
+      - vldata-2:/vlogs
+
+  # VictoriaMetrics instance, a single process responsible for
+  # scraping, storing metrics and serve read requests.
+  victoriametrics:
+    image: victoriametrics/victoria-metrics:v1.115.0
+    volumes:
+      - vmdata:/storage
+      - ./prometheus-vl-cluster.yml:/etc/prometheus/prometheus.yml
+    command:
+      - "--storageDataPath=/storage"
+      - "--promscrape.config=/etc/prometheus/prometheus.yml"
+    restart: always
+
+  # vmauth is a router and balancer for HTTP requests.
+  # It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
+  # depending on the requested path.
+  vmauth:
+    image: victoriametrics/vmauth:v1.115.0
+    depends_on:
+      - "victoriametrics"
+      - "vlselect-1"
+      - "vlselect-2"
+      - "vlinsert"
+    volumes:
+      - ./auth-vl-cluster.yml:/etc/auth.yml
+    command:
+      - "--auth.config=/etc/auth.yml"
+    ports:
+      - 8427:8427
+    restart: always
+
+  # vmalert executes alerting and recording rules according to given rule type.
+  vmalert:
+    image: victoriametrics/vmalert:v1.115.0
+    depends_on:
+      - "vmauth"
+      - "alertmanager"
+      - "victoriametrics"
+    ports:
+      - 8880:8880
+    volumes:
+      - ./rules/alerts.yml:/etc/alerts/alerts.yml
+      - ./rules/alerts-vlogs.yml:/etc/alerts/vlogs.yml
+      - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
+      - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
+      - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
+      # vlogs rule
+      - ./rules/vlogs-example-alerts.yml:/etc/alerts/vlogs-example-alerts.yml
+    command:
+      - "--datasource.url=http://vmauth:8427/"
+      - "--remoteRead.url=http://victoriametrics:8428/"
+      - "--remoteWrite.url=http://victoriametrics:8428/"
+      - "--notifier.url=http://alertmanager:9093/"
+      - "--rule=/etc/alerts/*.yml"
+      # display source of alerts in grafana
+      - "--external.url=http://127.0.0.1:3000" #grafana outside container
+    restart: always
+
+  # alertmanager receives alerting notifications from vmalert
+  # and distributes them according to --config.file.
+  alertmanager:
+    image: prom/alertmanager:v0.28.0
+    volumes:
+      - ./alertmanager.yml:/config/alertmanager.yml
+    command:
+      - "--config.file=/config/alertmanager.yml"
+    ports:
+      - 9093:9093
+    restart: always
+
+volumes:
+  vmdata: {}
+  vldata-1: {}
+  vldata-2: {}
+  grafanadata: {}

deployment/docker/compose-vl-single.yml

@@ -1,7 +1,6 @@
 services:
   # Grafana instance configured with VictoriaLogs as datasource
   grafana:
-    container_name: grafana
     image: grafana/grafana:11.5.0
     depends_on:
       - "victoriametrics"
@@ -10,21 +9,19 @@ services:
       - 3000:3000
     volumes:
       - grafanadata:/var/lib/grafana
-      - ./provisioning/datasources/victoriametrics-logs-datasource:/etc/grafana/provisioning/datasources
+      - ./provisioning/datasources/victoriametrics-logs-datasource/single.yml:/etc/grafana/provisioning/datasources/single.yml
       - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
       - ./provisioning/plugins/:/var/lib/grafana/plugins
       - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
       - ./../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json
     environment:
       - "GF_INSTALL_PLUGINS=victoriametrics-logs-datasource"
-    networks:
-      - vm_net
     restart: always
 
-  # vector is logs collector. It collects logs according to vector.yaml
+  # vector is logs collector. It collects logs according to vector.yml
   # and forwards them to VictoriaLogs
   vector:
-    image: docker.io/timberio/vector:0.42.X-distroless-libc
+    image: docker.io/timberio/vector:0.46.X-distroless-libc
     volumes:
       - type: bind
         source: /var/run/docker.sock
@@ -32,70 +29,52 @@ services:
       - type: bind
         source: /var/lib/docker
         target: /var/lib/docker
-      - ./vector.yaml:/etc/vector/vector.yaml:ro
+      - ./vector-vl-single.yml:/etc/vector/vector.yaml:ro
     depends_on: [victorialogs]
     ports:
       - "8686:8686"
     user: root
-    networks:
-      - vm_net
 
   #  VictoriaLogs instance, a single process responsible for
   #  storing logs and serving read queries.
   victorialogs:
-    container_name: victorialogs
-    image: victoriametrics/victoria-logs:v1.17.0-victorialogs
+    image: victoriametrics/victoria-logs:v1.19.0-victorialogs
     command:
       - "--storageDataPath=/vlogs"
-      - "--httpListenAddr=:9428"
     volumes:
       - vldata:/vlogs
-    ports:
-      - "9428:9428"
-    networks:
-      - vm_net
 
   # VictoriaMetrics instance, a single process responsible for
   # scraping, storing metrics and serve read requests.
   victoriametrics:
-    container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.113.0
-    ports:
-      - 8428:8428
+    image: victoriametrics/victoria-metrics:v1.115.0
     volumes:
       - vmdata:/storage
-      - ./prometheus-victorialogs.yml:/etc/prometheus/prometheus.yml
+      - ./prometheus-vl-single.yml:/etc/prometheus/prometheus.yml
     command:
       - "--storageDataPath=/storage"
-      - "--httpListenAddr=:8428"
       - "--promscrape.config=/etc/prometheus/prometheus.yml"
-    networks:
-      - vm_net
     restart: always
 
   # vmauth is a router and balancer for HTTP requests.
   # It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
   # depending on the requested path.
   vmauth:
-    container_name: vmauth
-    image: victoriametrics/vmauth:v1.113.0
+    image: victoriametrics/vmauth:v1.115.0
     depends_on:
       - "victoriametrics"
       - "victorialogs"
     volumes:
-      - ./auth-mixed-datasource.yml:/etc/auth.yml
+      - ./auth-vl-single.yml:/etc/auth.yml
     command:
       - "--auth.config=/etc/auth.yml"
     ports:
       - 8427:8427
-    networks:
-      - vm_net
     restart: always
 
-  # vmalert executes alerting and recording rules according to given rule type.
+  # vmalert executes alerting and recording rules according to the given rule type.
   vmalert:
-    container_name: vmalert
-    image: victoriametrics/vmalert:v1.113.0
+    image: victoriametrics/vmalert:v1.115.0
     depends_on:
       - "vmauth"
       - "alertmanager"
@@ -106,7 +85,6 @@ services:
       - ./rules/alerts.yml:/etc/alerts/alerts.yml
       - ./rules/alerts-vlogs.yml:/etc/alerts/vlogs.yml
       - ./rules/alerts-health.yml:/etc/alerts/alerts-health.yml
-      - ./rules/alerts-vmagent.yml:/etc/alerts/alerts-vmagent.yml
       - ./rules/alerts-vmalert.yml:/etc/alerts/alerts-vmalert.yml
       # vlogs rule
       - ./rules/vlogs-example-alerts.yml:/etc/alerts/vlogs-example-alerts.yml
@@ -118,14 +96,11 @@ services:
       - "--rule=/etc/alerts/*.yml"
       # display source of alerts in grafana
       - "--external.url=http://127.0.0.1:3000" #grafana outside container
-    networks:
-      - vm_net
     restart: always
 
   # alertmanager receives alerting notifications from vmalert
   # and distributes them according to --config.file.
   alertmanager:
-    container_name: alertmanager
     image: prom/alertmanager:v0.28.0
     volumes:
       - ./alertmanager.yml:/config/alertmanager.yml
@@ -133,13 +108,9 @@ services:
       - "--config.file=/config/alertmanager.yml"
     ports:
       - 9093:9093
-    networks:
-      - vm_net
     restart: always
 
 volumes:
   vmdata: {}
   vldata: {}
   grafanadata: {}
-networks:
-  vm_net:

deployment/docker/compose-vm-cluster.yml

@@ -3,23 +3,20 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    container_name: vmagent
     image: victoriametrics/vmagent:v1.115.0
     depends_on:
-      - "vminsert"
+      - "vmauth"
     ports:
       - 8429:8429
     volumes:
       - vmagentdata:/vmagentdata
-      - ./prometheus-cluster.yml:/etc/prometheus/prometheus.yml
+      - ./prometheus-vm-cluster.yml:/etc/prometheus/prometheus.yml
     command:
       - "--promscrape.config=/etc/prometheus/prometheus.yml"
-      - "--remoteWrite.url=http://vminsert:8480/insert/0/prometheus/"
+      - "--remoteWrite.url=http://vmauth:8427/insert/0/prometheus/api/v1/write"
     restart: always
 
-  # Grafana instance configured with VictoriaMetrics as datasource
   grafana:
-    container_name: grafana
     image: grafana/grafana:11.5.0
     depends_on:
       - "vmauth"
@@ -38,24 +35,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    container_name: vmstorage-1
     image: victoriametrics/vmstorage:v1.115.0-cluster
-    ports:
-      - 8482
-      - 8400
-      - 8401
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    container_name: vmstorage-2
     image: victoriametrics/vmstorage:v1.115.0-cluster
-    ports:
-      - 8482
-      - 8400
-      - 8401
     volumes:
       - strgdata-2:/storage
     command:
@@ -64,8 +51,16 @@ services:
 
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
-  vminsert:
-    container_name: vminsert
+  vminsert-1:
+    image: victoriametrics/vminsert:v1.115.0-cluster
+    depends_on:
+      - "vmstorage-1"
+      - "vmstorage-2"
+    command:
+      - "--storageNode=vmstorage-1:8400"
+      - "--storageNode=vmstorage-2:8400"
+    restart: always
+  vminsert-2:
     image: victoriametrics/vminsert:v1.115.0-cluster
     depends_on:
       - "vmstorage-1"
@@ -73,14 +68,11 @@ services:
     command:
       - "--storageNode=vmstorage-1:8400"
       - "--storageNode=vmstorage-2:8400"
-    ports:
-      - 8480:8480
     restart: always
 
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    container_name: vmselect-1
     image: victoriametrics/vmselect:v1.115.0-cluster
     depends_on:
       - "vmstorage-1"
@@ -89,11 +81,8 @@ services:
       - "--storageNode=vmstorage-1:8401"
       - "--storageNode=vmstorage-2:8401"
       - "--vmalert.proxyURL=http://vmalert:8880"
-    ports:
-      - 8481
     restart: always
   vmselect-2:
-    container_name: vmselect-2
     image: victoriametrics/vmselect:v1.115.0-cluster
     depends_on:
       - "vmstorage-1"
@@ -102,8 +91,6 @@ services:
       - "--storageNode=vmstorage-1:8401"
       - "--storageNode=vmstorage-2:8401"
       - "--vmalert.proxyURL=http://vmalert:8880"
-    ports:
-      - 8481
     restart: always
 
   # vmauth is a router and balancer for HTTP requests.
@@ -111,13 +98,12 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    container_name: vmauth
     image: victoriametrics/vmauth:v1.115.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
     volumes:
-      - ./auth-cluster.yml:/etc/auth.yml
+      - ./auth-vm-cluster.yml:/etc/auth.yml
     command:
       - "--auth.config=/etc/auth.yml"
     ports:
@@ -126,7 +112,6 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    container_name: vmalert
     image: victoriametrics/vmalert:v1.115.0
     depends_on:
       - "vmauth"
@@ -140,7 +125,7 @@ services:
     command:
       - "--datasource.url=http://vmauth:8427/select/0/prometheus"
       - "--remoteRead.url=http://vmauth:8427/select/0/prometheus"
-      - "--remoteWrite.url=http://vminsert:8480/insert/0/prometheus"
+      - "--remoteWrite.url=http://vmauth:8427/insert/0/prometheus/api/v1/write"
       - "--notifier.url=http://alertmanager:9093/"
       - "--rule=/etc/alerts/*.yml"
       # display source of alerts in grafana
@@ -151,7 +136,6 @@ services:
   # alertmanager receives alerting notifications from vmalert
   # and distributes them according to --config.file.
   alertmanager:
-    container_name: alertmanager
     image: prom/alertmanager:v0.28.0
     volumes:
       - ./alertmanager.yml:/config/alertmanager.yml

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,6 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    container_name: vmagent
     image: victoriametrics/vmagent:v1.115.0
     depends_on:
       - "victoriametrics"
@@ -11,7 +10,7 @@ services:
       - 8429:8429
     volumes:
       - vmagentdata:/vmagentdata
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./prometheus-vm-single.yml:/etc/prometheus/prometheus.yml
     command:
       - "--promscrape.config=/etc/prometheus/prometheus.yml"
       - "--remoteWrite.url=http://victoriametrics:8428/api/v1/write"
@@ -19,7 +18,6 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    container_name: victoriametrics
     image: victoriametrics/victoria-metrics:v1.115.0
     ports:
       - 8428:8428
@@ -39,9 +37,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
 
-  # Grafana instance configured with VictoriaMetrics as datasource
   grafana:
-    container_name: grafana
     image: grafana/grafana:11.5.0
     depends_on:
       - "victoriametrics"
@@ -58,7 +54,6 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    container_name: vmalert
     image: victoriametrics/vmalert:v1.115.0
     depends_on:
       - "victoriametrics"
@@ -84,7 +79,6 @@ services:
   # alertmanager receives alerting notifications from vmalert
   # and distributes them according to --config.file.
   alertmanager:
-    container_name: alertmanager
     image: prom/alertmanager:v0.28.0
     volumes:
       - ./alertmanager.yml:/config/alertmanager.yml

deployment/docker/prometheus-cluster.yml

@@ -1,22 +0,0 @@
-global:
-  scrape_interval: 10s
-
-scrape_configs:
-  - job_name: 'vmagent'
-    static_configs:
-      - targets: ['vmagent:8429']
-  - job_name: 'vmauth'
-    static_configs:
-      - targets: ['vmauth:8427']
-  - job_name: 'vmalert'
-    static_configs:
-      - targets: ['vmalert:8880']
-  - job_name: 'vminsert'
-    static_configs:
-      - targets: ['vminsert:8480']
-  - job_name: 'vmselect'
-    static_configs:
-      - targets: ['vmselect-1:8481', 'vmselect-2:8481']
-  - job_name: 'vmstorage'
-    static_configs:
-      - targets: ['vmstorage-1:8482', 'vmstorage-2:8482']

deployment/docker/prometheus-victorialogs.yml

@@ -1,16 +0,0 @@
-global:
-  scrape_interval: 10s
-
-scrape_configs:
-  - job_name: 'victoriametrics'
-    static_configs:
-      - targets: ['victoriametrics:8428']
-  - job_name: 'vmalert'
-    static_configs:
-      - targets: [ 'vmalert:8880' ]
-  - job_name: 'victorialogs'
-    static_configs:
-      - targets: ['victorialogs:9428']
-  - job_name: 'fluentbit'
-    static_configs:
-      - targets: ['fluentbit:2020/api/v1/metrics/prometheus']

deployment/docker/prometheus-vl-cluster.yml

@@ -0,0 +1,26 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+- job_name: victoriametrics
+  static_configs:
+  - targets:
+    - victoriametrics:8428
+- job_name: vmalert
+  static_configs:
+  - targets:
+    - vmalert:8880
+- job_name: vlstorage
+  static_configs:
+  - targets:
+    - vlstorage-1:9428
+    - vlstorage-2:9428
+- job_name: vlselect
+  static_configs:
+  - targets:
+    - vlselect-1:9428
+    - vlselect-2:9428
+- job_name: vlinsert
+  static_configs:
+  - targets:
+    - vlinsert:9428

deployment/docker/prometheus-vl-single.yml

@@ -0,0 +1,16 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+- job_name: victoriametrics
+  static_configs:
+  - targets:
+    - victoriametrics:8428
+- job_name: vmalert
+  static_configs:
+  - targets:
+    - vmalert:8880
+- job_name: victorialogs
+  static_configs:
+  - targets:
+    - victorialogs:9428

deployment/docker/prometheus-vm-cluster.yml

@@ -0,0 +1,30 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+- job_name: vmagent
+  static_configs:
+  - targets:
+    - vmagent:8429
+- job_name: vmauth
+  static_configs:
+  - targets:
+    - vmauth:8427
+- job_name: vmalert
+  static_configs:
+  - targets:
+    - vmalert:8880
+- job_name: vminsert
+  static_configs:
+  - targets:
+    - vminsert:8480
+- job_name: vmselect
+  static_configs:
+  - targets:
+    - vmselect-1:8481
+    - vmselect-2:8481
+- job_name: vmstorage
+  static_configs:
+  - targets:
+    - vmstorage-1:8482
+    - vmstorage-2:8482

deployment/docker/prometheus-vm-single.yml

@@ -0,0 +1,16 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+- job_name: vmagent
+  static_configs:
+  - targets:
+    - vmagent:8429
+- job_name: vmalert
+  static_configs:
+  - targets:
+    - vmalert:8880
+- job_name: victoriametrics
+  static_configs:
+  - targets:
+    - victoriametrics:8428