lib/promrelabel/debug: make metric name easier to distinguish from labels

Previously, users could get confused in cases when metric name which is valid for MetricsQL but not valid for Prom was used in relabeling debug. See #8584 for example.

This adds highlight to a metric name and prints empty labels set in order to explicitly show metric name and set of labels.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>

.github/pull_request_template.md

@@ -6,4 +6,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 
 The following checks are **mandatory**:
 
-- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
+- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).

.gitignore

@@ -27,4 +27,3 @@ _site
 coverage.txt
 cspell.json
 *~
-deployment/docker/provisioning/plugins/

Makefile

@@ -504,7 +504,7 @@ fmt:
 	gofmt -l -w -s ./apptest
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
+	go vet ./lib/...
 	go vet ./app/...
 	go vet ./apptest/...
 
@@ -513,29 +513,29 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test: victoria-metrics vmagent vmalert vmauth
 	go test ./apptest/... -skip="^TestCluster.*"
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
+	go test -bench=. ./lib/...
 	go test -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
@@ -564,7 +564,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run
 
 install-golangci-lint:
 	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.7

app/vlinsert/insertutil/common_params.go

@@ -219,42 +219,6 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 	}
 }
 
-// InsertRowProcessor is used by native data ingestion protocol parser.
-type InsertRowProcessor interface {
-	// AddInsertRow must add r to the underlying storage.
-	AddInsertRow(r *logstorage.InsertRow)
-}
-
-// AddInsertRow adds r to lmp.
-func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
-	lmp.rowsIngestedTotal.Inc()
-	n := logstorage.EstimatedJSONRowLen(r.Fields)
-	lmp.bytesIngestedTotal.Add(n)
-
-	if len(r.Fields) > *MaxFieldsPerLine {
-		line := logstorage.MarshalFieldsToJSON(nil, r.Fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(r.Fields), *MaxFieldsPerLine, line)
-		rowsDroppedTotalTooManyFields.Inc()
-		return
-	}
-
-	lmp.mu.Lock()
-	defer lmp.mu.Unlock()
-
-	lmp.lr.MustAddInsertRow(r)
-
-	if lmp.cp.Debug {
-		s := lmp.lr.GetRowString(0)
-		lmp.lr.ResetKeepSettings()
-		logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` arg: %s", lmp.cp.DebugRemoteAddr, lmp.cp.DebugRequestURI, s)
-		rowsDroppedTotalDebug.Inc()
-		return
-	}
-	if lmp.lr.NeedFlush() {
-		lmp.flushLocked()
-	}
-}
-
 // flushLocked must be called under locked lmp.mu.
 func (lmp *logMessageProcessor) flushLocked() {
 	lmp.lastFlushTime = time.Now()

app/vlinsert/internalinsert/internalinsert.go

@@ -1,96 +0,0 @@
-package internalinsert
-
-import (
-	"flag"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-)
-
-var (
-	disableInsert  = flag.Bool("internalinsert.disable", false, "Whether to disable /internal/insert HTTP endpoint")
-	maxRequestSize = flagutil.NewBytes("internalinsert.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single request, which can be accepted at /internal/insert HTTP endpoint")
-)
-
-// RequestHandler processes /internal/insert requests.
-func RequestHandler(w http.ResponseWriter, r *http.Request) {
-	if *disableInsert {
-		httpserver.Errorf(w, r, "requests to /internal/insert are disabled with -internalinsert.disable command-line flag")
-		return
-	}
-
-	startTime := time.Now()
-	if r.Method != "POST" {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-	version := r.FormValue("version")
-	if version != netinsert.ProtocolVersion {
-		httpserver.Errorf(w, r, "unsupported protocol version=%q; want %q", version, netinsert.ProtocolVersion)
-		return
-	}
-
-	requestsTotal.Inc()
-
-	cp, err := insertutil.GetCommonParams(r)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	if err := vlstorage.CanWriteData(); err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-
-	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("internalinsert", false)
-		irp := lmp.(insertutil.InsertRowProcessor)
-		err := parseData(irp, data)
-		lmp.MustClose()
-		return err
-	})
-	if err != nil {
-		errorsTotal.Inc()
-		httpserver.Errorf(w, r, "cannot parse internal insert request: %s", err)
-		return
-	}
-
-	requestDuration.UpdateDuration(startTime)
-}
-
-func parseData(irp insertutil.InsertRowProcessor, data []byte) error {
-	r := logstorage.GetInsertRow()
-	src := data
-	i := 0
-	for len(src) > 0 {
-		tail, err := r.UnmarshalInplace(src)
-		if err != nil {
-			return fmt.Errorf("cannot parse row #%d: %s", i, err)
-		}
-		src = tail
-		i++
-
-		irp.AddInsertRow(r)
-	}
-	logstorage.PutInsertRow(r)
-
-	return nil
-}
-
-var (
-	requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
-	errorsTotal   = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
-
-	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
-)

app/vlinsert/main.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/elasticsearch"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/internalinsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/journald"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/jsonline"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki"
@@ -29,11 +28,6 @@ func Stop() {
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 
-	if path == "/internal/insert" {
-		internalinsert.RequestHandler(w, r)
-		return true
-	}
-
 	if !strings.HasPrefix(path, "/insert/") {
 		// Skip requests, which do not start with /insert/, since these aren't our requests.
 		return false

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -41,26 +41,6 @@ func TestPushProtoOk(t *testing.T) {
 		`{"_msg":"log-line-message","severity":"Trace"}`,
 	)
 
-	// severities mapping
-	f([]pb.ResourceLogs{
-		{
-			ScopeLogs: []pb.ScopeLogs{
-				{
-					LogRecords: []pb.LogRecord{
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 13, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 24, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-					},
-				},
-			},
-		},
-	},
-		[]int64{1234, 1234, 1234},
-		`{"_msg":"log-line-message","severity":"Trace"}
-{"_msg":"log-line-message","severity":"Warn"}
-{"_msg":"log-line-message","severity":"Fatal4"}`,
-	)
-
 	// multi-line with resource attributes
 	f([]pb.ResourceLogs{
 		{
@@ -80,7 +60,7 @@ func TestPushProtoOk(t *testing.T) {
 				{
 					LogRecords: []pb.LogRecord{
 						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1234, SeverityNumber: 1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message")}},
-						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 25, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
+						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1235, SeverityNumber: 21, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
 						{Attributes: []*pb.KeyValue{}, TimeUnixNano: 1236, SeverityNumber: -1, Body: pb.AnyValue{StringValue: ptrTo("log-line-message-msg-2")}},
 					},
 				},

app/vlselect/internalselect/internalselect.go

@@ -1,324 +0,0 @@
-package internalselect
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"net/http"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/metrics"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-)
-
-var disableSelect = flag.Bool("internalselect.disable", false, "Whether to disable /internal/select/* HTTP endpoints")
-
-// RequestHandler processes requests to /internal/select/*
-func RequestHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	if *disableSelect {
-		httpserver.Errorf(w, r, "requests to /internal/select/* are disabled with -internalselect.disable command-line flag")
-		return
-	}
-
-	startTime := time.Now()
-
-	path := r.URL.Path
-	rh := requestHandlers[path]
-	if rh == nil {
-		httpserver.Errorf(w, r, "unsupported endpoint requested: %s", path)
-		return
-	}
-
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_requests_total{path=%q}`, path)).Inc()
-	if err := rh(ctx, w, r); err != nil && !netutil.IsTrivialNetworkError(err) {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vl_http_request_errors_total{path=%q}`, path)).Inc()
-		httpserver.Errorf(w, r, "%s", err)
-		// The return is skipped intentionally in order to track the duration of failed queries.
-	}
-	metrics.GetOrCreateSummary(fmt.Sprintf(`vl_http_request_duration_seconds{path=%q}`, path)).UpdateDuration(startTime)
-}
-
-var requestHandlers = map[string]func(ctx context.Context, w http.ResponseWriter, r *http.Request) error{
-	"/internal/select/query":               processQueryRequest,
-	"/internal/select/field_names":         processFieldNamesRequest,
-	"/internal/select/field_values":        processFieldValuesRequest,
-	"/internal/select/stream_field_names":  processStreamFieldNamesRequest,
-	"/internal/select/stream_field_values": processStreamFieldValuesRequest,
-	"/internal/select/streams":             processStreamsRequest,
-	"/internal/select/stream_ids":          processStreamIDsRequest,
-}
-
-func processQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.QueryProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	var wLock sync.Mutex
-	var dataLenBuf []byte
-
-	sendBuf := func(bb *bytesutil.ByteBuffer) error {
-		if len(bb.B) == 0 {
-			return nil
-		}
-
-		data := bb.B
-		if !cp.DisableCompression {
-			bufLen := len(bb.B)
-			bb.B = zstd.CompressLevel(bb.B, bb.B, 1)
-			data = bb.B[bufLen:]
-		}
-
-		wLock.Lock()
-		dataLenBuf = encoding.MarshalUint64(dataLenBuf[:0], uint64(len(data)))
-		_, err := w.Write(dataLenBuf)
-		if err == nil {
-			_, err = w.Write(data)
-		}
-		wLock.Unlock()
-
-		// Reset the sent buf
-		bb.Reset()
-
-		return err
-	}
-
-	var bufs atomicutil.Slice[bytesutil.ByteBuffer]
-
-	var errGlobalLock sync.Mutex
-	var errGlobal error
-
-	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
-		if errGlobal != nil {
-			return
-		}
-
-		bb := bufs.Get(workerID)
-
-		bb.B = db.Marshal(bb.B)
-
-		if len(bb.B) < 1024*1024 {
-			// Fast path - the bb is too small to be sent to the client yet.
-			return
-		}
-
-		// Slow path - the bb must be sent to the client.
-		if err := sendBuf(bb); err != nil {
-			errGlobalLock.Lock()
-			if errGlobal != nil {
-				errGlobal = err
-			}
-			errGlobalLock.Unlock()
-		}
-	}
-
-	if err := vlstorage.RunQuery(ctx, cp.TenantIDs, cp.Query, writeBlock); err != nil {
-		return err
-	}
-	if errGlobal != nil {
-		return errGlobal
-	}
-
-	// Send the remaining data
-	for _, bb := range bufs.GetSlice() {
-		if err := sendBuf(bb); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func processFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.FieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamFieldNamesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldNamesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldNames, err := vlstorage.GetStreamFieldNames(ctx, cp.TenantIDs, cp.Query)
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field names: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldNames, cp.DisableCompression)
-}
-
-func processStreamFieldValuesRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamFieldValuesProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	fieldName := r.FormValue("field")
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	fieldValues, err := vlstorage.GetStreamFieldValues(ctx, cp.TenantIDs, cp.Query, fieldName, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain stream field values: %w", err)
-	}
-
-	return writeValuesWithHits(w, fieldValues, cp.DisableCompression)
-}
-
-func processStreamsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streams, err := vlstorage.GetStreams(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streams, cp.DisableCompression)
-}
-
-func processStreamIDsRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
-	cp, err := getCommonParams(r, netselect.StreamIDsProtocolVersion)
-	if err != nil {
-		return err
-	}
-
-	limit, err := getInt64FromRequest(r, "limit")
-	if err != nil {
-		return err
-	}
-
-	streamIDs, err := vlstorage.GetStreamIDs(ctx, cp.TenantIDs, cp.Query, uint64(limit))
-	if err != nil {
-		return fmt.Errorf("cannot obtain streams: %w", err)
-	}
-
-	return writeValuesWithHits(w, streamIDs, cp.DisableCompression)
-}
-
-type commonParams struct {
-	TenantIDs []logstorage.TenantID
-	Query     *logstorage.Query
-
-	DisableCompression bool
-}
-
-func getCommonParams(r *http.Request, expectedProtocolVersion string) (*commonParams, error) {
-	version := r.FormValue("version")
-	if version != expectedProtocolVersion {
-		return nil, fmt.Errorf("unexpected version=%q; want %q", version, expectedProtocolVersion)
-	}
-
-	tenantIDsStr := r.FormValue("tenant_ids")
-	tenantIDs, err := logstorage.UnmarshalTenantIDs([]byte(tenantIDsStr))
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal tenant_ids=%q: %w", tenantIDsStr, err)
-	}
-
-	timestamp, err := getInt64FromRequest(r, "timestamp")
-	if err != nil {
-		return nil, err
-	}
-
-	qStr := r.FormValue("query")
-	q, err := logstorage.ParseQueryAtTimestamp(qStr, timestamp)
-	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal query=%q: %w", qStr, err)
-	}
-
-	s := r.FormValue("disable_compression")
-	disableCompression, err := strconv.ParseBool(s)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse disable_compression=%q: %w", s, err)
-	}
-
-	cp := &commonParams{
-		TenantIDs: tenantIDs,
-		Query:     q,
-
-		DisableCompression: disableCompression,
-	}
-	return cp, nil
-}
-
-func writeValuesWithHits(w http.ResponseWriter, vhs []logstorage.ValueWithHits, disableCompression bool) error {
-	var b []byte
-	for i := range vhs {
-		b = vhs[i].Marshal(b)
-	}
-
-	if !disableCompression {
-		b = zstd.CompressLevel(nil, b, 1)
-	}
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-
-	if _, err := w.Write(b); err != nil {
-		return fmt.Errorf("cannot send response to the client: %w", err)
-	}
-
-	return nil
-}
-
-func getInt64FromRequest(r *http.Request, argName string) (int64, error) {
-	s := r.FormValue(argName)
-	n, err := strconv.ParseInt(s, 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("cannot parse %s=%q: %w", argName, s, err)
-	}
-	return n, nil
-}

app/vlselect/logsql/buffered_writer.go

@@ -0,0 +1,47 @@
+package logsql
+
+import (
+	"bufio"
+	"io"
+	"sync"
+)
+
+func getBufferedWriter(w io.Writer) *bufferedWriter {
+	v := bufferedWriterPool.Get()
+	if v == nil {
+		return &bufferedWriter{
+			bw: bufio.NewWriter(w),
+		}
+	}
+	bw := v.(*bufferedWriter)
+	bw.bw.Reset(w)
+	return bw
+}
+
+func putBufferedWriter(bw *bufferedWriter) {
+	bw.reset()
+	bufferedWriterPool.Put(bw)
+}
+
+var bufferedWriterPool sync.Pool
+
+type bufferedWriter struct {
+	mu sync.Mutex
+	bw *bufio.Writer
+}
+
+func (bw *bufferedWriter) reset() {
+	// nothing to do
+}
+
+func (bw *bufferedWriter) WriteIgnoreErrors(p []byte) {
+	bw.mu.Lock()
+	_, _ = bw.bw.Write(p)
+	bw.mu.Unlock()
+}
+
+func (bw *bufferedWriter) FlushIgnoreErrors() {
+	bw.mu.Lock()
+	_ = bw.bw.Flush()
+	bw.mu.Unlock()
+}

app/vlselect/logsql/logsql.go

@@ -3,7 +3,6 @@ package logsql
 import (
 	"context"
 	"fmt"
-	"io"
 	"math"
 	"net/http"
 	"regexp"
@@ -12,14 +11,12 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/valyala/fastjson"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
@@ -60,13 +57,10 @@ func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
 
 	var mLock sync.Mutex
 	m := make(map[string][]facetEntry)
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		if rowsCount == 0 {
+	writeBlock := func(_ uint, _ []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
 			return
 		}
-
-		columns := db.Columns
 		if len(columns) != 3 {
 			logger.Panicf("BUG: expecting 3 columns; got %d columns", len(columns))
 		}
@@ -162,19 +156,17 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		if rowsCount == 0 {
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
 			return
 		}
 
-		columns := db.Columns
 		timestampValues := columns[0].Values
 		hitsValues := columns[len(columns)-1].Values
 		columns = columns[1 : len(columns)-1]
 
 		bb := blockResultPool.Get()
-		for i := 0; i < rowsCount; i++ {
+		for i := range timestamps {
 			timestampStr := strings.Clone(timestampValues[i])
 			hitsStr := strings.Clone(hitsValues[i])
 			hits, err := strconv.ParseUint(hitsStr, 10, 64)
@@ -213,8 +205,6 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	WriteHitsSeries(w, m)
 }
 
-var blockResultPool bytesutil.ByteBufferPool
-
 func getTopHitsSeries(m map[string]*hitsSeries, fieldsLimit int) map[string]*hitsSeries {
 	if fieldsLimit <= 0 || fieldsLimit >= len(m) {
 		return m
@@ -546,7 +536,7 @@ var liveTailRequests = metrics.NewCounter(`vl_live_tailing_requests`)
 const tailOffsetNsecs = 5e9
 
 type logRow struct {
-	timestamp string
+	timestamp int64
 	fields    []logstorage.Field
 }
 
@@ -562,7 +552,7 @@ type tailProcessor struct {
 	mu sync.Mutex
 
 	perStreamRows  map[string][]logRow
-	lastTimestamps map[string]string
+	lastTimestamps map[string]int64
 
 	err error
 }
@@ -572,12 +562,12 @@ func newTailProcessor(cancel func()) *tailProcessor {
 		cancel: cancel,
 
 		perStreamRows:  make(map[string][]logRow),
-		lastTimestamps: make(map[string]string),
+		lastTimestamps: make(map[string]int64),
 	}
 }
 
-func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
-	if db.RowsCount() == 0 {
+func (tp *tailProcessor) writeBlock(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+	if len(timestamps) == 0 {
 		return
 	}
 
@@ -589,8 +579,14 @@ func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
 	}
 
 	// Make sure columns contain _time field, since it is needed for proper tail work.
-	timestamps, ok := db.GetTimestamps()
-	if !ok {
+	hasTime := false
+	for _, c := range columns {
+		if c.Name == "_time" {
+			hasTime = true
+			break
+		}
+	}
+	if !hasTime {
 		tp.err = fmt.Errorf("missing _time field")
 		tp.cancel()
 		return
@@ -599,8 +595,8 @@ func (tp *tailProcessor) writeBlock(_ uint, db *logstorage.DataBlock) {
 	// Copy block rows to tp.perStreamRows
 	for i, timestamp := range timestamps {
 		streamID := ""
-		fields := make([]logstorage.Field, len(db.Columns))
-		for j, c := range db.Columns {
+		fields := make([]logstorage.Field, len(columns))
+		for j, c := range columns {
 			name := strings.Clone(c.Name)
 			value := strings.Clone(c.Values[i])
 
@@ -692,15 +688,12 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	m := make(map[string]*statsSeries)
 	var mLock sync.Mutex
 
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-
-		columns := db.Columns
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := 0; i < rowsCount; i++ {
+		for i := range timestamps {
 			// Do not move q.GetTimestamp() outside writeBlock, since ts
 			// must be initialized to query timestamp for every processed log row.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8312
@@ -809,14 +802,12 @@ func ProcessStatsQueryRequest(ctx context.Context, w http.ResponseWriter, r *htt
 	var rowsLock sync.Mutex
 
 	timestamp := q.GetTimestamp()
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		columns := db.Columns
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
-		for i := 0; i < rowsCount; i++ {
+		for i := range timestamps {
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if slices.Contains(byFields, c.Name) {
@@ -878,21 +869,11 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	sw := &syncWriter{
-		w: w,
-	}
-
-	var bwShards atomicutil.Slice[bufferedWriter]
-	bwShards.Init = func(shard *bufferedWriter) {
-		shard.sw = sw
-	}
+	bw := getBufferedWriter(w)
 	defer func() {
-		shards := bwShards.GetSlice()
-		for _, shard := range shards {
-			shard.FlushIgnoreErrors()
-		}
+		bw.FlushIgnoreErrors()
+		putBufferedWriter(bw)
 	}()
-
 	w.Header().Set("Content-Type", "application/stream+json")
 
 	if limit > 0 {
@@ -902,34 +883,32 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 				httpserver.Errorf(w, r, "%s", err)
 				return
 			}
-			bw := bwShards.Get(0)
+			bb := blockResultPool.Get()
+			b := bb.B
 			for i := range rows {
-				bw.buf = logstorage.MarshalFieldsToJSON(bw.buf, rows[i].fields)
-				bw.buf = append(bw.buf, '\n')
-				if len(bw.buf) > 16*1024 {
-					bw.FlushIgnoreErrors()
-				}
+				b = logstorage.MarshalFieldsToJSON(b[:0], rows[i].fields)
+				b = append(b, '\n')
+				bw.WriteIgnoreErrors(b)
 			}
+			bb.B = b
+			blockResultPool.Put(bb)
 			return
 		}
 
 		q.AddPipeLimit(uint64(limit))
 	}
 
-	writeBlock := func(workerID uint, db *logstorage.DataBlock) {
-		rowsCount := db.RowsCount()
-		if rowsCount == 0 {
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
+		if len(columns) == 0 || len(columns[0].Values) == 0 {
 			return
 		}
-		columns := db.Columns
 
-		bw := bwShards.Get(workerID)
-		for i := 0; i < rowsCount; i++ {
-			WriteJSONRow(bw, columns, i)
-			if len(bw.buf) > 16*1024 {
-				bw.FlushIgnoreErrors()
-			}
+		bb := blockResultPool.Get()
+		for i := range timestamps {
+			WriteJSONRow(bb, columns, i)
 		}
+		bw.WriteIgnoreErrors(bb.B)
+		blockResultPool.Put(bb)
 	}
 
 	if err := vlstorage.RunQuery(ctx, tenantIDs, q, writeBlock); err != nil {
@@ -938,37 +917,14 @@ func ProcessQueryRequest(ctx context.Context, w http.ResponseWriter, r *http.Req
 	}
 }
 
-type syncWriter struct {
-	mu sync.Mutex
-	w  io.Writer
+var blockResultPool bytesutil.ByteBufferPool
+
+type row struct {
+	timestamp int64
+	fields    []logstorage.Field
 }
 
-func (sw *syncWriter) Write(p []byte) (int, error) {
-	sw.mu.Lock()
-	n, err := sw.w.Write(p)
-	sw.mu.Unlock()
-	return n, err
-}
-
-type bufferedWriter struct {
-	buf []byte
-	sw  *syncWriter
-}
-
-func (bw *bufferedWriter) Write(p []byte) (int, error) {
-	bw.buf = append(bw.buf, p...)
-
-	// Do not send bw.buf to bw.sw here, since the data at bw.buf may be incomplete (it must end with '\n')
-
-	return len(p), nil
-}
-
-func (bw *bufferedWriter) FlushIgnoreErrors() {
-	_, _ = bw.sw.Write(bw.buf)
-	bw.buf = bw.buf[:0]
-}
-
-func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
+func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
 	limitUpper := 2 * limit
 	q.AddPipeLimit(uint64(limitUpper))
 
@@ -1037,7 +993,7 @@ func getLastNQueryResults(ctx context.Context, tenantIDs []logstorage.TenantID,
 	}
 }
 
-func getLastNRows(rows []logRow, limit int) []logRow {
+func getLastNRows(rows []row, limit int) []row {
 	sort.Slice(rows, func(i, j int) bool {
 		return rows[i].timestamp < rows[j].timestamp
 	})
@@ -1047,31 +1003,18 @@ func getLastNRows(rows []logRow, limit int) []logRow {
 	return rows
 }
 
-func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]logRow, error) {
+func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit int) ([]row, error) {
 	ctxWithCancel, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	var missingTimeColumn atomic.Bool
-	var rows []logRow
+	var rows []row
 	var rowsLock sync.Mutex
-	writeBlock := func(_ uint, db *logstorage.DataBlock) {
-		if missingTimeColumn.Load() {
-			return
-		}
-
-		columns := db.Columns
+	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
 
-		timestamps, ok := db.GetTimestamps()
-		if !ok {
-			missingTimeColumn.Store(true)
-			cancel()
-			return
-		}
-
 		for i, timestamp := range timestamps {
 			fields := make([]logstorage.Field, len(columns))
 			for j := range columns {
@@ -1081,7 +1024,7 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			}
 
 			rowsLock.Lock()
-			rows = append(rows, logRow{
+			rows = append(rows, row{
 				timestamp: timestamp,
 				fields:    fields,
 			})
@@ -1092,13 +1035,11 @@ func getQueryResultsWithLimit(ctx context.Context, tenantIDs []logstorage.Tenant
 			cancel()
 		}
 	}
-	err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock)
-
-	if missingTimeColumn.Load() {
-		return nil, fmt.Errorf("missing _time column in the result for the query [%s]", q)
+	if err := vlstorage.RunQuery(ctxWithCancel, tenantIDs, q, writeBlock); err != nil {
+		return nil, err
 	}
 
-	return rows, err
+	return rows, nil
 }
 
 func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID, error) {

app/vlselect/main.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/internalselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect/logsql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -73,7 +72,7 @@ var vmuiFileServer = http.FileServer(http.FS(vmuiFiles))
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 
-	if !strings.HasPrefix(path, "/select/") && !strings.HasPrefix(path, "/internal/select/") {
+	if !strings.HasPrefix(path, "/select/") {
 		// Skip requests, which do not start with /select/, since these aren't our requests.
 		return false
 	}
@@ -100,17 +99,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	ctx := r.Context()
-	if path == "/select/logsql/tail" {
-		logsqlTailRequests.Inc()
-		// Process live tailing request without timeout, since it is OK to run live tailing requests for very long time.
-		// Also do not apply concurrency limit to tail requests, since these limits are intended for non-tail requests.
-		logsql.ProcessLiveTailRequest(ctx, w, r)
-		return true
-	}
-
 	// Limit the number of concurrent queries, which can consume big amounts of CPU time.
 	startTime := time.Now()
+	ctx := r.Context()
 	d := getMaxQueryDuration(r)
 	ctxWithTimeout, cancel := context.WithTimeout(ctx, d)
 	defer cancel()
@@ -120,10 +111,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 	defer decRequestConcurrency()
 
-	if strings.HasPrefix(path, "/internal/select/") {
-		// Process internal request from vlselect without timeout (e.g. use ctx instead of ctxWithTimeout),
-		// since the timeout must be controlled by the vlselect.
-		internalselect.RequestHandler(ctx, w, r)
+	if path == "/select/logsql/tail" {
+		logsqlTailRequests.Inc()
+		// Process live tailing request without timeout (e.g. use ctx instead of ctxWithTimeout),
+		// since it is OK to run live tailing requests for very long time.
+		logsql.ProcessLiveTailRequest(ctx, w, r)
 		return true
 	}
 
@@ -132,17 +124,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return false
 	}
 
-	logRequestErrorIfNeeded(ctxWithTimeout, w, r, startTime)
-	return true
-}
-
-func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http.Request, startTime time.Time) {
-	err := ctx.Err()
+	err := ctxWithTimeout.Err()
 	switch err {
 	case nil:
 		// nothing to do
 	case context.Canceled:
-		// do not log canceled requests, since they are expected and legal.
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Infof("client has canceled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+			time.Since(startTime).Seconds(), remoteAddr, requestURI)
 	case context.DeadlineExceeded:
 		err = &httpserver.ErrorWithStatusCode{
 			Err: fmt.Errorf("the request couldn't be executed in %.3f seconds; possible solutions: "+
@@ -153,6 +143,8 @@ func logRequestErrorIfNeeded(ctx context.Context, w http.ResponseWriter, r *http
 	default:
 		httpserver.Errorf(w, r, "unexpected error: %s", err)
 	}
+
+	return true
 }
 
 func incRequestConcurrency(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

app/vlselect/vmui/index.html

@@ -35,10 +35,10 @@
   <meta property="og:title" content="UI for VictoriaLogs">
   <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
   <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-C1OmTur2.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-BSp13qCn.js">
+  <script type="module" crossorigin src="./assets/index-BgdvCSTM.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-DoEyHMxg.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-u4IOGr0E.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vlstorage/main.go

@@ -10,14 +10,11 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
 var (
@@ -42,55 +39,14 @@ var (
 		"the storage stops accepting new data")
 
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
-
-	storageNodeAddrs = flagutil.NewArrayString("storageNode", "Comma-separated list of TCP addresses for storage nodes to route the ingested logs to and to send select queries to. "+
-		"If the list is empty, then the ingested logs are stored and queried locally from -storageDataPath")
-	insertConcurrency        = flag.Int("insert.concurrency", 2, "The average number of concurrent data ingestion requests, which can be sent to every -storageNode")
-	insertDisableCompression = flag.Bool("insert.disableCompression", false, "Whether to disable compression when sending the ingested data to -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-	selectDisableCompression = flag.Bool("select.disableCompression", false, "Whether to disable compression for select query responses received from -storageNode nodes. "+
-		"Disabled compression reduces CPU usage at the cost of higher network usage")
-
-	storageNodeUsername     = flagutil.NewArrayString("storageNode.username", "Optional basic auth username to use for the corresponding -storageNode")
-	storageNodePassword     = flagutil.NewArrayString("storageNode.password", "Optional basic auth password to use for the corresponding -storageNode")
-	storageNodePasswordFile = flagutil.NewArrayString("storageNode.passwordFile", "Optional path to basic auth password to use for the corresponding -storageNode. "+
-		"The file is re-read every second")
-	storageNodeBearerToken     = flagutil.NewArrayString("storageNode.bearerToken", "Optional bearer auth token to use for the corresponding -storageNode")
-	storageNodeBearerTokenFile = flagutil.NewArrayString("storageNode.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -storageNode. "+
-		"The token is re-read from the file every second")
-
-	storageNodeTLS = flagutil.NewArrayBool("storageNode.tls", "Whether to use TLS (HTTPS) protocol for communicating with the corresponding -storageNode. "+
-		"By default communication is performed via HTTP")
-	storageNodeTLSCAFile = flagutil.NewArrayString("storageNode.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -storageNode. "+
-		"By default, system CA is used")
-	storageNodeTLSCertFile = flagutil.NewArrayString("storageNode.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
-		"to the corresponding -storageNode")
-	storageNodeTLSKeyFile    = flagutil.NewArrayString("storageNode.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -storageNode")
-	storageNodeTLSServerName = flagutil.NewArrayString("storageNode.tlsServerName", "Optional TLS server name to use for connections to the corresponding -storageNode. "+
-		"By default, the server name from -storageNode is used")
-	storageNodeTLSInsecureSkipVerify = flagutil.NewArrayBool("storageNode.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -storageNode")
 )
 
-var localStorage *logstorage.Storage
-var localStorageMetrics *metrics.Set
-
-var netstorageInsert *netinsert.Storage
-var netstorageSelect *netselect.Storage
-
 // Init initializes vlstorage.
 //
 // Stop must be called when vlstorage is no longer needed
 func Init() {
-	if len(*storageNodeAddrs) == 0 {
-		initLocalStorage()
-	} else {
-		initNetworkStorage()
-	}
-}
-
-func initLocalStorage() {
-	if localStorage != nil {
-		logger.Panicf("BUG: initLocalStorage() has been already called")
+	if strg != nil {
+		logger.Panicf("BUG: Init() has been already called")
 	}
 
 	if retentionPeriod.Duration() < 24*time.Hour {
@@ -107,139 +63,60 @@ func initLocalStorage() {
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
-	localStorage = logstorage.MustOpenStorage(*storageDataPath, cfg)
+	strg = logstorage.MustOpenStorage(*storageDataPath, cfg)
 
 	var ss logstorage.StorageStats
-	localStorage.UpdateStats(&ss)
+	strg.UpdateStats(&ss)
 	logger.Infof("successfully opened storage in %.3f seconds; smallParts: %d; bigParts: %d; smallPartBlocks: %d; bigPartBlocks: %d; smallPartRows: %d; bigPartRows: %d; "+
 		"smallPartSize: %d bytes; bigPartSize: %d bytes",
 		time.Since(startTime).Seconds(), ss.SmallParts, ss.BigParts, ss.SmallPartBlocks, ss.BigPartBlocks, ss.SmallPartRowsCount, ss.BigPartRowsCount,
 		ss.CompressedSmallPartSize, ss.CompressedBigPartSize)
 
-	// register local storage metrics
-	localStorageMetrics = metrics.NewSet()
-	localStorageMetrics.RegisterMetricsWriter(func(w io.Writer) {
-		writeStorageMetrics(w, localStorage)
+	// register storage metrics
+	storageMetrics = metrics.NewSet()
+	storageMetrics.RegisterMetricsWriter(func(w io.Writer) {
+		writeStorageMetrics(w, strg)
 	})
-	metrics.RegisterSet(localStorageMetrics)
-}
-
-func initNetworkStorage() {
-	if netstorageInsert != nil || netstorageSelect != nil {
-		logger.Panicf("BUG: initNetworkStorage() has been already called")
-	}
-
-	authCfgs := make([]*promauth.Config, len(*storageNodeAddrs))
-	isTLSs := make([]bool, len(*storageNodeAddrs))
-	for i := range authCfgs {
-		authCfgs[i] = newAuthConfigForStorageNode(i)
-		isTLSs[i] = storageNodeTLS.GetOptionalArg(i)
-	}
-
-	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
-	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
-
-	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
-	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
-
-	logger.Infof("initialized all the network services")
-}
-
-func newAuthConfigForStorageNode(argIdx int) *promauth.Config {
-	username := storageNodeUsername.GetOptionalArg(argIdx)
-	password := storageNodePassword.GetOptionalArg(argIdx)
-	passwordFile := storageNodePasswordFile.GetOptionalArg(argIdx)
-	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
-		basicAuthCfg = &promauth.BasicAuthConfig{
-			Username:     username,
-			Password:     promauth.NewSecret(password),
-			PasswordFile: passwordFile,
-		}
-	}
-
-	token := storageNodeBearerToken.GetOptionalArg(argIdx)
-	tokenFile := storageNodeBearerTokenFile.GetOptionalArg(argIdx)
-
-	tlsCfg := &promauth.TLSConfig{
-		CAFile:             storageNodeTLSCAFile.GetOptionalArg(argIdx),
-		CertFile:           storageNodeTLSCertFile.GetOptionalArg(argIdx),
-		KeyFile:            storageNodeTLSKeyFile.GetOptionalArg(argIdx),
-		ServerName:         storageNodeTLSServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: storageNodeTLSInsecureSkipVerify.GetOptionalArg(argIdx),
-	}
-
-	opts := &promauth.Options{
-		BasicAuth:       basicAuthCfg,
-		BearerToken:     token,
-		BearerTokenFile: tokenFile,
-		TLSConfig:       tlsCfg,
-	}
-	ac, err := opts.NewConfig()
-	if err != nil {
-		logger.Panicf("FATAL: cannot populate auth config for storage node #%d: %s", argIdx, err)
-	}
-
-	return ac
+	metrics.RegisterSet(storageMetrics)
 }
 
 // Stop stops vlstorage.
 func Stop() {
-	if localStorage != nil {
-		metrics.UnregisterSet(localStorageMetrics, true)
-		localStorageMetrics = nil
+	metrics.UnregisterSet(storageMetrics, true)
+	storageMetrics = nil
 
-		localStorage.MustClose()
-		localStorage = nil
-	} else {
-		netstorageInsert.MustStop()
-		netstorageInsert = nil
-
-		netstorageSelect.MustStop()
-		netstorageSelect = nil
-	}
+	strg.MustClose()
+	strg = nil
 }
 
 // RequestHandler is a storage request handler.
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 	if path == "/internal/force_merge" {
-		return processForceMerge(w, r)
+		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
+			return true
+		}
+		// Run force merge in background
+		partitionNamePrefix := r.FormValue("partition_prefix")
+		go func() {
+			activeForceMerges.Inc()
+			defer activeForceMerges.Dec()
+			logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
+			startTime := time.Now()
+			strg.MustForceMerge(partitionNamePrefix)
+			logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
+		}()
+		return true
 	}
 	return false
 }
 
-func processForceMerge(w http.ResponseWriter, r *http.Request) bool {
-	if localStorage == nil {
-		// Force merge isn't supported by non-local storage
-		return false
-	}
+var strg *logstorage.Storage
+var storageMetrics *metrics.Set
 
-	if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
-		return true
-	}
-
-	// Run force merge in background
-	partitionNamePrefix := r.FormValue("partition_prefix")
-	go func() {
-		activeForceMerges.Inc()
-		defer activeForceMerges.Dec()
-		logger.Infof("forced merge for partition_prefix=%q has been started", partitionNamePrefix)
-		startTime := time.Now()
-		localStorage.MustForceMerge(partitionNamePrefix)
-		logger.Infof("forced merge for partition_prefix=%q has been successfully finished in %.3f seconds", partitionNamePrefix, time.Since(startTime).Seconds())
-	}()
-	return true
-}
-
-// CanWriteData returns non-nil error if it cannot write data to vlstorage
+// CanWriteData returns non-nil error if it cannot write data to vlstorage.
 func CanWriteData() error {
-	if localStorage == nil {
-		// The data can be always written in non-local mode.
-		return nil
-	}
-
-	if localStorage.IsReadOnly() {
+	if strg.IsReadOnly() {
 		return &httpserver.ErrorWithStatusCode{
 			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
 				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
@@ -253,77 +130,50 @@ func CanWriteData() error {
 //
 // It is advised to call CanWriteData() before calling MustAddRows()
 func MustAddRows(lr *logstorage.LogRows) {
-	if localStorage != nil {
-		// Store lr in the local storage.
-		localStorage.MustAddRows(lr)
-	} else {
-		// Store lr across the remote storage nodes.
-		lr.ForEachRow(netstorageInsert.AddRow)
-	}
+	strg.MustAddRows(lr)
 }
 
 // RunQuery runs the given q and calls writeBlock for the returned data blocks
-func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	if localStorage != nil {
-		return localStorage.RunQuery(ctx, tenantIDs, q, writeBlock)
-	}
-	return netstorageSelect.RunQuery(ctx, tenantIDs, q, writeBlock)
+func RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteBlockFunc) error {
+	return strg.RunQuery(ctx, tenantIDs, q, writeBlock)
 }
 
 // GetFieldNames executes q and returns field names seen in results.
 func GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetFieldNames(ctx, tenantIDs, q)
+	return strg.GetFieldNames(ctx, tenantIDs, q)
 }
 
 // GetFieldValues executes q and returns unique values for the fieldName seen in results.
 //
 // If limit > 0, then up to limit unique values are returned.
 func GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	return strg.GetFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreamFieldNames executes q and returns stream field names seen in results.
 func GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldNames(ctx, tenantIDs, q)
-	}
-	return netstorageSelect.GetStreamFieldNames(ctx, tenantIDs, q)
+	return strg.GetStreamFieldNames(ctx, tenantIDs, q)
 }
 
 // GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
 //
 // If limit > 0, then up to limit unique stream field values are returned.
 func GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	}
-	return netstorageSelect.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
+	return strg.GetStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
 }
 
 // GetStreams executes q and returns streams seen in query results.
 //
 // If limit > 0, then up to limit unique streams are returned.
 func GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreams(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreams(ctx, tenantIDs, q, limit)
+	return strg.GetStreams(ctx, tenantIDs, q, limit)
 }
 
 // GetStreamIDs executes q and returns streamIDs seen in query results.
 //
 // If limit > 0, then up to limit unique streamIDs are returned.
 func GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	if localStorage != nil {
-		return localStorage.GetStreamIDs(ctx, tenantIDs, q, limit)
-	}
-	return netstorageSelect.GetStreamIDs(ctx, tenantIDs, q, limit)
+	return strg.GetStreamIDs(ctx, tenantIDs, q, limit)
 }
 
 func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {

app/vlstorage/netinsert/netinsert.go

@@ -1,369 +0,0 @@
-package netinsert
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/valyala/fastrand"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
-)
-
-// the maximum size of a single data block sent to storage node.
-const maxInsertBlockSize = 2 * 1024 * 1024
-
-// ProtocolVersion is the version of the data ingestion protocol.
-//
-// It must be changed every time the data encoding at /internal/insert HTTP endpoint is changed.
-const ProtocolVersion = "v1"
-
-// Storage is a network storage for sending data to remote storage nodes in the cluster.
-type Storage struct {
-	sns []*storageNode
-
-	disableCompression bool
-
-	srt *streamRowsTracker
-
-	pendingDataBuffers chan *bytesutil.ByteBuffer
-
-	stopCh chan struct{}
-	wg     sync.WaitGroup
-}
-
-type storageNode struct {
-	// scheme is http or https scheme to communicate with addr
-	scheme string
-
-	// addr is TCP address of storage node to send the ingested data to
-	addr string
-
-	// s is a storage, which holds the given storageNode
-	s *Storage
-
-	// c is an http client used for sending data blocks to addr.
-	c *http.Client
-
-	// ac is auth config used for setting request headers such as Authorization and Host.
-	ac *promauth.Config
-
-	// pendingData contains pending data, which must be sent to the storage node at the addr.
-	pendingDataMu        sync.Mutex
-	pendingData          *bytesutil.ByteBuffer
-	pendingDataLastFlush time.Time
-
-	// the unix timestamp until the storageNode is disabled for data writing.
-	disabledUntil atomic.Uint64
-}
-
-func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
-	tr := httputil.NewTransport(false, "vlinsert_backend")
-	tr.TLSHandshakeTimeout = 20 * time.Second
-	tr.DisableCompression = true
-
-	scheme := "http"
-	if isTLS {
-		scheme = "https"
-	}
-
-	sn := &storageNode{
-		scheme: scheme,
-		addr:   addr,
-		s:      s,
-		c: &http.Client{
-			Transport: ac.NewRoundTripper(tr),
-		},
-		ac: ac,
-
-		pendingData: &bytesutil.ByteBuffer{},
-	}
-
-	s.wg.Add(1)
-	go func() {
-		defer s.wg.Done()
-		sn.backgroundFlusher()
-	}()
-
-	return sn
-}
-
-func (sn *storageNode) backgroundFlusher() {
-	t := time.NewTicker(time.Second)
-	defer t.Stop()
-
-	for {
-		select {
-		case <-sn.s.stopCh:
-			return
-		case <-t.C:
-			sn.flushPendingData()
-		}
-	}
-}
-
-func (sn *storageNode) flushPendingData() {
-	sn.pendingDataMu.Lock()
-	if time.Since(sn.pendingDataLastFlush) < time.Second {
-		// nothing to flush
-		sn.pendingDataMu.Unlock()
-		return
-	}
-
-	pendingData := sn.grabPendingDataForFlushLocked()
-	sn.pendingDataMu.Unlock()
-
-	sn.mustSendInsertRequest(pendingData)
-}
-
-func (sn *storageNode) addRow(r *logstorage.InsertRow) {
-	bb := bbPool.Get()
-	b := bb.B
-
-	b = r.Marshal(b)
-
-	if len(b) > maxInsertBlockSize {
-		logger.Warnf("skipping too long log entry, since its length exceeds %d bytes; the actual log entry length is %d bytes; log entry contents: %s", maxInsertBlockSize, len(b), b)
-		return
-	}
-
-	var pendingData *bytesutil.ByteBuffer
-	sn.pendingDataMu.Lock()
-	if sn.pendingData.Len()+len(b) > maxInsertBlockSize {
-		pendingData = sn.grabPendingDataForFlushLocked()
-	}
-	sn.pendingData.MustWrite(b)
-	sn.pendingDataMu.Unlock()
-
-	bb.B = b
-	bbPool.Put(bb)
-
-	if pendingData != nil {
-		sn.mustSendInsertRequest(pendingData)
-	}
-}
-
-var bbPool bytesutil.ByteBufferPool
-
-func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
-	sn.pendingDataLastFlush = time.Now()
-	pendingData := sn.pendingData
-	sn.pendingData = <-sn.s.pendingDataBuffers
-
-	return pendingData
-}
-
-func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
-	defer func() {
-		pendingData.Reset()
-		sn.s.pendingDataBuffers <- pendingData
-	}()
-
-	err := sn.sendInsertRequest(pendingData)
-	if err == nil {
-		return
-	}
-
-	if !errors.Is(err, errTemporarilyDisabled) {
-		logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
-	}
-	for !sn.s.sendInsertRequestToAnyNode(pendingData) {
-		logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
-
-		t := timerpool.Get(time.Second)
-		select {
-		case <-sn.s.stopCh:
-			timerpool.Put(t)
-			logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
-			return
-		case <-t.C:
-			timerpool.Put(t)
-		}
-	}
-}
-
-func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
-	dataLen := pendingData.Len()
-	if dataLen == 0 {
-		// Nothing to send.
-		return nil
-	}
-
-	if sn.disabledUntil.Load() > fasttime.UnixTimestamp() {
-		return errTemporarilyDisabled
-	}
-
-	ctx, cancel := contextutil.NewStopChanContext(sn.s.stopCh)
-	defer cancel()
-
-	var body io.Reader
-	if !sn.s.disableCompression {
-		bb := zstdBufPool.Get()
-		defer zstdBufPool.Put(bb)
-
-		bb.B = zstd.CompressLevel(bb.B[:0], pendingData.B, 1)
-		body = bb.NewReader()
-	} else {
-		body = pendingData.NewReader()
-	}
-
-	reqURL := sn.getRequestURL("/internal/insert")
-	req, err := http.NewRequestWithContext(ctx, "POST", reqURL, body)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating an http request: %s", err)
-	}
-	req.Header.Set("Content-Type", "application/octet-stream")
-	if !sn.s.disableCompression {
-		req.Header.Set("Content-Encoding", "zstd")
-	}
-	if err := sn.ac.SetHeaders(req, true); err != nil {
-		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
-	}
-
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		// Disable sn for data writing for 10 seconds.
-		sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-		return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 == 2 {
-		return nil
-	}
-
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		respBody = []byte(fmt.Sprintf("%s", err))
-	}
-
-	// Disable sn for data writing for 10 seconds.
-	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
-	return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
-}
-
-func (sn *storageNode) getRequestURL(path string) string {
-	return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
-}
-
-var zstdBufPool bytesutil.ByteBufferPool
-
-// NewStorage returns new Storage for the given addrs with the given authCfgs.
-//
-// The concurrency is the average number of concurrent connections per every addr.
-//
-// If disableCompression is set, then the data is sent uncompressed to the remote storage.
-//
-// Call MustStop on the returned storage when it is no longer needed.
-func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
-	pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
-	for i := 0; i < cap(pendingDataBuffers); i++ {
-		pendingDataBuffers <- &bytesutil.ByteBuffer{}
-	}
-
-	s := &Storage{
-		disableCompression: disableCompression,
-		pendingDataBuffers: pendingDataBuffers,
-		stopCh:             make(chan struct{}),
-	}
-
-	sns := make([]*storageNode, len(addrs))
-	for i, addr := range addrs {
-		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
-	}
-	s.sns = sns
-
-	s.srt = newStreamRowsTracker(len(sns))
-
-	return s
-}
-
-// MustStop stops the s.
-func (s *Storage) MustStop() {
-	close(s.stopCh)
-	s.wg.Wait()
-	s.sns = nil
-}
-
-// AddRow adds the given log row into s.
-func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
-	idx := s.srt.getNodeIdx(streamHash)
-	sn := s.sns[idx]
-	sn.addRow(r)
-}
-
-func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
-	startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
-	for i := range s.sns {
-		idx := (startIdx + i) % len(s.sns)
-		sn := s.sns[idx]
-		err := sn.sendInsertRequest(pendingData)
-		if err == nil {
-			return true
-		}
-		if !errors.Is(err, errTemporarilyDisabled) {
-			logger.Warnf("cannot send pending data to the storage node %q: %s; trying to send it to another storage node", sn.addr, err)
-		}
-	}
-	return false
-}
-
-var errTemporarilyDisabled = fmt.Errorf("writing to the node is temporarily disabled")
-
-type streamRowsTracker struct {
-	mu sync.Mutex
-
-	nodesCount    int64
-	rowsPerStream map[uint64]uint64
-}
-
-func newStreamRowsTracker(nodesCount int) *streamRowsTracker {
-	return &streamRowsTracker{
-		nodesCount:    int64(nodesCount),
-		rowsPerStream: make(map[uint64]uint64),
-	}
-}
-
-func (srt *streamRowsTracker) getNodeIdx(streamHash uint64) uint64 {
-	if srt.nodesCount == 1 {
-		// Fast path for a single node.
-		return 0
-	}
-
-	srt.mu.Lock()
-	defer srt.mu.Unlock()
-
-	streamRows := srt.rowsPerStream[streamHash] + 1
-	srt.rowsPerStream[streamHash] = streamRows
-
-	if streamRows <= 1000 {
-		// Write the initial rows for the stream to a single storage node for better locality.
-		// This should work great for log streams containing small number of logs, since will be distributed
-		// evenly among available storage nodes because they have different streamHash.
-		return streamHash % uint64(srt.nodesCount)
-	}
-
-	// The log stream contains more than 1000 rows. Distribute them among storage nodes at random
-	// in order to improve query performance over this stream (the data for the log stream
-	// can be processed in parallel on all the storage nodes).
-	//
-	// The random distribution is preferred over round-robin distribution in order to avoid possible
-	// dependency between the order of the ingested logs and the number of storage nodes,
-	// which may lead to non-uniform distribution of logs among storage nodes.
-	return uint64(fastrand.Uint32n(uint32(srt.nodesCount)))
-}

app/vlstorage/netinsert/netinsert_test.go

@@ -1,57 +0,0 @@
-package netinsert
-
-import (
-	"fmt"
-	"math"
-	"math/rand"
-	"testing"
-
-	"github.com/cespare/xxhash/v2"
-)
-
-func TestStreamRowsTracker(t *testing.T) {
-	f := func(rowsCount, streamsCount, nodesCount int) {
-		t.Helper()
-
-		// generate stream hashes
-		streamHashes := make([]uint64, streamsCount)
-		for i := range streamHashes {
-			streamHashes[i] = xxhash.Sum64([]byte(fmt.Sprintf("stream %d.", i)))
-		}
-
-		srt := newStreamRowsTracker(nodesCount)
-
-		rng := rand.New(rand.NewSource(0))
-		rowsPerNode := make([]uint64, nodesCount)
-		for i := 0; i < rowsCount; i++ {
-			streamIdx := rng.Intn(streamsCount)
-			h := streamHashes[streamIdx]
-			nodeIdx := srt.getNodeIdx(h)
-			rowsPerNode[nodeIdx]++
-		}
-
-		// Verify that rows are uniformly distributed among nodes.
-		expectedRowsPerNode := float64(rowsCount) / float64(nodesCount)
-		for nodeIdx, nodeRows := range rowsPerNode {
-			if math.Abs(float64(nodeRows)-expectedRowsPerNode)/expectedRowsPerNode > 0.15 {
-				t.Fatalf("non-uniform distribution of rows among nodes; node %d has %d rows, while it must have %v rows; rowsPerNode=%d",
-					nodeIdx, nodeRows, expectedRowsPerNode, rowsPerNode)
-			}
-		}
-	}
-
-	rowsCount := 10000
-	streamsCount := 9
-	nodesCount := 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 10000
-	streamsCount = 100
-	nodesCount = 2
-	f(rowsCount, streamsCount, nodesCount)
-
-	rowsCount = 100000
-	streamsCount = 1000
-	nodesCount = 9
-	f(rowsCount, streamsCount, nodesCount)
-}

app/vlstorage/netselect/netselect.go

@@ -1,469 +0,0 @@
-package netselect
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"net/url"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/contextutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
-)
-
-const (
-	// FieldNamesProtocolVersion is the version of the protocol used for /internal/select/field_names HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	FieldNamesProtocolVersion = "v1"
-
-	// FieldValuesProtocolVersion is the version of the protocol used for /internal/select/field_values HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	FieldValuesProtocolVersion = "v1"
-
-	// StreamFieldNamesProtocolVersion is the version of the protocol used for /internal/select/stream_field_names HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamFieldNamesProtocolVersion = "v1"
-
-	// StreamFieldValuesProtocolVersion is the version of the protocol used for /internal/select/stream_field_values HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamFieldValuesProtocolVersion = "v1"
-
-	// StreamsProtocolVersion is the version of the protocol used for /internal/select/streams HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamsProtocolVersion = "v1"
-
-	// StreamIDsProtocolVersion is the version of the protocol used for /internal/select/stream_ids HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	StreamIDsProtocolVersion = "v1"
-
-	// QueryProtocolVersion is the version of the protocol used for /internal/select/query HTTP endpoint.
-	//
-	// It must be updated every time the protocol changes.
-	QueryProtocolVersion = "v1"
-)
-
-// Storage is a network storage for querying remote storage nodes in the cluster.
-type Storage struct {
-	sns []*storageNode
-
-	disableCompression bool
-}
-
-type storageNode struct {
-	// scheme is http or https scheme to communicate with addr
-	scheme string
-
-	// addr is TCP address of the storage node to query
-	addr string
-
-	// s is a storage, which holds the given storageNode
-	s *Storage
-
-	// c is an http client used for querying storage node at addr.
-	c *http.Client
-
-	// ac is auth config used for setting request headers such as Authorization and Host.
-	ac *promauth.Config
-}
-
-func newStorageNode(s *Storage, addr string, ac *promauth.Config, isTLS bool) *storageNode {
-	tr := httputil.NewTransport(false, "vlselect_backend")
-	tr.TLSHandshakeTimeout = 20 * time.Second
-	tr.DisableCompression = true
-
-	scheme := "http"
-	if isTLS {
-		scheme = "https"
-	}
-
-	sn := &storageNode{
-		scheme: scheme,
-		addr:   addr,
-		s:      s,
-		c: &http.Client{
-			Transport: ac.NewRoundTripper(tr),
-		},
-		ac: ac,
-	}
-	return sn
-}
-
-func (sn *storageNode) runQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, processBlock func(db *logstorage.DataBlock)) error {
-	args := sn.getCommonArgs(QueryProtocolVersion, tenantIDs, q)
-
-	reqURL := sn.getRequestURL("/internal/select/query", args)
-	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
-	}
-	if err := sn.ac.SetHeaders(req, true); err != nil {
-		return fmt.Errorf("cannot set auth headers for %q: %w", reqURL, err)
-	}
-
-	// send the request to the storage node
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		responseBody, err := io.ReadAll(resp.Body)
-		if err != nil {
-			responseBody = []byte(err.Error())
-		}
-		return fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
-	}
-
-	// read the response
-	var dataLenBuf [8]byte
-	var buf []byte
-	var db logstorage.DataBlock
-	var valuesBuf []string
-	for {
-		if _, err := io.ReadFull(resp.Body, dataLenBuf[:]); err != nil {
-			if errors.Is(err, io.EOF) {
-				// The end of response stream
-				return nil
-			}
-			return fmt.Errorf("cannot read block size from %q: %w", reqURL, err)
-		}
-		blockLen := encoding.UnmarshalUint64(dataLenBuf[:])
-		if blockLen > math.MaxInt {
-			return fmt.Errorf("too big data block: %d bytes; mustn't exceed %v bytes", blockLen, math.MaxInt)
-		}
-
-		buf = slicesutil.SetLength(buf, int(blockLen))
-		if _, err := io.ReadFull(resp.Body, buf); err != nil {
-			return fmt.Errorf("cannot read block with size of %d bytes from %q: %w", blockLen, reqURL, err)
-		}
-
-		src := buf
-		if !sn.s.disableCompression {
-			bufLen := len(buf)
-			var err error
-			buf, err = zstd.Decompress(buf, buf)
-			if err != nil {
-				return fmt.Errorf("cannot decompress data block: %w", err)
-			}
-			src = buf[bufLen:]
-		}
-
-		for len(src) > 0 {
-			tail, vb, err := db.UnmarshalInplace(src, valuesBuf[:0])
-			if err != nil {
-				return fmt.Errorf("cannot unmarshal data block received from %q: %w", reqURL, err)
-			}
-			valuesBuf = vb
-			src = tail
-
-			processBlock(&db)
-
-			clear(valuesBuf)
-		}
-	}
-}
-
-func (sn *storageNode) getFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(FieldNamesProtocolVersion, tenantIDs, q)
-
-	return sn.getValuesWithHits(ctx, "/internal/select/field_names", args)
-}
-
-func (sn *storageNode) getFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(FieldValuesProtocolVersion, tenantIDs, q)
-	args.Set("field", fieldName)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/field_values", args)
-}
-
-func (sn *storageNode) getStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamFieldNamesProtocolVersion, tenantIDs, q)
-
-	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_names", args)
-}
-
-func (sn *storageNode) getStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamFieldValuesProtocolVersion, tenantIDs, q)
-	args.Set("field", fieldName)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/stream_field_values", args)
-}
-
-func (sn *storageNode) getStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamsProtocolVersion, tenantIDs, q)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/streams", args)
-}
-
-func (sn *storageNode) getStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	args := sn.getCommonArgs(StreamIDsProtocolVersion, tenantIDs, q)
-	args.Set("limit", fmt.Sprintf("%d", limit))
-
-	return sn.getValuesWithHits(ctx, "/internal/select/stream_ids", args)
-}
-
-func (sn *storageNode) getCommonArgs(version string, tenantIDs []logstorage.TenantID, q *logstorage.Query) url.Values {
-	args := url.Values{}
-	args.Set("version", version)
-	args.Set("tenant_ids", string(logstorage.MarshalTenantIDs(nil, tenantIDs)))
-	args.Set("query", q.String())
-	args.Set("timestamp", fmt.Sprintf("%d", q.GetTimestamp()))
-	args.Set("disable_compression", fmt.Sprintf("%v", sn.s.disableCompression))
-	return args
-}
-
-func (sn *storageNode) getValuesWithHits(ctx context.Context, path string, args url.Values) ([]logstorage.ValueWithHits, error) {
-	data, err := sn.executeRequestAt(ctx, path, args)
-	if err != nil {
-		return nil, err
-	}
-	return unmarshalValuesWithHits(data)
-}
-
-func (sn *storageNode) executeRequestAt(ctx context.Context, path string, args url.Values) ([]byte, error) {
-	reqURL := sn.getRequestURL(path, args)
-	req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
-	if err != nil {
-		logger.Panicf("BUG: unexpected error when creating a request: %s", err)
-	}
-
-	// send the request to the storage node
-	resp, err := sn.c.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		responseBody, err := io.ReadAll(resp.Body)
-		if err != nil {
-			responseBody = []byte(err.Error())
-		}
-		return nil, fmt.Errorf("unexpected status code for the request to %q: %d; want %d; response: %q", reqURL, resp.StatusCode, http.StatusOK, responseBody)
-	}
-
-	// read the response
-	var bb bytesutil.ByteBuffer
-	if _, err := bb.ReadFrom(resp.Body); err != nil {
-		return nil, fmt.Errorf("cannot read response from %q: %w", reqURL, err)
-	}
-
-	if sn.s.disableCompression {
-		return bb.B, nil
-	}
-
-	bbLen := len(bb.B)
-	bb.B, err = zstd.Decompress(bb.B, bb.B)
-	if err != nil {
-		return nil, err
-	}
-	return bb.B[bbLen:], nil
-}
-
-func (sn *storageNode) getRequestURL(path string, args url.Values) string {
-	return fmt.Sprintf("%s://%s%s?%s", sn.scheme, sn.addr, path, args.Encode())
-}
-
-// NewStorage returns new Storage for the given addrs and the given authCfgs.
-//
-// If disableCompression is set, then uncompressed responses are received from storage nodes.
-//
-// Call MustStop on the returned storage when it is no longer needed.
-func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, disableCompression bool) *Storage {
-	s := &Storage{
-		disableCompression: disableCompression,
-	}
-
-	sns := make([]*storageNode, len(addrs))
-	for i, addr := range addrs {
-		sns[i] = newStorageNode(s, addr, authCfgs[i], isTLSs[i])
-	}
-	s.sns = sns
-
-	return s
-}
-
-// MustStop stops the s.
-func (s *Storage) MustStop() {
-	s.sns = nil
-}
-
-// RunQuery runs the given q and calls writeBlock for the returned data blocks
-func (s *Storage) RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	nqr, err := logstorage.NewNetQueryRunner(ctx, tenantIDs, q, s.RunQuery, writeBlock)
-	if err != nil {
-		return err
-	}
-
-	search := func(stopCh <-chan struct{}, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-		return s.runQuery(stopCh, tenantIDs, q, writeBlock)
-	}
-
-	concurrency := q.GetConcurrency()
-	return nqr.Run(ctx, concurrency, search)
-}
-
-func (s *Storage) runQuery(stopCh <-chan struct{}, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
-	ctxWithCancel, cancel := contextutil.NewStopChanContext(stopCh)
-	defer cancel()
-
-	errs := make([]error, len(s.sns))
-
-	var wg sync.WaitGroup
-	for i := range s.sns {
-		wg.Add(1)
-		go func(nodeIdx int) {
-			defer wg.Done()
-			sn := s.sns[nodeIdx]
-			err := sn.runQuery(ctxWithCancel, tenantIDs, q, func(db *logstorage.DataBlock) {
-				writeBlock(uint(nodeIdx), db)
-			})
-			if err != nil {
-				// Cancel the remaining parallel queries
-				cancel()
-			}
-
-			errs[nodeIdx] = err
-		}(i)
-	}
-	wg.Wait()
-
-	return getFirstNonCancelError(errs)
-}
-
-// GetFieldNames executes q and returns field names seen in results.
-func (s *Storage) GetFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getFieldNames(ctx, tenantIDs, q)
-	})
-}
-
-// GetFieldValues executes q and returns unique values for the fieldName seen in results.
-//
-// If limit > 0, then up to limit unique values are returned.
-func (s *Storage) GetFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	})
-}
-
-// GetStreamFieldNames executes q and returns stream field names seen in results.
-func (s *Storage) GetStreamFieldNames(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, 0, false, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreamFieldNames(ctx, tenantIDs, q)
-	})
-}
-
-// GetStreamFieldValues executes q and returns stream field values for the given fieldName seen in results.
-//
-// If limit > 0, then up to limit unique stream field values are returned.
-func (s *Storage) GetStreamFieldValues(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, fieldName string, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreamFieldValues(ctx, tenantIDs, q, fieldName, limit)
-	})
-}
-
-// GetStreams executes q and returns streams seen in query results.
-//
-// If limit > 0, then up to limit unique streams are returned.
-func (s *Storage) GetStreams(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreams(ctx, tenantIDs, q, limit)
-	})
-}
-
-// GetStreamIDs executes q and returns streamIDs seen in query results.
-//
-// If limit > 0, then up to limit unique streamIDs are returned.
-func (s *Storage) GetStreamIDs(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, limit uint64) ([]logstorage.ValueWithHits, error) {
-	return s.getValuesWithHits(ctx, limit, true, func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error) {
-		return sn.getStreamIDs(ctx, tenantIDs, q, limit)
-	})
-}
-
-func (s *Storage) getValuesWithHits(ctx context.Context, limit uint64, resetHitsOnLimitExceeded bool,
-	callback func(ctx context.Context, sn *storageNode) ([]logstorage.ValueWithHits, error)) ([]logstorage.ValueWithHits, error) {
-
-	ctxWithCancel, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	results := make([][]logstorage.ValueWithHits, len(s.sns))
-	errs := make([]error, len(s.sns))
-
-	var wg sync.WaitGroup
-	for i := range s.sns {
-		wg.Add(1)
-		go func(nodeIdx int) {
-			defer wg.Done()
-
-			sn := s.sns[nodeIdx]
-			vhs, err := callback(ctxWithCancel, sn)
-			results[nodeIdx] = vhs
-			errs[nodeIdx] = err
-
-			if err != nil {
-				// Cancel the remaining parallel requests
-				cancel()
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	if err := getFirstNonCancelError(errs); err != nil {
-		return nil, err
-	}
-
-	vhs := logstorage.MergeValuesWithHits(results, limit, resetHitsOnLimitExceeded)
-
-	return vhs, nil
-}
-
-func getFirstNonCancelError(errs []error) error {
-	for _, err := range errs {
-		if err != nil && !errors.Is(err, context.Canceled) {
-			return err
-		}
-	}
-	return nil
-}
-
-func unmarshalValuesWithHits(src []byte) ([]logstorage.ValueWithHits, error) {
-	var vhs []logstorage.ValueWithHits
-	for len(src) > 0 {
-		var vh logstorage.ValueWithHits
-		tail, err := vh.UnmarshalInplace(src)
-		if err != nil {
-			return nil, fmt.Errorf("cannot unmarshal ValueWithHits #%d: %w", len(vhs), err)
-		}
-		src = tail
-
-		// Clone vh.Value, since it points to src.
-		vh.Value = strings.Clone(vh.Value)
-
-		vhs = append(vhs, vh)
-	}
-
-	return vhs, nil
-}

app/vmagent/main.go

@@ -105,7 +105,7 @@ func main() {
 	// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
 	// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
 	// exceeds 0.05 for extended periods of time.
-	cgroup.SetGOGC(50)
+	cgroup.SetGOGC(30)
 
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -443,10 +443,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		// https://prometheus.io/docs/prometheus/latest/querying/api/#targets
 		state := r.FormValue("state")
-		scrapePool := r.FormValue("scrapePool")
-		promscrape.WriteAPIV1Targets(w, state, scrapePool)
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/prometheus/target_response", "/target_response":
 		promscrapeTargetResponseRequests.Inc()

app/vmagent/remotewrite/client.go

@@ -10,22 +10,20 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 var (
@@ -90,8 +88,7 @@ type client struct {
 	remoteWriteURL string
 
 	// Whether to use VictoriaMetrics remote write protocol for sending the data to remoteWriteURL
-	useVMProto          atomic.Bool
-	canDowngradeVMProto atomic.Bool
+	useVMProto bool
 
 	fq *persistentqueue.FastQueue
 	hc *http.Client
@@ -129,7 +126,8 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 
-	tr := httputil.NewTransport(false, "vmagent_remotewrite")
+	tr := httputil.NewTransport(false)
+	tr.DialContext = netutil.NewStatDialFunc("vmagent_remotewrite")
 	tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
 	tr.MaxConnsPerHost = 2 * concurrency
 	tr.MaxIdleConnsPerHost = 2 * concurrency
@@ -170,11 +168,17 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
 	}
 	if !useVMProto && !usePromProto {
-		// The VM protocol could be downgraded later at runtime if unsupported media type response status is received.
-		useVMProto = true
-		c.canDowngradeVMProto.Store(true)
+		// Auto-detect whether the remote storage supports VictoriaMetrics remote write protocol.
+		doRequest := func(url string) (*http.Response, error) {
+			return c.doRequest(url, nil)
+		}
+		useVMProto = protoparserutil.HandleVMProtoClientHandshake(c.remoteWriteURL, doRequest)
+		if !useVMProto {
+			logger.Infof("the remote storage at %q doesn't support VictoriaMetrics remote write protocol. Switching to Prometheus remote write protocol. "+
+				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", sanitizedURL)
+		}
 	}
-	c.useVMProto.Store(useVMProto)
+	c.useVMProto = useVMProto
 
 	return c
 }
@@ -382,7 +386,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
-	if encoding.IsZstd(body) {
+	if c.useVMProto {
 		h.Set("Content-Encoding", "zstd")
 		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
 	} else {
@@ -418,7 +422,7 @@ again:
 		if retryDuration > maxRetryDuration {
 			retryDuration = maxRetryDuration
 		}
-		remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
+		logger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
 			len(block), c.sanitizedURL, err, retryDuration.Seconds())
 		t := timerpool.Get(retryDuration)
 		select {
@@ -431,7 +435,6 @@ again:
 		c.retriesCount.Inc()
 		goto again
 	}
-
 	statusCode := resp.StatusCode
 	if statusCode/100 == 2 {
 		_ = resp.Body.Close()
@@ -440,43 +443,21 @@ again:
 		c.blocksSent.Inc()
 		return true
 	}
-
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="%d"}`, c.sanitizedURL, statusCode)).Inc()
-	if statusCode == 409 {
-		logBlockRejected(block, c.sanitizedURL, resp)
-
-		// Just drop block on 409 status code like Prometheus does.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
-		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
+	if statusCode == 409 || statusCode == 400 {
+		body, err := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
-		c.packetsDropped.Inc()
-		return true
-		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
-		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
-		// - Real-world implementations of v1 use both 400 and 415 status codes.
-		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
-	} else if statusCode == 415 || statusCode == 400 {
-		if c.canDowngradeVMProto.Swap(false) {
-			logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
-				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-			c.useVMProto.Store(false)
+		if err != nil {
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
+				"failed to read response body: %s",
+				len(block), c.sanitizedURL, statusCode, err)
+		} else {
+			remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
+				len(block), c.sanitizedURL, statusCode, string(body))
 		}
-
-		if encoding.IsZstd(block) {
-			logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
-				"See https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol", c.sanitizedURL)
-
-			block = mustRepackBlockFromZstdToSnappy(block)
-
-			c.retriesCount.Inc()
-			_ = resp.Body.Close()
-			goto again
-		}
-
-		// Just drop snappy blocks on 400 or 415 status codes like Prometheus does.
+		// Just drop block on 409 and 400 status codes like Prometheus does.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/873
 		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1149
-		logBlockRejected(block, c.sanitizedURL, resp)
 		_ = resp.Body.Close()
 		c.packetsDropped.Inc()
 		return true
@@ -509,7 +490,6 @@ again:
 }
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
-var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 
 // getRetryDuration returns retry duration.
 // retryAfterDuration has the highest priority.
@@ -532,28 +512,6 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 	return retryDuration
 }
 
-func mustRepackBlockFromZstdToSnappy(zstdBlock []byte) []byte {
-	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
-	if err != nil {
-		logger.Panicf("FATAL: cannot re-pack block with size %d bytes from Zstd to Snappy: %s", len(zstdBlock), err)
-	}
-
-	return snappy.Encode(nil, plainBlock)
-}
-
-func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; "+
-			"failed to read response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, err)
-	} else {
-		remoteWriteRejectedLogger.Errorf("sending a block with size %d bytes to %q was rejected (skipping the block): status code %d; response body: %s",
-			len(block), sanitizedURL, resp.StatusCode, string(body))
-	}
-}
-
 // parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
 // retryAfterString should be in either HTTP-date or a number of seconds.
 // It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.

app/vmagent/remotewrite/client_test.go

@@ -5,9 +5,6 @@ import (
 	"net/http"
 	"testing"
 	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/golang/snappy"
 )
 
 func TestCalculateRetryDuration(t *testing.T) {
@@ -100,19 +97,3 @@ func helper(d time.Duration) time.Duration {
 
 	return d + dv
 }
-
-func TestRepackBlockFromZstdToSnappy(t *testing.T) {
-	expectedPlainBlock := []byte(`foobar`)
-
-	zstdBlock := encoding.CompressZSTDLevel(nil, expectedPlainBlock, 1)
-	snappyBlock := mustRepackBlockFromZstdToSnappy(zstdBlock)
-
-	actualPlainBlock, err := snappy.Decode(nil, snappyBlock)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-
-	if string(actualPlainBlock) != string(expectedPlainBlock) {
-		t.Fatalf("unexpected plain block; got %q; want %q", actualPlainBlock, expectedPlainBlock)
-	}
-}

app/vmagent/remotewrite/pendingseries.go

@@ -16,7 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/golang/snappy"
@@ -40,7 +39,7 @@ type pendingSeries struct {
 	periodicFlusherWG sync.WaitGroup
 }
 
-func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Bool, significantFigures, roundDigits int) *pendingSeries {
+func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite bool, significantFigures, roundDigits int) *pendingSeries {
 	var ps pendingSeries
 	ps.wr.fq = fq
 	ps.wr.isVMRemoteWrite = isVMRemoteWrite
@@ -100,7 +99,7 @@ type writeRequest struct {
 	fq *persistentqueue.FastQueue
 
 	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	isVMRemoteWrite *atomic.Bool
+	isVMRemoteWrite bool
 
 	// How many significant figures must be left before sending the writeRequest to fq.
 	significantFigures int
@@ -138,7 +137,7 @@ func (wr *writeRequest) reset() {
 // This is needed in order to properly save in-memory data to persistent queue on graceful shutdown.
 func (wr *writeRequest) mustFlushOnStop() {
 	wr.wr.Timeseries = wr.tss
-	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite.Load()) {
+	if !tryPushWriteRequest(&wr.wr, wr.mustWriteBlock, wr.isVMRemoteWrite) {
 		logger.Panicf("BUG: final flush must always return true")
 	}
 	wr.reset()
@@ -152,7 +151,7 @@ func (wr *writeRequest) mustWriteBlock(block []byte) bool {
 func (wr *writeRequest) tryFlush() bool {
 	wr.wr.Timeseries = wr.tss
 	wr.lastFlushTime.Store(fasttime.UnixTimestamp())
-	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite.Load()) {
+	if !tryPushWriteRequest(&wr.wr, wr.fq.TryWriteBlock, wr.isVMRemoteWrite) {
 		return false
 	}
 	wr.reset()
@@ -198,43 +197,28 @@ func (wr *writeRequest) tryPush(src []prompbmarshal.TimeSeries) bool {
 }
 
 func (wr *writeRequest) copyTimeSeries(dst, src *prompbmarshal.TimeSeries) {
-	labelsSrc := src.Labels
-
-	// Pre-allocate memory for labels.
+	labelsDst := wr.labels
 	labelsLen := len(wr.labels)
-	wr.labels = slicesutil.SetLength(wr.labels, labelsLen+len(labelsSrc))
-	labelsDst := wr.labels[labelsLen:]
+	samplesDst := wr.samples
+	buf := wr.buf
+	for i := range src.Labels {
+		labelsDst = append(labelsDst, prompbmarshal.Label{})
+		dstLabel := &labelsDst[len(labelsDst)-1]
+		srcLabel := &src.Labels[i]
 
-	// Pre-allocate memory for byte slice needed for storing label names and values.
-	neededBufLen := 0
-	for i := range labelsSrc {
-		label := &labelsSrc[i]
-		neededBufLen += len(label.Name) + len(label.Value)
-	}
-	bufLen := len(wr.buf)
-	wr.buf = slicesutil.SetLength(wr.buf, bufLen+neededBufLen)
-	buf := wr.buf[:bufLen]
-
-	// Copy labels
-	for i := range labelsSrc {
-		dstLabel := &labelsDst[i]
-		srcLabel := &labelsSrc[i]
-
-		bufLen := len(buf)
 		buf = append(buf, srcLabel.Name...)
-		dstLabel.Name = bytesutil.ToUnsafeString(buf[bufLen:])
-
-		bufLen = len(buf)
+		dstLabel.Name = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Name):])
 		buf = append(buf, srcLabel.Value...)
-		dstLabel.Value = bytesutil.ToUnsafeString(buf[bufLen:])
+		dstLabel.Value = bytesutil.ToUnsafeString(buf[len(buf)-len(srcLabel.Value):])
 	}
-	wr.buf = buf
-	dst.Labels = labelsDst
+	dst.Labels = labelsDst[labelsLen:]
 
-	// Copy samples
-	samplesLen := len(wr.samples)
-	wr.samples = append(wr.samples, src.Samples...)
-	dst.Samples = wr.samples[samplesLen:]
+	samplesDst = append(samplesDst, src.Samples...)
+	dst.Samples = samplesDst[len(samplesDst)-len(src.Samples):]
+
+	wr.samples = samplesDst
+	wr.labels = labelsDst
+	wr.buf = buf
 }
 
 // marshalConcurrency limits the maximum number of concurrent workers, which marshal and compress WriteRequest.

app/vmagent/remotewrite/remotewrite.go

@@ -807,7 +807,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 	pss := make([]*pendingSeries, pssLen)
 	for i := range pss {
-		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
+		pss[i] = newPendingSeries(fq, c.useVMProto, sf, rd)
 	}
 
 	rwctx := &remoteWriteCtx{

app/vmagent/remotewrite/remotewrite_test.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"math"
 	"reflect"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -69,9 +68,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		allRelabelConfigs.Store(rcs)
 
 		pss := make([]*pendingSeries, 1)
-		isVMProto := &atomic.Bool{}
-		isVMProto.Store(true)
-		pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
+		pss[0] = newPendingSeries(nil, true, 0, 100)
 		rwctx := &remoteWriteCtx{
 			idx:                    0,
 			streamAggrKeepInput:    keepInput,

app/vmalert-tool/deployment/Dockerfile

@@ -1,8 +0,0 @@
-ARG base_image=non-existing
-FROM $base_image
-
-EXPOSE 8880
-
-ENTRYPOINT ["/vmalert-tool-prod"]
-ARG src_binary=non-existing
-COPY $src_binary ./vmalert-tool-prod

app/vmalert/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"bytes"
+	"crypto/md5"
 	"flag"
 	"fmt"
 	"hash/fnv"
@@ -10,12 +11,11 @@ import (
 	"sort"
 	"strings"
 
-	"gopkg.in/yaml.v2"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/log"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+	"gopkg.in/yaml.v2"
 )
 
 var (
@@ -67,7 +67,7 @@ func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
 	if g.Type.Get() == "" {
 		g.Type = NewRawType(*defaultRuleType)
 	}
-	h := fnv.New64a()
+	h := md5.New()
 	h.Write(b)
 	g.Checksum = fmt.Sprintf("%x", h.Sum(nil))
 	return nil

app/vmalert/datasource/client_prom.go

@@ -35,8 +35,6 @@ type promResponse struct {
 	Stats struct {
 		SeriesFetched *string `json:"seriesFetched,omitempty"`
 	} `json:"stats,omitempty"`
-	// IsPartial supported by VictoriaMetrics
-	IsPartial *bool `json:"isPartial,omitempty"`
 }
 
 // see https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
@@ -211,7 +209,7 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result
 	if err != nil {
 		return res, err
 	}
-	res = Result{Data: ms, IsPartial: r.IsPartial}
+	res = Result{Data: ms}
 	if r.Stats.SeriesFetched != nil {
 		intV, err := strconv.Atoi(*r.Stats.SeriesFetched)
 		if err != nil {

app/vmalert/datasource/client_test.go

@@ -72,14 +72,12 @@ func TestVMInstantQuery(t *testing.T) {
 			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		case 7:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]},"stats":{"seriesFetched": "42"}}`))
-		case 8:
-			w.Write([]byte(`{"status":"success", "isPartial":true, "data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		}
 	})
 	mux.HandleFunc("/render", func(w http.ResponseWriter, _ *http.Request) {
 		c++
 		switch c {
-		case 9:
+		case 8:
 			w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
 		}
 	})
@@ -102,9 +100,9 @@ func TestVMInstantQuery(t *testing.T) {
 			t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
-		case 10:
+		case 9:
 			w.Write([]byte("[]"))
-		case 11:
+		case 10:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"total","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"total","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
 		}
 	})
@@ -205,18 +203,10 @@ func TestVMInstantQuery(t *testing.T) {
 			*res.SeriesFetched)
 	}
 
-	res, _, err = pq.Query(ctx, vmQuery, ts) // 8
-	if err != nil {
-		t.Fatalf("unexpected %s", err)
-	}
-	if res.IsPartial != nil && !*res.IsPartial {
-		t.Fatalf("unexpected metric isPartial want %+v", true)
-	}
-
 	// test graphite
 	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
 
-	res, _, err = gq.Query(ctx, queryRender, ts) // 9 - graphite
+	res, _, err = gq.Query(ctx, queryRender, ts) // 8 - graphite
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -236,9 +226,9 @@ func TestVMInstantQuery(t *testing.T) {
 	vlogs := datasourceVLogs
 	pq = s.BuildWithParams(QuerierParams{DataSourceType: string(vlogs), EvaluationInterval: 15 * time.Second})
 
-	expErr(vlogsQuery, "error parsing response") // 10
+	expErr(vlogsQuery, "error parsing response") // 9
 
-	res, _, err = pq.Query(ctx, vlogsQuery, ts) // 11
+	res, _, err = pq.Query(ctx, vlogsQuery, ts) // 10
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}

app/vmalert/datasource/datasource.go

@@ -34,9 +34,6 @@ type Result struct {
 	// If nil, then this feature is not supported by the datasource.
 	// SeriesFetched is supported by VictoriaMetrics since v1.90.
 	SeriesFetched *int
-	// IsPartial is used by VictoriaMetrics to indicate
-	// whether response data is partial.
-	IsPartial *bool
 }
 
 // QuerierBuilder builds Querier with given params.

app/vmalert/datasource/faker.go

@@ -10,9 +10,8 @@ import (
 // FakeQuerier is a mock querier that return predefined results and error message
 type FakeQuerier struct {
 	sync.Mutex
-	metrics   []Metric
-	err       error
-	isPartial *bool
+	metrics []Metric
+	err     error
 }
 
 // SetErr sets query error message
@@ -22,19 +21,11 @@ func (fq *FakeQuerier) SetErr(err error) {
 	fq.Unlock()
 }
 
-// SetPartialResponse marks query response as partial
-func (fq *FakeQuerier) SetPartialResponse(partial bool) {
-	fq.Lock()
-	fq.isPartial = &partial
-	fq.Unlock()
-}
-
 // Reset reset querier's error message and results
 func (fq *FakeQuerier) Reset() {
 	fq.Lock()
 	fq.err = nil
 	fq.metrics = fq.metrics[:0]
-	fq.isPartial = nil
 	fq.Unlock()
 }
 
@@ -66,7 +57,7 @@ func (fq *FakeQuerier) Query(_ context.Context, _ string, _ time.Time) (Result,
 	cp := make([]Metric, len(fq.metrics))
 	copy(cp, fq.metrics)
 	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
-	return Result{Data: cp, IsPartial: fq.isPartial}, req, nil
+	return Result{Data: cp}, req, nil
 }
 
 // FakeQuerierWithRegistry can store different results for different query expr

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -82,10 +83,11 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	if err := httputil.CheckURL(*addr); err != nil {
 		return nil, fmt.Errorf("invalid -datasource.url: %w", err)
 	}
-	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify, "vmalert_datasource")
+	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
 	}
+	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {

app/vmalert/notifier/alertmanager.go

@@ -161,7 +161,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig
 	}
-	tr, err := promauth.NewTLSTransport(tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify, "vmalert_notifier")
+	tr, err := promauth.NewTLSTransport(tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
 
@@ -198,10 +198,8 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 		argFunc:        fn,
 		authCfg:        aCfg,
 		relabelConfigs: relabelCfg,
-		client: &http.Client{
-			Transport: tr,
-		},
-		timeout: timeout,
-		metrics: newNotifierMetrics(alertManagerURL),
+		client:         &http.Client{Transport: tr},
+		timeout:        timeout,
+		metrics:        newNotifierMetrics(alertManagerURL),
 	}, nil
 }

app/vmalert/notifier/config.go

@@ -1,8 +1,8 @@
 package notifier
 
 import (
+	"crypto/md5"
 	"fmt"
-	"hash/fnv"
 	"net/url"
 	"os"
 	"path"
@@ -99,7 +99,7 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(any) error) error {
 	if err != nil {
 		return fmt.Errorf("failed to marshal configuration for checksum: %w", err)
 	}
-	h := fnv.New64a()
+	h := md5.New()
 	h.Write(b)
 	cfg.Checksum = fmt.Sprintf("%x", h.Sum(nil))
 	return nil

app/vmalert/remoteread/init.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -69,11 +70,12 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err := httputil.CheckURL(*addr); err != nil {
 		return nil, fmt.Errorf("invalid -remoteRead.url: %w", err)
 	}
-	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify, "vmalert_remoteread")
+	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/remotewrite/client.go

@@ -93,7 +93,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		cfg.FlushInterval = defaultFlushInterval
 	}
 	if cfg.Transport == nil {
-		cfg.Transport = httputil.NewTransport(false, "vmalert_remotewrite")
+		cfg.Transport = httputil.NewTransport(false)
 	}
 	cc := defaultConcurrency
 	if cfg.Concurrency > 0 {
@@ -146,13 +146,8 @@ func (c *Client) Close() error {
 		return fmt.Errorf("client is already closed")
 	}
 	close(c.input)
-
-	start := time.Now()
-	logger.Infof("shutting down remote write client: flushing remained series")
 	close(c.doneCh)
 	c.wg.Wait()
-	logger.Infof("shutting down remote write client: finished in %v", time.Since(start))
-
 	return nil
 }
 
@@ -161,16 +156,21 @@ func (c *Client) run(ctx context.Context) {
 	wr := &prompbmarshal.WriteRequest{}
 	shutdown := func() {
 		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
+		logger.Infof("shutting down remote write client and flushing remained series")
 
+		shutdownFlushCnt := 0
 		for ts := range c.input {
 			wr.Timeseries = append(wr.Timeseries, ts)
 			if len(wr.Timeseries) >= c.maxBatchSize {
+				shutdownFlushCnt += len(wr.Timeseries)
 				c.flush(lastCtx, wr)
 			}
 		}
 		// flush the last batch. `flush` will re-check and avoid flushing empty batch.
+		shutdownFlushCnt += len(wr.Timeseries)
 		c.flush(lastCtx, wr)
 
+		logger.Infof("shutting down remote write client flushed %d series", shutdownFlushCnt)
 		cancel()
 	}
 	c.wg.Add(1)

app/vmalert/remotewrite/debug_client.go

@@ -33,7 +33,7 @@ func NewDebugClient() (*DebugClient, error) {
 	if err := httputil.CheckURL(*addr); err != nil {
 		return nil, fmt.Errorf("invalid -remoteWrite.url: %w", err)
 	}
-	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify, "vmalert_remotewrite_debug")
+	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}

app/vmalert/remotewrite/init.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -72,11 +73,12 @@ func Init(ctx context.Context) (*Client, error) {
 	if err := httputil.CheckURL(*addr); err != nil {
 		return nil, fmt.Errorf("invalid -remoteWrite.url: %w", err)
 	}
-	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify, "vmalert_remotewrite")
+	tr, err := promauth.NewTLSTransport(*tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	tr.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/rule/alerting.go

@@ -207,8 +207,8 @@ func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string
 	if !ar.Debug {
 		return
 	}
-	prefix := fmt.Sprintf("DEBUG alerting rule %q, %q:%q (%d) at %v: ",
-		ar.File, ar.GroupName, ar.Name, ar.RuleID, at.Format(time.RFC3339))
+	prefix := fmt.Sprintf("DEBUG rule %q:%q (%d) at %v: ",
+		ar.GroupName, ar.Name, ar.RuleID, at.Format(time.RFC3339))
 
 	if a != nil {
 		labelKeys := make([]string, len(a.Labels))
@@ -408,8 +408,8 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
+	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s)", curState.Samples, curState.Duration)
 
-	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
 	qFn := func(query string) ([]datasource.Metric, error) {
 		res, _, err := ar.q.Query(ctx, query, ts)
 		return res.Data, err

app/vmalert/rule/alerting_test.go

@@ -22,32 +22,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 )
 
-func TestNewAlertingRule(t *testing.T) {
-	f := func(group *Group, rule config.Rule, expectRule *AlertingRule) {
-		t.Helper()
-
-		r := NewAlertingRule(&datasource.FakeQuerier{}, group, rule)
-		if err := CompareRules(t, expectRule, r); err != nil {
-			t.Fatalf("unexpected rule mismatch: %s", err)
-		}
-	}
-
-	f(&Group{Name: "foo"},
-		config.Rule{
-			Alert: "health",
-			Expr:  "up == 0",
-			Labels: map[string]string{
-				"foo": "bar",
-			},
-		}, &AlertingRule{
-			Name: "health",
-			Expr: "up == 0",
-			Labels: map[string]string{
-				"foo": "bar",
-			},
-		})
-}
-
 func TestAlertingRuleToTimeSeries(t *testing.T) {
 	timestamp := time.Now()
 
@@ -1348,23 +1322,3 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		t.Fatalf("processed labels mismatch, got: %v, want: %v", ls.processed, expectedProcessedLabels)
 	}
 }
-
-func TestAlertingRuleExec_Partial(t *testing.T) {
-	fq := &datasource.FakeQuerier{}
-	fq.Add(metricWithValueAndLabels(t, 10, "__name__", "bar"))
-	fq.SetPartialResponse(true)
-
-	ar := newTestAlertingRule("test", 0)
-	ar.Debug = true
-	ar.Labels = map[string]string{"job": "test"}
-	ar.q = fq
-	ar.For = time.Second
-
-	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
-
-	ts := time.Now()
-	_, err := ar.exec(context.TODO(), ts, 0)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-}

app/vmalert/rule/group_test.go

@@ -36,178 +36,144 @@ func TestMain(m *testing.M) {
 }
 
 func TestUpdateWith(t *testing.T) {
-	f := func(oldG, newG config.Group) {
+	f := func(currentRules, newRules []config.Rule) {
 		t.Helper()
 
 		ns := metrics.NewSet()
+		g := &Group{
+			Name:    "test",
+			metrics: &groupMetrics{set: ns},
+		}
 		qb := &datasource.FakeQuerier{}
-		for i := range oldG.Rules {
-			oldG.Rules[i].ID = config.HashRule(oldG.Rules[i])
-		}
-		for i := range newG.Rules {
-			newG.Rules[i].ID = config.HashRule(newG.Rules[i])
+		for _, r := range currentRules {
+			r.ID = config.HashRule(r)
+			g.Rules = append(g.Rules, g.newRule(qb, r))
 		}
 
-		g := NewGroup(oldG, qb, 0, nil)
-		g.metrics = &groupMetrics{set: ns}
-		expect := NewGroup(newG, qb, 0, nil)
+		ng := &Group{
+			Name: "test",
+		}
+		for _, r := range newRules {
+			r.ID = config.HashRule(r)
+			ng.Rules = append(ng.Rules, ng.newRule(qb, r))
+		}
 
-		err := g.updateWith(expect)
+		err := g.updateWith(ng)
 		if err != nil {
 			t.Fatalf("cannot update rule: %s", err)
 		}
 
-		if len(g.Rules) != len(expect.Rules) {
-			t.Fatalf("expected to have %d rules; got: %d", len(expect.Rules), len(g.Rules))
+		if len(g.Rules) != len(newRules) {
+			t.Fatalf("expected to have %d rules; got: %d", len(g.Rules), len(newRules))
 		}
 		sort.Slice(g.Rules, func(i, j int) bool {
 			return g.Rules[i].ID() < g.Rules[j].ID()
 		})
-		sort.Slice(expect.Rules, func(i, j int) bool {
-			return expect.Rules[i].ID() < expect.Rules[j].ID()
+		sort.Slice(ng.Rules, func(i, j int) bool {
+			return ng.Rules[i].ID() < ng.Rules[j].ID()
 		})
 		for i, r := range g.Rules {
-			got, want := r, expect.Rules[i]
+			got, want := r, ng.Rules[i]
 			if got.ID() != want.ID() {
 				t.Fatalf("expected to have rule %q; got %q", want, got)
 			}
 			if err := CompareRules(t, got, want); err != nil {
-				t.Fatalf("comparison1 error: %s", err)
+				t.Fatalf("comparison error: %s", err)
 			}
 		}
 	}
 
 	// new rule
-	f(config.Group{}, config.Group{
-		Rules: []config.Rule{
-			{Alert: "bar"},
-		}})
+	f(nil, []config.Rule{
+		{Alert: "bar"},
+	})
 
 	// update alerting rule
-	f(config.Group{
-		Rules: []config.Rule{
-			{
-				Alert: "foo",
-				Expr:  "up > 0",
-				For:   promutil.NewDuration(time.Second),
-				Labels: map[string]string{
-					"bar": "baz",
-				},
-				Annotations: map[string]string{
-					"summary":     "{{ $value|humanize }}",
-					"description": "{{$labels}}",
-				},
-			},
-			{
-				Alert: "bar",
-				Expr:  "up > 0",
-				For:   promutil.NewDuration(time.Second),
-				Labels: map[string]string{
-					"bar": "baz",
-				},
-			},
-		}}, config.Group{
-		Rules: []config.Rule{
-			{
-				Alert: "foo",
-				Expr:  "up > 10",
-				For:   promutil.NewDuration(time.Second),
-				Labels: map[string]string{
-					"baz": "bar",
-				},
-				Annotations: map[string]string{
-					"summary": "none",
-				},
-			},
-			{
-				Alert:         "bar",
-				Expr:          "up > 0",
-				For:           promutil.NewDuration(2 * time.Second),
-				KeepFiringFor: promutil.NewDuration(time.Minute),
-				Labels: map[string]string{
-					"bar": "baz",
-				},
-			},
-		}})
-
-	// update recording rule
-	f(config.Group{
-		Rules: []config.Rule{{
-			Record: "foo",
-			Expr:   "max(up)",
+	f([]config.Rule{
+		{
+			Alert: "foo",
+			Expr:  "up > 0",
+			For:   promutil.NewDuration(time.Second),
 			Labels: map[string]string{
 				"bar": "baz",
 			},
-		}}}, config.Group{
-		Rules: []config.Rule{{
-			Record: "foo",
-			Expr:   "min(up)",
-			Debug:  true,
+			Annotations: map[string]string{
+				"summary":     "{{ $value|humanize }}",
+				"description": "{{$labels}}",
+			},
+		},
+		{
+			Alert: "bar",
+			Expr:  "up > 0",
+			For:   promutil.NewDuration(time.Second),
+			Labels: map[string]string{
+				"bar": "baz",
+			},
+		},
+	}, []config.Rule{
+		{
+			Alert: "foo",
+			Expr:  "up > 10",
+			For:   promutil.NewDuration(time.Second),
 			Labels: map[string]string{
 				"baz": "bar",
 			},
-		}}})
+			Annotations: map[string]string{
+				"summary": "none",
+			},
+		},
+		{
+			Alert:         "bar",
+			Expr:          "up > 0",
+			For:           promutil.NewDuration(2 * time.Second),
+			KeepFiringFor: promutil.NewDuration(time.Minute),
+			Labels: map[string]string{
+				"bar": "baz",
+			},
+		},
+	})
 
-	// update debug
-	f(config.Group{
-		Rules: []config.Rule{
-			{
-				Record: "foo",
-				Expr:   "max(up)",
-			},
-			{
-				Alert: "foo",
-				Expr:  "up > 0",
-				Debug: true,
-				For:   promutil.NewDuration(time.Second),
-			},
-		}}, config.Group{
-		Rules: []config.Rule{
-			{
-				Record: "foo",
-				Expr:   "max(up)",
-				Debug:  true,
-			},
-			{
-				Alert: "foo",
-				Expr:  "up > 0",
-				For:   promutil.NewDuration(time.Second),
-			},
-		}})
+	// update recording rule
+	f([]config.Rule{{
+		Record: "foo",
+		Expr:   "max(up)",
+		Labels: map[string]string{
+			"bar": "baz",
+		},
+	}}, []config.Rule{{
+		Record: "foo",
+		Expr:   "min(up)",
+		Labels: map[string]string{
+			"baz": "bar",
+		},
+	}})
 
 	// empty rule
-	f(config.Group{
-		Rules: []config.Rule{{Alert: "foo"}, {Record: "bar"}}}, config.Group{})
+	f([]config.Rule{{Alert: "foo"}, {Record: "bar"}}, nil)
 
 	// multiple rules
-	f(config.Group{
-		Rules: []config.Rule{
-			{Alert: "bar"},
-			{Alert: "baz"},
-			{Alert: "foo"},
-		}}, config.Group{
-		Rules: []config.Rule{
-			{Alert: "baz"},
-			{Record: "foo"},
-		}})
+	f([]config.Rule{
+		{Alert: "bar"},
+		{Alert: "baz"},
+		{Alert: "foo"},
+	}, []config.Rule{
+		{Alert: "baz"},
+		{Record: "foo"},
+	})
 
 	// replace rule
-	f(config.Group{
-		Rules: []config.Rule{{Alert: "foo1"}}}, config.Group{
-		Rules: []config.Rule{{Alert: "foo2"}}})
+	f([]config.Rule{{Alert: "foo1"}}, []config.Rule{{Alert: "foo2"}})
 
 	// replace multiple rules
-	f(config.Group{
-		Rules: []config.Rule{
-			{Alert: "foo1"},
-			{Record: "foo2"},
-			{Alert: "foo3"},
-		}}, config.Group{
-		Rules: []config.Rule{
-			{Alert: "foo3"},
-			{Alert: "foo4"},
-			{Record: "foo5"},
-		}})
+	f([]config.Rule{
+		{Alert: "foo1"},
+		{Record: "foo2"},
+		{Alert: "foo3"},
+	}, []config.Rule{
+		{Alert: "foo3"},
+		{Alert: "foo4"},
+		{Record: "foo5"},
+	})
 }
 
 func TestUpdateDuringRandSleep(t *testing.T) {

app/vmalert/rule/recording.go

@@ -30,7 +30,6 @@ type RecordingRule struct {
 	GroupID   uint64
 	GroupName string
 	File      string
-	Debug     bool
 
 	q datasource.Querier
 
@@ -92,14 +91,12 @@ func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID:   group.GetID(),
 		GroupName: group.Name,
 		File:      group.File,
-		Debug:     cfg.Debug,
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:            group.Type.String(),
 			ApplyIntervalAsTimeFilter: setIntervalAsTimeFilter(group.Type.String(), cfg.Expr),
 			EvaluationInterval:        group.Interval,
 			QueryParams:               group.Params,
 			Headers:                   group.Headers,
-			Debug:                     cfg.Debug,
 		}),
 	}
 
@@ -172,8 +169,6 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 		return nil, curState.Err
 	}
 
-	rr.logDebugf(ts, "query returned %d samples (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
-
 	qMetrics := res.Data
 	numSeries := len(qMetrics)
 	if limit > 0 && numSeries > limit {
@@ -207,17 +202,6 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 	return tss, nil
 }
 
-func (rr *RecordingRule) logDebugf(at time.Time, format string, args ...any) {
-	if !rr.Debug {
-		return
-	}
-	prefix := fmt.Sprintf("DEBUG recording rule %q, %q:%q (%d) at %v: ",
-		rr.File, rr.GroupName, rr.Name, rr.RuleID, at.Format(time.RFC3339))
-
-	msg := fmt.Sprintf(format, args...)
-	logger.Infof("%s", prefix+msg)
-}
-
 func stringToLabels(s string) []prompbmarshal.Label {
 	labels := strings.Split(s, ",")
 	rLabels := make([]prompbmarshal.Label, 0, len(labels))
@@ -278,7 +262,6 @@ func (rr *RecordingRule) updateWith(r Rule) error {
 	rr.Expr = nr.Expr
 	rr.Labels = nr.Labels
 	rr.q = nr.q
-	rr.Debug = nr.Debug
 	return nil
 }
 

app/vmalert/rule/recording_test.go

@@ -9,34 +9,11 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-func TestNewRecordingRule(t *testing.T) {
-	f := func(group *Group, rule config.Rule, expectRule *AlertingRule) {
-		t.Helper()
-
-		r := NewAlertingRule(&datasource.FakeQuerier{}, group, rule)
-		if err := CompareRules(t, expectRule, r); err != nil {
-			t.Fatalf("unexpected rule mismatch: %s", err)
-		}
-	}
-
-	f(&Group{Name: "foo"},
-		config.Rule{
-			Alert:  "health",
-			Expr:   "up == 0",
-			Labels: map[string]string{},
-		}, &AlertingRule{
-			Name:   "health",
-			Expr:   "up == 0",
-			Labels: map[string]string{},
-		})
-}
-
 func TestRecordingRule_Exec(t *testing.T) {
 	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
 	const defaultStep = 5 * time.Millisecond
@@ -330,8 +307,7 @@ func TestRecordingRuleLimit_Failure(t *testing.T) {
 
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-		rule := &RecordingRule{
-			Name:  "job:foo",
+		rule := &RecordingRule{Name: "job:foo",
 			state: &ruleState{entries: make([]StateEntry, 10)},
 			Labels: map[string]string{
 				"source": "test_limit",
@@ -366,8 +342,7 @@ func TestRecordingRuleLimit_Success(t *testing.T) {
 
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-		rule := &RecordingRule{
-			Name:  "job:foo",
+		rule := &RecordingRule{Name: "job:foo",
 			state: &ruleState{entries: make([]StateEntry, 10)},
 			Labels: map[string]string{
 				"source": "test_limit",
@@ -446,36 +421,3 @@ func TestSetIntervalAsTimeFilter(t *testing.T) {
 	f(`error AND _time:5m | count()`, "vlogs", false)
 	f(`* | error AND _time:5m | count()`, "vlogs", false)
 }
-
-func TestRecordingRuleExec_Partial(t *testing.T) {
-	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
-	fq := &datasource.FakeQuerier{}
-
-	m := metricWithValueAndLabels(t, 10, "__name__", "bar")
-	fq.Add(m)
-	fq.SetPartialResponse(true)
-	rule := &RecordingRule{
-		GroupName: "Bar",
-		Name:      "foo",
-		state: &ruleState{
-			entries: make([]StateEntry, 10),
-		},
-	}
-	rule.Debug = true
-	rule.q = fq
-	got, err := rule.exec(context.TODO(), ts, 0)
-	want := []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "foo",
-			},
-		}),
-	}
-	if err != nil {
-		t.Fatalf("fail to test rule %s: unexpected error: %s", rule.Name, err)
-	}
-	if err := compareTimeSeries(t, want, got); err != nil {
-		t.Fatalf("fail to test rule %s: time series mismatch: %s", rule.Name, err)
-	}
-}

app/vmalert/rule/test_helpers.go

@@ -38,9 +38,6 @@ func compareRecordingRules(t *testing.T, a, b *RecordingRule) error {
 	if a.Expr != b.Expr {
 		return fmt.Errorf("expected to have expression %q; got %q", a.Expr, b.Expr)
 	}
-	if a.Debug != b.Debug {
-		return fmt.Errorf("expected to have debug=%t; got %t", a.Debug, b.Debug)
-	}
 	if !reflect.DeepEqual(a.Labels, b.Labels) {
 		return fmt.Errorf("expected to have labels %#v; got %#v", a.Labels, b.Labels)
 	}
@@ -67,9 +64,6 @@ func compareAlertingRules(t *testing.T, a, b *AlertingRule) error {
 	if a.Type.String() != b.Type.String() {
 		return fmt.Errorf("expected to have Type %#v; got %#v", a.Type.String(), b.Type.String())
 	}
-	if a.Debug != b.Debug {
-		return fmt.Errorf("expected to have debug=%t; got %t", a.Debug, b.Debug)
-	}
 	return nil
 }
 

app/vmalert/rule/utils.go

@@ -105,10 +105,3 @@ func isSecreteHeader(str string) bool {
 	}
 	return false
 }
-
-func isPartialResponse(res datasource.Result) bool {
-	if res.IsPartial != nil && *res.IsPartial {
-		return true
-	}
-	return false
-}

app/vmauth/main.go

@@ -319,7 +319,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 				// Timed out request must be counted as errors, since this usually means that the backend is slow.
 				ui.backendErrors.Inc()
 			}
-			return false, false
+			return true, false
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
@@ -508,7 +508,7 @@ func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, i
 		return nil, fmt.Errorf("cannot initialize promauth.Config: %w", err)
 	}
 
-	tr := httputil.NewTransport(false, "vmauth_backend")
+	tr := httputil.NewTransport(false)
 	tr.ResponseHeaderTimeout = *responseTimeout
 	// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
 	tr.DisableCompression = true
@@ -517,6 +517,7 @@ func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, i
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
+	tr.DialContext = netutil.NewStatDialFunc("vmauth_backend")
 
 	rt := cfg.NewRoundTripper(tr)
 	return rt, nil

app/vmctl/main.go

@@ -70,7 +70,7 @@ func main() {
 						return fmt.Errorf("invalid -%s: %w", otsdbAddr, err)
 					}
 
-					tr, err := promauth.NewTLSTransport(certFile, keyFile, caFile, serverName, insecureSkipVerify, "vmctl_opentsdb")
+					tr, err := promauth.NewTLSTransport(certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
 						return fmt.Errorf("failed to create transport for -%s=%q: %s", otsdbAddr, addr, err)
 					}
@@ -185,7 +185,7 @@ func main() {
 					serverName := c.String(remoteReadServerName)
 					insecureSkipVerify := c.Bool(remoteReadInsecureSkipVerify)
 
-					tr, err := promauth.NewTLSTransport(certFile, keyFile, caFile, serverName, insecureSkipVerify, "vmctl_remoteread")
+					tr, err := promauth.NewTLSTransport(certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
 						return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
 					}
@@ -315,7 +315,7 @@ func main() {
 						return fmt.Errorf("failed to create TLS Config: %s", err)
 					}
 
-					trSrc := httputil.NewTransport(false, "vmctl_src")
+					trSrc := httputil.NewTransport(false)
 					trSrc.DisableKeepAlives = disableKeepAlive
 					trSrc.TLSClientConfig = srcTC
 
@@ -345,7 +345,7 @@ func main() {
 						return fmt.Errorf("failed to create TLS Config: %s", err)
 					}
 
-					trDst := httputil.NewTransport(false, "vmctl_dst")
+					trDst := httputil.NewTransport(false)
 					trDst.DisableKeepAlives = disableKeepAlive
 					trDst.TLSClientConfig = dstTC
 
@@ -455,7 +455,7 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 	serverName := c.String(vmServerName)
 	insecureSkipVerify := c.Bool(vmInsecureSkipVerify)
 
-	tr, err := promauth.NewTLSTransport(certFile, keyFile, caFile, serverName, insecureSkipVerify, "vmctl_client")
+	tr, err := promauth.NewTLSTransport(certFile, keyFile, caFile, serverName, insecureSkipVerify)
 	if err != nil {
 		return vm.Config{}, fmt.Errorf("failed to create transport for -%s=%q: %s", vmAddr, addr, err)
 	}

app/vmctl/prometheus.go

@@ -43,12 +43,60 @@ func (pp *prometheusProcessor) run() error {
 		return nil
 	}
 
-	if err := pp.processBlocks(blocks); err != nil {
-		return fmt.Errorf("migration failed: %s", err)
+	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
+	if err := barpool.Start(); err != nil {
+		return err
+	}
+	defer barpool.Stop()
+
+	blockReadersCh := make(chan tsdb.BlockReader)
+	errCh := make(chan error, pp.cc)
+	pp.im.ResetStats()
+
+	var wg sync.WaitGroup
+	wg.Add(pp.cc)
+	for i := 0; i < pp.cc; i++ {
+		go func() {
+			defer wg.Done()
+			for br := range blockReadersCh {
+				if err := pp.do(br); err != nil {
+					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
+					return
+				}
+				bar.Increment()
+			}
+		}()
+	}
+	// any error breaks the import
+	for _, br := range blocks {
+		select {
+		case promErr := <-errCh:
+			close(blockReadersCh)
+			return fmt.Errorf("prometheus error: %s", promErr)
+		case vmErr := <-pp.im.Errors():
+			close(blockReadersCh)
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
+		case blockReadersCh <- br:
+		}
+	}
+
+	close(blockReadersCh)
+	wg.Wait()
+	// wait for all buffers to flush
+	pp.im.Close()
+	close(errCh)
+	// drain import errors channel
+	for vmErr := range pp.im.Errors() {
+		if vmErr.Err != nil {
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
+		}
+	}
+	for err := range errCh {
+		return fmt.Errorf("import process failed: %s", err)
 	}
 
 	log.Println("Import finished!")
-	log.Println(pp.im.Stats())
+	log.Print(pp.im.Stats())
 	return nil
 }
 
@@ -108,59 +156,3 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	}
 	return ss.Err()
 }
-
-func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
-	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
-	if err := barpool.Start(); err != nil {
-		return err
-	}
-	defer barpool.Stop()
-
-	blockReadersCh := make(chan tsdb.BlockReader)
-	errCh := make(chan error, pp.cc)
-	pp.im.ResetStats()
-
-	var wg sync.WaitGroup
-	wg.Add(pp.cc)
-	for i := 0; i < pp.cc; i++ {
-		go func() {
-			defer wg.Done()
-			for br := range blockReadersCh {
-				if err := pp.do(br); err != nil {
-					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
-					return
-				}
-				bar.Increment()
-			}
-		}()
-	}
-	// any error breaks the import
-	for _, br := range blocks {
-		select {
-		case promErr := <-errCh:
-			close(blockReadersCh)
-			return fmt.Errorf("prometheus error: %s", promErr)
-		case vmErr := <-pp.im.Errors():
-			close(blockReadersCh)
-			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
-		case blockReadersCh <- br:
-		}
-	}
-
-	close(blockReadersCh)
-	wg.Wait()
-	// wait for all buffers to flush
-	pp.im.Close()
-	close(errCh)
-	// drain import errors channel
-	for vmErr := range pp.im.Errors() {
-		if vmErr.Err != nil {
-			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
-		}
-	}
-	for err := range errCh {
-		return fmt.Errorf("import process failed: %s", err)
-	}
-
-	return nil
-}

app/vmctl/prometheus_test.go

@@ -2,214 +2,167 @@ package main
 
 import (
 	"context"
-	"fmt"
-	"log"
 	"os"
 	"testing"
 	"time"
 
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/storage"
-	"github.com/prometheus/prometheus/tsdb"
-	"github.com/prometheus/prometheus/tsdb/chunkenc"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
-	remote_read_integration "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 )
 
+// If you want to run this test:
+// 1. provide test snapshot path in const testSnapshot
+// 2. define httpAddr const with your victoriametrics address
+// 3. run victoria metrics with defined address
+// 4. remove t.Skip() from Test_prometheusProcessor_run
+// 5. run tests one by one not all at one time
+
 const (
-	testSnapshot = "./testdata/snapshots/20250118T124506Z-59d1b952d7eaf547"
-	blockData    = "./testdata/snapshots/20250118T124506Z-59d1b952d7eaf547/01JHWQ445Y2P1TDYB05AEKD6MC"
+	httpAddr     = "http://127.0.0.1:8428/"
+	testSnapshot = "./testdata/20220427T130947Z-70ba49b1093fd0bf"
 )
 
 // This test simulates close process if user abort it
 func TestPrometheusProcessorRun(t *testing.T) {
+	t.Skip()
 
-	f := func(startStr, endStr string, numOfSeries int, resultExpected []vm.TimeSeries) {
-		t.Helper()
+	defer func() { isSilent = false }()
 
-		dst := remote_read_integration.NewRemoteWriteServer(t)
+	type fields struct {
+		cfg    prometheus.Config
+		vmCfg  vm.Config
+		cl     func(prometheus.Config) *prometheus.Client
+		im     func(vm.Config) *vm.Importer
+		closer func(importer *vm.Importer)
+		cc     int
+	}
+	type args struct {
+		silent  bool
+		verbose bool
+	}
 
-		defer func() {
-			dst.Close()
-		}()
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "simulate syscall.SIGINT",
+			fields: fields{
+				cfg: prometheus.Config{
+					Snapshot: testSnapshot,
+					Filter:   prometheus.Filter{},
+				},
+				cl: func(cfg prometheus.Config) *prometheus.Client {
+					client, err := prometheus.NewClient(cfg)
+					if err != nil {
+						t.Fatalf("error init prometeus client: %s", err)
+					}
+					return client
+				},
+				im: func(vmCfg vm.Config) *vm.Importer {
+					importer, err := vm.NewImporter(context.Background(), vmCfg)
+					if err != nil {
+						t.Fatalf("error init importer: %s", err)
+					}
+					return importer
+				},
+				closer: func(importer *vm.Importer) {
+					// simulate syscall.SIGINT
+					time.Sleep(time.Second * 5)
+					if importer != nil {
+						importer.Close()
+					}
+				},
+				vmCfg: vm.Config{Addr: httpAddr, Concurrency: 1},
+				cc:    2,
+			},
+			args: args{
+				silent:  false,
+				verbose: false,
+			},
+			wantErr: true,
+		},
+		{
+			name: "simulate correct work",
+			fields: fields{
+				cfg: prometheus.Config{
+					Snapshot: testSnapshot,
+					Filter:   prometheus.Filter{},
+				},
+				cl: func(cfg prometheus.Config) *prometheus.Client {
+					client, err := prometheus.NewClient(cfg)
+					if err != nil {
+						t.Fatalf("error init prometeus client: %s", err)
+					}
+					return client
+				},
+				im: func(vmCfg vm.Config) *vm.Importer {
+					importer, err := vm.NewImporter(context.Background(), vmCfg)
+					if err != nil {
+						t.Fatalf("error init importer: %s", err)
+					}
+					return importer
+				},
+				closer: nil,
+				vmCfg:  vm.Config{Addr: httpAddr, Concurrency: 5},
+				cc:     2,
+			},
+			args: args{
+				silent:  true,
+				verbose: false,
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client := tt.fields.cl(tt.fields.cfg)
+			importer := tt.fields.im(tt.fields.vmCfg)
+			isSilent = tt.args.silent
+			pp := &prometheusProcessor{
+				cl:        client,
+				im:        importer,
+				cc:        tt.fields.cc,
+				isVerbose: tt.args.verbose,
+			}
 
-		dst.Series(resultExpected)
-		dst.ExpectedSeries(resultExpected)
+			// we should answer on prompt
+			if !tt.args.silent {
+				input := []byte("Y\n")
 
-		if err := fillStorage(resultExpected); err != nil {
-			t.Fatalf("cannot fill storage: %s", err)
-		}
+				r, w, err := os.Pipe()
+				if err != nil {
+					t.Fatal(err)
+				}
 
-		isSilent = true
-		defer func() { isSilent = false }()
+				_, err = w.Write(input)
+				if err != nil {
+					t.Fatalf("cannot send 'Y' to importer: %s", err)
+				}
+				err = w.Close()
+				if err != nil {
+					t.Fatalf("cannot close writer: %s", err)
+				}
 
-		bf, err := backoff.New(1, 1.8, time.Second*2)
-		if err != nil {
-			t.Fatalf("cannot create backoff: %s", err)
-		}
+				stdin := os.Stdin
+				// Restore stdin right after the test.
+				defer func() {
+					os.Stdin = stdin
+					_ = r.Close()
+				}()
+				os.Stdin = r
+			}
 
-		importerCfg := vm.Config{
-			Addr:        dst.URL(),
-			Transport:   nil,
-			Concurrency: 1,
-			Backoff:     bf,
-		}
+			// simulate close if needed
+			if tt.fields.closer != nil {
+				go tt.fields.closer(importer)
+			}
 
-		ctx := context.Background()
-		importer, err := vm.NewImporter(ctx, importerCfg)
-		if err != nil {
-			t.Fatalf("cannot create importer: %s", err)
-		}
-		defer importer.Close()
-
-		matchName := "__name__"
-		matchValue := ".*"
-		filter := prometheus.Filter{
-			TimeMin:    startStr,
-			TimeMax:    endStr,
-			Label:      matchName,
-			LabelValue: matchValue,
-		}
-
-		runnner, err := prometheus.NewClient(prometheus.Config{
-			Snapshot: testSnapshot,
-			Filter:   filter,
+			if err := pp.run(); (err != nil) != tt.wantErr {
+				t.Fatalf("run() error = %v, wantErr %v", err, tt.wantErr)
+			}
 		})
-		if err != nil {
-			t.Fatalf("cannot create prometheus client: %s", err)
-		}
-		p := &prometheusProcessor{
-			cl: runnner,
-			im: importer,
-			cc: 1,
-		}
-
-		if err := p.run(); err != nil {
-			t.Fatalf("run() error: %s", err)
-		}
-
-		collectedTs := dst.GetCollectedTimeSeries()
-		t.Logf("collected timeseries: %d; expected timeseries: %d", len(collectedTs), len(resultExpected))
-		if len(collectedTs) != len(resultExpected) {
-			t.Fatalf("unexpected number of collected time series; got %d; want %d", len(collectedTs), numOfSeries)
-		}
-
-		deleted, err := deleteSeries(matchName, matchValue)
-		if err != nil {
-			t.Fatalf("cannot delete series: %s", err)
-		}
-		if deleted != numOfSeries {
-			t.Fatalf("unexpected number of deleted series; got %d; want %d", deleted, numOfSeries)
-		}
 	}
-
-	processFlags()
-	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	defer func() {
-		vmstorage.Stop()
-		if err := os.RemoveAll(storagePath); err != nil {
-			log.Fatalf("cannot remove %q: %s", storagePath, err)
-		}
-	}()
-
-	barpool.Disable(true)
-	defer func() {
-		barpool.Disable(false)
-	}()
-
-	b, err := tsdb.OpenBlock(nil, blockData, nil, nil)
-	if err != nil {
-		t.Fatalf("cannot open block: %s", err)
-	}
-	// timestamp is equal to minTime and maxTime from meta.json
-	ss, err := readBlock(b, 1737204082361, 1737204302539)
-	if err != nil {
-		t.Fatalf("cannot read block: %s", err)
-	}
-
-	resultExpected, err := prepareExpectedData(ss)
-	if err != nil {
-		t.Fatalf("cannot prepare expected data: %s", err)
-	}
-
-	f("2025-01-18T12:40:00Z", "2025-01-18T12:46:00Z", 2792, resultExpected)
-}
-
-func readBlock(b tsdb.BlockReader, timeMin int64, timeMax int64) (storage.SeriesSet, error) {
-	minTime, maxTime := b.Meta().MinTime, b.Meta().MaxTime
-
-	if timeMin != 0 {
-		minTime = timeMin
-	}
-	if timeMax != 0 {
-		maxTime = timeMax
-	}
-
-	q, err := tsdb.NewBlockQuerier(b, minTime, maxTime)
-	if err != nil {
-		return nil, err
-	}
-	matchName := "__name__"
-	matchValue := ".*"
-	ctx := context.Background()
-	ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, matchName, matchValue))
-	return ss, nil
-}
-
-func prepareExpectedData(ss storage.SeriesSet) ([]vm.TimeSeries, error) {
-	var expectedSeriesSet []vm.TimeSeries
-	var it chunkenc.Iterator
-	for ss.Next() {
-		var name string
-		var labelPairs []vm.LabelPair
-		series := ss.At()
-
-		for _, label := range series.Labels() {
-			if label.Name == "__name__" {
-				name = label.Value
-				continue
-			}
-			labelPairs = append(labelPairs, vm.LabelPair{
-				Name:  label.Name,
-				Value: label.Value,
-			})
-		}
-		if name == "" {
-			return nil, fmt.Errorf("failed to find `__name__` label in labelset for block")
-		}
-
-		var timestamps []int64
-		var values []float64
-		it = series.Iterator(it)
-		for {
-			typ := it.Next()
-			if typ == chunkenc.ValNone {
-				break
-			}
-			if typ != chunkenc.ValFloat {
-				// Skip unsupported values
-				continue
-			}
-			t, v := it.At()
-			timestamps = append(timestamps, t)
-			values = append(values, v)
-		}
-		if err := it.Err(); err != nil {
-			return nil, err
-		}
-		ts := vm.TimeSeries{
-			Name:       name,
-			LabelPairs: labelPairs,
-			Timestamps: timestamps,
-			Values:     values,
-		}
-		expectedSeriesSet = append(expectedSeriesSet, ts)
-	}
-	return expectedSeriesSet, nil
 }

app/vmctl/remote_read_test.go

@@ -73,7 +73,7 @@ func TestRemoteRead(t *testing.T) {
 			vmCfg: vm.Config{
 				Addr:        "",
 				Concurrency: 1,
-				Transport:   httputil.NewTransport(false, "vmctl_test_read"),
+				Transport:   httputil.NewTransport(false),
 			},
 			start:            "2022-09-26T11:23:05+02:00",
 			end:              "2022-11-26T11:24:05+02:00",

app/vmctl/testdata/servers_integration_test/remote_write_server.go

@@ -41,7 +41,6 @@ type RemoteWriteServer struct {
 	server         *httptest.Server
 	series         []vm.TimeSeries
 	expectedSeries []vm.TimeSeries
-	tss            []vm.TimeSeries
 }
 
 // NewRemoteWriteServer prepares test remote write server
@@ -74,10 +73,6 @@ func (rws *RemoteWriteServer) ExpectedSeries(series []vm.TimeSeries) {
 	rws.expectedSeries = append(rws.expectedSeries, series...)
 }
 
-func (rws *RemoteWriteServer) GetCollectedTimeSeries() []vm.TimeSeries {
-	return rws.tss
-}
-
 // URL returns server url
 func (rws *RemoteWriteServer) URL() string {
 	return rws.server.URL
@@ -85,6 +80,7 @@ func (rws *RemoteWriteServer) URL() string {
 
 func (rws *RemoteWriteServer) getWriteHandler(t *testing.T) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var tss []vm.TimeSeries
 		scanner := bufio.NewScanner(r.Body)
 		var rows parser.Rows
 		for scanner.Scan() {
@@ -106,11 +102,17 @@ func (rws *RemoteWriteServer) getWriteHandler(t *testing.T) http.Handler {
 				ts.Timestamps = append(ts.Timestamps, row.Timestamps...)
 				ts.Name = nameValue
 				ts.LabelPairs = labelPairs
-				rws.tss = append(rws.tss, ts)
+				tss = append(tss, ts)
 			}
 			rows.Reset()
 		}
 
+		if !reflect.DeepEqual(tss, rws.expectedSeries) {
+			w.WriteHeader(http.StatusInternalServerError)
+			t.Fatalf("datasets not equal, expected: %#v; \n got: %#v", rws.expectedSeries, tss)
+			return
+		}
+
 		w.WriteHeader(http.StatusNoContent)
 		return
 	})

app/vmctl/testdata/snapshots/20250118T124506Z-59d1b952d7eaf547/01JHWQ445Y2P1TDYB05AEKD6MC/meta.json

@@ -1,17 +0,0 @@
-{
-	"ulid": "01JHWQ445Y2P1TDYB05AEKD6MC",
-	"minTime": 1737204082361,
-	"maxTime": 1737204302539,
-	"stats": {
-		"numSamples": 60275,
-		"numSeries": 2792,
-		"numChunks": 2792
-	},
-	"compaction": {
-		"level": 1,
-		"sources": [
-			"01JHWQ445Y2P1TDYB05AEKD6MC"
-		]
-	},
-	"version": 1
-}

app/vmctl/vm_native_test.go

@@ -23,9 +23,8 @@ import (
 )
 
 const (
-	storagePath       = "TestStorage"
-	retentionPeriod   = "100y"
-	deleteSeriesLimit = 3e3
+	storagePath     = "TestStorage"
+	retentionPeriod = "100y"
 )
 
 func TestVMNativeProcessorRun(t *testing.T) {
@@ -67,7 +66,7 @@ func TestVMNativeProcessorRun(t *testing.T) {
 			t.Fatalf("cannot add series to storage: %s", err)
 		}
 
-		tr := httputil.NewTransport(false, "test_client")
+		tr := httputil.NewTransport(false)
 		tr.DisableKeepAlives = false
 
 		srcClient := &native.Client{
@@ -260,7 +259,7 @@ func deleteSeries(name, value string) (int, error) {
 	if err := tfs.Add([]byte(name), []byte(value), false, true); err != nil {
 		return 0, fmt.Errorf("unexpected error in TagFilters.Add: %w", err)
 	}
-	return vmstorage.DeleteSeries(nil, []*storage.TagFilters{tfs}, deleteSeriesLimit)
+	return vmstorage.DeleteSeries(nil, []*storage.TagFilters{tfs}, 1e3)
 }
 
 func TestBuildMatchWithFilter_Failure(t *testing.T) {

app/vminsert/main.go

@@ -320,10 +320,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/prometheus/api/v1/targets", "/api/v1/targets":
 		promscrapeAPIV1TargetsRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		// https://prometheus.io/docs/prometheus/latest/querying/api/#targets
 		state := r.FormValue("state")
-		scrapePool := r.FormValue("scrapePool")
-		promscrape.WriteAPIV1Targets(w, state, scrapePool)
+		promscrape.WriteAPIV1Targets(w, state)
 		return true
 	case "/prometheus/target_response", "/target_response":
 		promscrapeTargetResponseRequests.Inc()

app/vmselect/prometheus/prometheus.go

@@ -806,6 +806,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	} else {
 		queryOffset = 0
 	}
+	qs := &promql.QueryStats{}
 	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 start,
@@ -821,10 +822,9 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
-	}
-	qs := promql.NewQueryStats(query, nil, ec)
-	ec.QueryStats = qs
 
+		QueryStats: qs,
+	}
 	result, err := promql.Exec(qt, ec, query, true)
 	if err != nil {
 		return fmt.Errorf("error when executing query=%q for (time=%d, step=%d): %w", query, start, step, err)
@@ -853,7 +853,6 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot flush query response to remote client: %w", err)
 	}
-
 	return nil
 }
 
@@ -915,6 +914,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		start, end = promql.AdjustStartEnd(start, end, step)
 	}
 
+	qs := &promql.QueryStats{}
 	ec := &promql.EvalConfig{
 		Start:               start,
 		End:                 end,
@@ -930,10 +930,9 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
-	}
-	qs := promql.NewQueryStats(query, nil, ec)
-	ec.QueryStats = qs
 
+		QueryStats: qs,
+	}
 	result, err := promql.Exec(qt, ec, query, false)
 	if err != nil {
 		return err
@@ -962,7 +961,6 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	if err := bw.Flush(); err != nil {
 		return fmt.Errorf("cannot send query range response to remote client: %w", err)
 	}
-
 	return nil
 }
 

app/vmselect/prometheus/query_range_response.qtpl

@@ -34,7 +34,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
 			// It cannot be converted to int without breaking backwards compatibility at vmalert :(
 		%}
 		"seriesFetched": "{%dl qs.SeriesFetched.Load() %}",
-		"executionTimeMsec": {%dl qs.ExecutionDuration.Load().Milliseconds() %}
+		"executionTimeMsec": {%dl qs.ExecutionTimeMsec.Load() %}
 	}
 	{% code
 		qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)

app/vmselect/prometheus/query_range_response.qtpl.go

@@ -71,9 +71,9 @@ func StreamQueryRangeResponse(qw422016 *qt422016.Writer, rs []netstorage.Result,
 	qw422016.N().DL(qs.SeriesFetched.Load())
 //line app/vmselect/prometheus/query_range_response.qtpl:36
 	qw422016.N().S(`","executionTimeMsec":`)
-//line app/vmselect/prometheus/query_range_response.qtpl:38
-	qw422016.N().DL(qs.ExecutionDuration.Load().Milliseconds())
-//line app/vmselect/prometheus/query_range_response.qtpl:38
+//line app/vmselect/prometheus/query_range_response.qtpl:37
+	qw422016.N().DL(qs.ExecutionTimeMsec.Load())
+//line app/vmselect/prometheus/query_range_response.qtpl:37
 	qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_range_response.qtpl:40
 	qt.Printf("generate /api/v1/query_range response for series=%d, points=%d", seriesCount, pointsCount)

app/vmselect/prometheus/query_response.qtpl

@@ -36,7 +36,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
 			// It cannot be converted to int without breaking backwards compatibility at vmalert :(
 		%}
 		"seriesFetched": "{%dl qs.SeriesFetched.Load() %}",
-		"executionTimeMsec": {%dl qs.ExecutionDuration.Load().Milliseconds() %}
+		"executionTimeMsec": {%dl qs.ExecutionTimeMsec.Load() %}
 	}
 	{% code
 		qt.Printf("generate /api/v1/query response for series=%d", seriesCount)

app/vmselect/prometheus/query_response.qtpl.go

@@ -81,9 +81,9 @@ func StreamQueryResponse(qw422016 *qt422016.Writer, rs []netstorage.Result, qt *
 	qw422016.N().DL(qs.SeriesFetched.Load())
 //line app/vmselect/prometheus/query_response.qtpl:38
 	qw422016.N().S(`","executionTimeMsec":`)
-//line app/vmselect/prometheus/query_response.qtpl:40
-	qw422016.N().DL(qs.ExecutionDuration.Load().Milliseconds())
-//line app/vmselect/prometheus/query_response.qtpl:40
+//line app/vmselect/prometheus/query_response.qtpl:39
+	qw422016.N().DL(qs.ExecutionTimeMsec.Load())
+//line app/vmselect/prometheus/query_response.qtpl:39
 	qw422016.N().S(`}`)
 //line app/vmselect/prometheus/query_response.qtpl:42
 	qt.Printf("generate /api/v1/query response for series=%d", seriesCount)

app/vmselect/prometheus/tsdb_status_response.qtpl

@@ -11,7 +11,7 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb .
 	"data":{
 		"totalSeries": {%dul= status.TotalSeries %},
 		"totalLabelValuePairs": {%dul= status.TotalLabelValuePairs %},
-		"seriesCountByMetricName":{%= tsdbStatusMetricNameEntries(status.SeriesCountByMetricName,status.SeriesQueryStatsByMetricName) %},
+		"seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %},
 		"seriesCountByLabelName":{%= tsdbStatusEntries(status.SeriesCountByLabelName) %},
 		"seriesCountByFocusLabelValue":{%= tsdbStatusEntries(status.SeriesCountByFocusLabelValue) %},
 		"seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %},
@@ -34,32 +34,4 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb .
 ]
 {% endfunc %}
 
-{% func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) %}
-{% code
-  queryStatsByMetricName := make(map[string]storage.MetricNamesStatsRecord,len(queryStats))
-  for _, record := range queryStats{
-      queryStatsByMetricName[record.MetricName] = record
-    }
-%}
-[
-	{% for i, e := range a %}
-		{
-      {% code
-      entry, ok := queryStatsByMetricName[e.Name]
-      %}
-			"name":{%q= e.Name %},
-      {% if !ok %}
-			"value":{%d= int(e.Count) %}
-      {% else %}
-			"value":{%d= int(e.Count) %},
-			"requestsCount":{%d= int(entry.RequestsCount) %},
-			"lastRequestTimestamp":{%d= int(entry.LastRequestTs) %}
-      {% endif %}
-		}
-		{% if i+1 < len(a) %},{% endif %}
-	{% endfor %}
-]
-{% endfunc %}
-
-
 {% endstripspace %}

app/vmselect/prometheus/tsdb_status_response.qtpl.go

@@ -37,9 +37,9 @@ func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBSta
 	qw422016.N().DUL(status.TotalLabelValuePairs)
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:13
 	qw422016.N().S(`,"seriesCountByMetricName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
-	streamtsdbStatusMetricNameEntries(qw422016, status.SeriesCountByMetricName, status.SeriesQueryStatsByMetricName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
 	qw422016.N().S(`,"seriesCountByLabelName":`)
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:15
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelName)
@@ -147,89 +147,3 @@ func tsdbStatusEntries(a []storage.TopHeapEntry) string {
 	return qs422016
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:35
 }
-
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:38
-func streamtsdbStatusMetricNameEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:40
-	queryStatsByMetricName := make(map[string]storage.MetricNamesStatsRecord, len(queryStats))
-	for _, record := range queryStats {
-		queryStatsByMetricName[record.MetricName] = record
-	}
-
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:44
-	qw422016.N().S(`[`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:46
-	for i, e := range a {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:46
-		qw422016.N().S(`{`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:49
-		entry, ok := queryStatsByMetricName[e.Name]
-
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:50
-		qw422016.N().S(`"name":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
-		qw422016.N().Q(e.Name)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
-		qw422016.N().S(`,`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:52
-		if !ok {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:52
-			qw422016.N().S(`"value":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:53
-			qw422016.N().D(int(e.Count))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:54
-		} else {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:54
-			qw422016.N().S(`"value":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
-			qw422016.N().D(int(e.Count))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
-			qw422016.N().S(`,"requestsCount":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:56
-			qw422016.N().D(int(entry.RequestsCount))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:56
-			qw422016.N().S(`,"lastRequestTimestamp":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:57
-			qw422016.N().D(int(entry.LastRequestTs))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:58
-		}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:58
-		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
-		if i+1 < len(a) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
-		}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:61
-	}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:61
-	qw422016.N().S(`]`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-}
-
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-func writetsdbStatusMetricNameEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	streamtsdbStatusMetricNameEntries(qw422016, a, queryStats)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-}
-
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	writetsdbStatusMetricNameEntries(qb422016, a, queryStats)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
-}

app/vmselect/promql/eval.go

@@ -172,6 +172,30 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	return &ec
 }
 
+// QueryStats contains various stats for the query.
+type QueryStats struct {
+	// SeriesFetched contains the number of series fetched from storage during the query evaluation.
+	SeriesFetched atomic.Int64
+
+	// ExecutionTimeMsec contains the number of milliseconds the query took to execute.
+	ExecutionTimeMsec atomic.Int64
+}
+
+func (qs *QueryStats) addSeriesFetched(n int) {
+	if qs == nil {
+		return
+	}
+	qs.SeriesFetched.Add(int64(n))
+}
+
+func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
+	if qs == nil {
+		return
+	}
+	d := time.Since(startTime).Milliseconds()
+	qs.ExecutionTimeMsec.Add(d)
+}
+
 func (ec *EvalConfig) validate() {
 	if ec.Start > ec.End {
 		logger.Panicf("BUG: start cannot exceed end; got %d vs %d", ec.Start, ec.End)
@@ -1697,13 +1721,12 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	if err != nil {
 		return nil, err
 	}
-	qs := ec.QueryStats
 	rssLen := rss.Len()
 	if rssLen == 0 {
 		rss.Cancel()
 		return nil, nil
 	}
-	qs.addSeriesFetched(rssLen)
+	ec.QueryStats.addSeriesFetched(rssLen)
 
 	// Verify timeseries fit available memory during rollup calculations.
 	timeseriesLen := rssLen

app/vmselect/promql/query_stats.go

@@ -1,55 +0,0 @@
-package promql
-
-import (
-	"sync/atomic"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
-)
-
-// QueryStats contains various stats of the query evaluation.
-type QueryStats struct {
-	// ExecutionDuration contains the time duration the query took to execute.
-	ExecutionDuration atomic.Pointer[time.Duration]
-	// SeriesFetched contains the number of series fetched from storage or cache.
-	SeriesFetched atomic.Int64
-
-	at *auth.Token
-
-	query     string
-	queryType string
-	start     int64
-	end       int64
-	step      int64
-}
-
-// NewQueryStats creates a new QueryStats object.
-func NewQueryStats(query string, at *auth.Token, ec *EvalConfig) *QueryStats {
-	qs := &QueryStats{
-		at:        at,
-		query:     query,
-		step:      ec.Step,
-		start:     ec.Start,
-		end:       ec.End,
-		queryType: "range",
-	}
-	if qs.start == qs.end {
-		qs.queryType = "instant"
-	}
-	return qs
-}
-
-func (qs *QueryStats) addSeriesFetched(n int) {
-	if qs == nil {
-		return
-	}
-	qs.SeriesFetched.Add(int64(n))
-}
-
-func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
-	if qs == nil {
-		return
-	}
-	d := time.Since(startTime)
-	qs.ExecutionDuration.Store(&d)
-}

app/vmselect/stats/stats.go

@@ -3,7 +3,6 @@ package stats
 import (
 	"fmt"
 	"net/http"
-	"regexp"
 	"strconv"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
@@ -34,12 +33,6 @@ func MetricNamesStatsHandler(qt *querytracer.Tracer, w http.ResponseWriter, r *h
 		le = n
 	}
 	matchPattern := r.FormValue("match_pattern")
-	if len(matchPattern) > 0 {
-		_, err := regexp.Compile(matchPattern)
-		if err != nil {
-			return fmt.Errorf("match_pattern=%q must be valid regex: %w", matchPattern, err)
-		}
-	}
 	stats, err := netstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
 	if err != nil {
 		return err

app/vmselect/vmui/index.html

@@ -36,10 +36,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-BF2w5kzJ.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-BSp13qCn.js">
+  <script type="module" crossorigin src="./assets/index-Bc6E56oa.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-sXHL6qTd.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-u4IOGr0E.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -336,7 +336,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotPath := Storage.MustCreateSnapshot()
+		snapshotPath, err := Storage.CreateSnapshot()
+		if err != nil {
+			err = fmt.Errorf("cannot create snapshot: %w", err)
+			jsonResponseError(w, err)
+			snapshotsCreateErrorsTotal.Inc()
+			return true
+		}
 		if prometheusCompatibleResponse {
 			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotPath))
 		} else {
@@ -346,7 +352,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/list":
 		snapshotsListTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshots := Storage.MustListSnapshots()
+		snapshots, err := Storage.ListSnapshots()
+		if err != nil {
+			err = fmt.Errorf("cannot list snapshots: %w", err)
+			jsonResponseError(w, err)
+			snapshotsListErrorsTotal.Inc()
+			return true
+		}
 		fmt.Fprintf(w, `{"status":"ok","snapshots":[`)
 		if len(snapshots) > 0 {
 			for _, snapshot := range snapshots[:len(snapshots)-1] {
@@ -361,7 +373,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
 
-		snapshots := Storage.MustListSnapshots()
+		snapshots, err := Storage.ListSnapshots()
+		if err != nil {
+			err = fmt.Errorf("cannot list snapshots: %w", err)
+			jsonResponseError(w, err)
+			snapshotsDeleteErrorsTotal.Inc()
+			return true
+		}
 		for _, snName := range snapshots {
 			if snName == snapshotName {
 				if err := Storage.DeleteSnapshot(snName); err != nil {
@@ -375,13 +393,19 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			}
 		}
 
-		err := fmt.Errorf("cannot find snapshot %q", snapshotName)
+		err = fmt.Errorf("cannot find snapshot %q", snapshotName)
 		jsonResponseError(w, err)
 		return true
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshots := Storage.MustListSnapshots()
+		snapshots, err := Storage.ListSnapshots()
+		if err != nil {
+			err = fmt.Errorf("cannot list snapshots: %w", err)
+			jsonResponseError(w, err)
+			snapshotsDeleteAllErrorsTotal.Inc()
+			return true
+		}
 		for _, snapshotName := range snapshots {
 			if err := Storage.DeleteSnapshot(snapshotName); err != nil {
 				err = fmt.Errorf("cannot delete snapshot %q: %w", snapshotName, err)
@@ -415,7 +439,10 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 				return
 			case <-t.C:
 			}
-			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
+			if err := strg.DeleteStaleSnapshots(snapshotsMaxAgeDur); err != nil {
+				// Use logger.Errorf instead of logger.Fatalf in the hope the error is temporary.
+				logger.Errorf("cannot delete stale snapshots: %s", err)
+			}
 		}
 	}()
 }
@@ -433,9 +460,11 @@ var (
 var (
 	activeForceMerges = metrics.NewCounter("vm_active_force_merges")
 
-	snapshotsCreateTotal = metrics.NewCounter(`vm_http_requests_total{path="/snapshot/create"}`)
+	snapshotsCreateTotal       = metrics.NewCounter(`vm_http_requests_total{path="/snapshot/create"}`)
+	snapshotsCreateErrorsTotal = metrics.NewCounter(`vm_http_request_errors_total{path="/snapshot/create"}`)
 
-	snapshotsListTotal = metrics.NewCounter(`vm_http_requests_total{path="/snapshot/list"}`)
+	snapshotsListTotal       = metrics.NewCounter(`vm_http_requests_total{path="/snapshot/list"}`)
+	snapshotsListErrorsTotal = metrics.NewCounter(`vm_http_request_errors_total{path="/snapshot/list"}`)
 
 	snapshotsDeleteTotal       = metrics.NewCounter(`vm_http_requests_total{path="/snapshot/delete"}`)
 	snapshotsDeleteErrorsTotal = metrics.NewCounter(`vm_http_request_errors_total{path="/snapshot/delete"}`)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.24.2 AS build-web-stage
+FROM golang:1.24.1 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/Makefile

@@ -1,7 +1,7 @@
 # All these commands must run from repository root.
 
 copy-metricsql-docs:
-	cp docs/victoriametrics/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
+	cp docs/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
 
 vmui-package-base-image:
 	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui

app/vmui/packages/vmui/package.json

@@ -8,21 +8,21 @@
     "@types/lodash.debounce": "^4.0.9",
     "@types/lodash.get": "^4.4.9",
     "@types/qs": "^6.9.18",
-    "@types/react": "^19.1.2",
+    "@types/react": "^19.0.12",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
     "classnames": "^2.5.1",
     "dayjs": "^1.11.13",
     "lodash.debounce": "^4.0.8",
     "lodash.get": "^4.4.2",
-    "marked": "^15.0.8",
+    "marked": "^15.0.7",
     "marked-emoji": "^2.0.0",
-    "preact": "^10.26.5",
+    "preact": "^10.26.4",
     "qs": "^6.14.0",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.5.0",
+    "react-router-dom": "^7.4.0",
     "uplot": "^1.6.32",
-    "vite": "^6.2.6",
+    "vite": "^6.2.3",
     "web-vitals": "^4.2.4"
   },
   "scripts": {
@@ -55,25 +55,25 @@
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6",
     "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
     "@eslint/eslintrc": "^3.3.1",
-    "@eslint/js": "^9.24.0",
+    "@eslint/js": "^9.23.0",
     "@preact/preset-vite": "^2.10.1",
     "@testing-library/jest-dom": "^6.6.3",
     "@testing-library/preact": "^3.2.4",
-    "@types/node": "^22.14.1",
-    "@typescript-eslint/eslint-plugin": "^8.30.1",
-    "@typescript-eslint/parser": "^8.30.1",
+    "@types/node": "^22.13.13",
+    "@typescript-eslint/eslint-plugin": "^8.28.0",
+    "@typescript-eslint/parser": "^8.28.0",
     "cross-env": "^7.0.3",
-    "eslint": "^9.24.0",
-    "eslint-plugin-react": "^7.37.5",
+    "eslint": "^9.23.0",
+    "eslint-plugin-react": "^7.37.4",
     "globals": "^16.0.0",
-    "http-proxy-middleware": "^3.0.5",
-    "jsdom": "^26.1.0",
+    "http-proxy-middleware": "^3.0.3",
+    "jsdom": "^26.0.0",
     "postcss": "^8.5.3",
     "rollup-plugin-visualizer": "^5.14.0",
-    "sass": "^1.86.3",
-    "sass-embedded": "^1.86.3",
-    "typescript": "^5.8.3",
-    "vitest": "^3.1.1",
-    "webpack": "^5.99.5"
+    "sass": "^1.86.0",
+    "sass-embedded": "^1.86.0",
+    "typescript": "^5.8.2",
+    "vitest": "^3.0.9",
+    "webpack": "^5.98.0"
   }
 }

app/vmui/packages/vmui/src/components/Chart/BarHitsChart/hooks/useBarHitsOptions.ts

@@ -1,5 +1,5 @@
 import { useMemo, useState } from "preact/compat";
-import { getAxes, getMinMaxBuffer, handleDestroy, setSelect } from "../../../../utils/uplot";
+import { getAxes, handleDestroy, setSelect } from "../../../../utils/uplot";
 import dayjs from "dayjs";
 import { dateFromSeconds, formatDateForNativeInput } from "../../../../utils/time";
 import uPlot, { AlignedData, Band, Options, Series } from "uplot";
@@ -9,7 +9,6 @@ import { MinMax, SetMinMax } from "../../../../types";
 import { LogHits } from "../../../../api/types";
 import getSeriesPaths from "../../../../utils/uplot/paths";
 import { GraphOptions, GRAPH_STYLES } from "../types";
-import { getMaxFromArray } from "../../../../utils/math";
 
 const seriesColors = [
   "color-log-hits-bar-1",
@@ -45,12 +44,6 @@ export const getLabelFromLogHit = (logHit: LogHits) => {
   return fields.map((value) => value || "\"\"").join(", ");
 };
 
-const getYRange = (u: uPlot, _initMin = 0, initMax = 1) => {
-  const maxValues = u.series.filter(({ scale }) => scale === "y").map(({ max }) => max || initMax);
-  const max = getMaxFromArray(maxValues);
-  return getMinMaxBuffer(0, max || initMax);
-};
-
 const useBarHitsOptions = ({
   data,
   logHits,
@@ -106,9 +99,6 @@ const useBarHitsOptions = ({
       x: {
         time: true,
         range: () => [xRange.min, xRange.max]
-      },
-      y: {
-        range: getYRange
       }
     },
     hooks: {

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -12,6 +12,7 @@ import Timezones from "./Timezones/Timezones";
 import ThemeControl from "../ThemeControl/ThemeControl";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useBoolean from "../../../hooks/useBoolean";
+import SwitchMarkdownParsing from "../LogsSettings/MarkdownParsing/SwitchMarkdownParsing";
 import { APP_TYPE_LOGS } from "../../../constants/appType";
 
 const title = "Settings";
@@ -57,6 +58,10 @@ const GlobalSettings: FC = () => {
         onClose={handleClose}
       />
     },
+    {
+      show: APP_TYPE_LOGS,
+      component: <SwitchMarkdownParsing/>
+    },
     {
       show: true,
       component: <Timezones ref={timezoneSettingRef}/>

app/vmui/packages/vmui/src/components/Configurators/LogsSettings/LogParsingSwitches.tsx

@@ -1,57 +0,0 @@
-import React, { FC } from "preact/compat";
-import Switch from "../../Main/Switch/Switch";
-import useDeviceDetect from "../../../hooks/useDeviceDetect";
-import { useLogsDispatch, useLogsState } from "../../../state/logsPanel/LogsStateContext";
-
-const LogParsingSwitches: FC = () => {
-  const { isMobile } = useDeviceDetect();
-  const { markdownParsing, ansiParsing } = useLogsState();
-  const dispatch = useLogsDispatch();
-
-  const handleChangeMarkdownParsing = (val: boolean) => {
-    dispatch({ type: "SET_MARKDOWN_PARSING", payload: val });
-
-    if (ansiParsing) {
-      dispatch({ type: "SET_ANSI_PARSING", payload: false });
-    }
-  };
-
-  const handleChangeAnsiParsing = (val: boolean) => {
-    dispatch({ type: "SET_ANSI_PARSING", payload: val });
-
-    if (markdownParsing) {
-      dispatch({ type: "SET_MARKDOWN_PARSING", payload: false });
-    }
-  };
-
-  return (
-    <>
-      <div className="vm-group-logs-configurator-item">
-        <Switch
-          label={"Enable markdown parsing"}
-          value={markdownParsing}
-          onChange={handleChangeMarkdownParsing}
-          fullWidth={isMobile}
-        />
-        <div className="vm-group-logs-configurator-item__info">
-          Toggle this switch to enable or disable the Markdown formatting for log entries.
-          Enabling this will parse log texts to Markdown.
-        </div>
-      </div>
-      <div className="vm-group-logs-configurator-item">
-        <Switch
-          label={"Enable ANSI parsing"}
-          value={ansiParsing}
-          onChange={handleChangeAnsiParsing}
-          fullWidth={isMobile}
-        />
-        <div className="vm-group-logs-configurator-item__info">
-          Toggle this switch to enable or disable ANSI escape sequence parsing for log entries.
-          Enabling this will interpret ANSI codes to render colored log output.
-        </div>
-      </div>
-    </>
-  );
-};
-
-export default LogParsingSwitches;

app/vmui/packages/vmui/src/components/Configurators/LogsSettings/MarkdownParsing/SwitchMarkdownParsing.tsx

@@ -0,0 +1,35 @@
+import React, { FC } from "preact/compat";
+import Switch from "../../../Main/Switch/Switch";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import { useLogsDispatch, useLogsState } from "../../../../state/logsPanel/LogsStateContext";
+
+const SwitchMarkdownParsing: FC = () => {
+  const { isMobile } = useDeviceDetect();
+  const { markdownParsing } = useLogsState();
+  const dispatch = useLogsDispatch();
+
+
+  const handleChangeMarkdownParsing = (val: boolean) => {
+    dispatch({ type: "SET_MARKDOWN_PARSING", payload: val });
+  };
+
+  return (
+    <div>
+      <div className="vm-server-configurator__title">
+        Markdown Parsing for Logs
+      </div>
+      <Switch
+        label={markdownParsing ? "Disable markdown parsing" : "Enable markdown parsing"}
+        value={markdownParsing}
+        onChange={handleChangeMarkdownParsing}
+        fullWidth={isMobile}
+      />
+      <div className="vm-server-configurator__info">
+        Toggle this switch to enable or disable the Markdown formatting for log entries.
+        Enabling this will parse log texts to Markdown.
+      </div>
+    </div>
+  );
+};
+
+export default SwitchMarkdownParsing;

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -1,11 +1,11 @@
 import React, { FC, useState, useEffect, useMemo, useCallback } from "preact/compat";
 import Autocomplete from "../../Main/Autocomplete/Autocomplete";
 import { useFetchQueryOptions } from "../../../hooks/useFetchQueryOptions";
+import { escapeRegexp, hasUnclosedQuotes } from "../../../utils/regexp";
 import useGetMetricsQL from "../../../hooks/useGetMetricsQL";
 import { QueryContextType } from "../../../types";
 import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete";
 import { QueryEditorAutocompleteProps } from "./QueryEditor";
-import { getExprLastPart, getValueByContext, getContext } from "./autocompleteUtils";
 
 const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   value,
@@ -26,7 +26,11 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     return { beforeCursor, afterCursor };
   }, [value, caretPosition]);
 
-  const exprLastPart = useMemo(() => getExprLastPart(values.beforeCursor), [values]);
+  const exprLastPart = useMemo(() => {
+    const regexpSplit = /\s(or|and|unless|default|ifnot|if|group_left|group_right)\s|}|\+|\|-|\*|\/|\^/i;
+    const parts = values.beforeCursor.split(regexpSplit);
+    return parts[parts.length - 1];
+  }, [values]);
 
   const metric = useMemo(() => {
     const regex1 = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
@@ -50,9 +54,44 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     return match ? match[match.length - 1] : "";
   }, [exprLastPart]);
 
-  const context = useMemo(() => getContext(values.beforeCursor, metric, label), [values, metric, label]);
+  const shouldSuppressAutoSuggestion = (value: string) => {
+    const pattern = /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right|by|without|on|ignoring)\b)/i;
+    const parts = value.split(/\s+/);
+    const partsCount = parts.length;
+    const lastPart = parts[partsCount - 1];
+    const preLastPart = parts[partsCount - 2];
 
-  const valueByContext = useMemo(() => getValueByContext(values.beforeCursor), [values.beforeCursor]);
+    const hasEmptyPartAndQuotes = !lastPart && hasUnclosedQuotes(value);
+    const suppressPreLast = (!lastPart || parts.length > 1) && !pattern.test(preLastPart);
+    return hasEmptyPartAndQuotes || suppressPreLast;
+  };
+
+  const context = useMemo(() => {
+    const valueBeforeCursor = values.beforeCursor.trim();
+    const endOfClosedBrackets = ["}", ")"].some(char => valueBeforeCursor.endsWith(char));
+    const endOfClosedQuotes = !hasUnclosedQuotes(valueBeforeCursor) && ["`", "'", "\""].some(char => valueBeforeCursor.endsWith(char));
+    if (!values.beforeCursor || endOfClosedBrackets || endOfClosedQuotes || shouldSuppressAutoSuggestion(values.beforeCursor)) {
+      return QueryContextType.empty;
+    }
+
+    const labelRegexp = /(?:by|without|on|ignoring)\s*\(\s*[^)]*$|\{[^}]*$/i;
+    const patternLabelValue = `(${escapeRegexp(metric)})?{?.+${escapeRegexp(label)}(=|!=|=~|!~)"?([^"]*)$`;
+    const labelValueRegexp = new RegExp(patternLabelValue, "g");
+
+    switch (true) {
+      case labelValueRegexp.test(values.beforeCursor):
+        return QueryContextType.labelValue;
+      case labelRegexp.test(values.beforeCursor):
+        return QueryContextType.label;
+      default:
+        return QueryContextType.metricsql;
+    }
+  }, [values, metric, label]);
+
+  const valueByContext = useMemo(() => {
+    const wordMatch = values.beforeCursor.match(/([\w_.:]+(?![},]))$/);
+    return wordMatch ? wordMatch[0] : "";
+  }, [values.beforeCursor]);
 
   const { metrics, labels, labelValues, loading } = useFetchQueryOptions({
     valueByContext,

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/autocompleteUtils.test.ts

@@ -1,87 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { getContext, getExprLastPart, getValueByContext } from "./autocompleteUtils";
-
-// Mock QueryContextType enum
-const QueryContextType = {
-  empty: "empty",
-  metricsql: "metricsql",
-  label: "label",
-  labelValue: "labelValue",
-};
-
-describe("autocompleteUtils", () => {
-  describe("getExprLastPart", () => {
-    const tests = [
-      { input: "", expected: "" },
-      { input: "metric", expected: "metric" },
-      { input: "metric1 + metric2", expected: "metric2" },
-      { input: "rate(proc", expected: "proc" },
-      { input: "sum(rate(proc", expected: "proc" },
-      { input: "rate(metric{label=\"value\",", expected: "metric{label=\"value\"," },
-      { input: "quantile_over_time(0.9, me", expected: "me" },
-      { input: "quantile_over_time(0.9, metric{label=", expected: "metric{label=" },
-      { input: "quantile_over_time(0.9, metric{label=\"value\",", expected: "metric{label=\"value\"," },
-      { input: "quantile_over_time(0.9, metric{label=\"value\"}", expected: "" },
-      { input: "sum by (instance) (rate(proc", expected: "proc" },
-      { input: "metric{label=", expected: "metric{label=" },
-      { input: "metric{label1=\"value1\",label2=", expected: "metric{label1=\"value1\",label2=" },
-      { input: "quantile_over_time(0.9, node_cpu", expected: "node_cpu" },
-      { input: "sum(max_over_time(rate(node_cpu", expected: "node_cpu" },
-      { input: "clamp_min(metric1, m", expected: "m" },
-    ];
-
-    tests.forEach(({ input, expected }) => {
-      it(`should return "${expected}" for input "${input}"`, () => {
-        const result = getExprLastPart(input);
-        expect(result).toBe(expected);
-      });
-    });
-  });
-
-  describe("getValueByContext", () => {
-    const tests = [
-      { input: "", expected: "" },
-      { input: "metric", expected: "metric" },
-      { input: "metric1 + metric2", expected: "metric2" },
-      { input: "rate(proc", expected: "proc" },
-      { input: "sum(rate(proc", expected: "proc" },
-      { input: "quantile_over_time(0.9, node_cpu", expected: "node_cpu" },
-      { input: "clamp_min(metric1, m", expected: "m" },
-    ];
-
-    tests.forEach(({ input, expected }) => {
-      it(`should return "${expected}" for input "${input}"`, () => {
-        const result = getValueByContext(input);
-        expect(result).toBe(expected);
-      });
-    });
-  });
-
-  describe("getContext", () => {
-    const tests = [
-      { input: { beforeCursor: "" }, expected: QueryContextType.empty },
-      { input: { beforeCursor: "metric" }, expected: QueryContextType.metricsql },
-      { input: { beforeCursor: "rate(proc" }, expected: QueryContextType.metricsql },
-      { input: { beforeCursor: "sum(rate(proc" }, expected: QueryContextType.metricsql },
-      { input: { beforeCursor: "sum by (instance) (rate(proc" }, expected: QueryContextType.metricsql },
-      { input: { beforeCursor: "metric{" }, expected: QueryContextType.label },
-      { input: { beforeCursor: "metric{label=" }, expected: QueryContextType.labelValue, metric: "metric", label: "label" },
-      { input: { beforeCursor: "metric{label1=\"value1\",label2=" }, expected: QueryContextType.labelValue, metric: "metric", label: "label2" },
-      { input: { beforeCursor: "sum by (" }, expected: QueryContextType.label },
-      { input: { beforeCursor: "quantile_over_time(0.9, node_cpu" }, expected: QueryContextType.metricsql },
-      { input: { beforeCursor: "sum(max_over_time(rate(node_cpu" }, expected: QueryContextType.metricsql },
-      { input: { beforeCursor: "clamp_min(metric1, m" }, expected: QueryContextType.metricsql },
-      { input: { beforeCursor: "rate(node_cpu_seconds_total)" }, expected: QueryContextType.empty },
-    ];
-
-    tests.forEach((test) => {
-      const { beforeCursor } = test.input;
-      const metric = test.metric || "";
-      const label = test.label || "";
-      it(`should return "${test.expected}" for input "${beforeCursor}"`, () => {
-        const result = getContext(beforeCursor, metric, label);
-        expect(result).toBe(test.expected);
-      });
-    });
-  });
-});

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/autocompleteUtils.ts

@@ -1,112 +0,0 @@
-/**
- * Utility functions for query editor autocomplete functionality
- */
-import { QueryContextType } from "../../../types";
-import { escapeRegexp, hasUnclosedQuotes } from "../../../utils/regexp";
-
-/**
- * Extracts the last part of an expression that's relevant for auto-suggestion
- * @param beforeCursor The text before the cursor position
- * @returns The relevant part of the expression for auto-suggestion
- */
-export function getExprLastPart(beforeCursor: string): string {
-  const regexpSplit =
-    /\s(or|and|unless|default|ifnot|if|group_left|group_right)\s|}|\+|\|-|\*|\/|\^/i;
-  const parts = beforeCursor.split(regexpSplit);
-  const lastPart = parts[parts.length - 1].trim();
-
-  // Check if we're inside a function's parameters
-  const functionRegex = /.*\(([^)]*)$/;
-  const functionMatch = lastPart.match(functionRegex);
-
-  if (functionMatch && functionMatch[1]) {
-    const params = functionMatch[1];
-
-    if (params.lastIndexOf("{") > params.lastIndexOf("}")) {
-      const wordMatch = params.match(/([\w_.:]+)\{[^}]*$/);
-      return wordMatch ? wordMatch[0] : lastPart;
-    }
-
-    const lastCommaPos = params.lastIndexOf(",");
-    if (lastCommaPos !== -1) {
-      return params.substring(lastCommaPos + 1).trim();
-    }
-
-    return params;
-  }
-
-  return lastPart;
-}
-
-/**
- * Extracts the current word or value at the cursor position for auto-suggestion matching
- * @param beforeCursor The text before the cursor position
- * @returns The current word or value at the cursor position
- */
-export function getValueByContext(beforeCursor: string): string {
-  const wordMatch = beforeCursor.match(/([\w_.:]+(?![},]))$/);
-  return wordMatch ? wordMatch[0] : "";
-}
-
-/**
- * Determines if auto-suggestion should be suppressed based on the query
- * @param value The query value
- * @returns Whether auto-suggestion should be suppressed
- */
-export function shouldSuppressAutoSuggestion(value: string): boolean {
-  const pattern =
-    /([{(),+\-*/^]|\b(?:or|and|unless|default|ifnot|if|group_left|group_right|by|without|on|ignoring)\b)/i;
-  const parts = value.split(/\s+/);
-  const partsCount = parts.length;
-  const lastPart = parts[partsCount - 1];
-  const preLastPart = parts[partsCount - 2];
-
-  const hasEmptyPartAndQuotes = !lastPart && hasUnclosedQuotes(value);
-  const suppressPreLast =
-    (!lastPart || parts.length > 1) && !pattern.test(preLastPart);
-  return hasEmptyPartAndQuotes || suppressPreLast;
-}
-
-/**
- * Determines the context type for auto-suggestion based on the query
- * @param beforeCursor The text before the cursor position
- * @param metric The current metric name
- * @param label The current label name
- * @returns The context type for auto-suggestion
- */
-export function getContext(
-  beforeCursor: string,
-  metric: string = "",
-  label: string = ""
-): QueryContextType {
-  const valueBeforeCursor = beforeCursor.trim();
-  const endOfClosedBrackets = ["}", ")"].some((char) =>
-    valueBeforeCursor.endsWith(char)
-  );
-  const endOfClosedQuotes =
-    !hasUnclosedQuotes(valueBeforeCursor) &&
-    ["`", "'", "\""].some((char) => valueBeforeCursor.endsWith(char));
-  if (
-    !valueBeforeCursor ||
-    endOfClosedBrackets ||
-    endOfClosedQuotes ||
-    shouldSuppressAutoSuggestion(valueBeforeCursor)
-  ) {
-    return QueryContextType.empty;
-  }
-
-  const labelRegexp = /(?:by|without|on|ignoring)\s*\(\s*[^)]*$|\{[^}]*$/i;
-  const patternLabelValue = `(${escapeRegexp(metric)})?{?.+${escapeRegexp(
-    label
-  )}(=|!=|=~|!~)"?([^"]*)$`;
-  const labelValueRegexp = new RegExp(patternLabelValue, "g");
-
-  switch (true) {
-    case labelValueRegexp.test(valueBeforeCursor):
-      return QueryContextType.labelValue;
-    case labelRegexp.test(valueBeforeCursor):
-      return QueryContextType.label;
-    default:
-      return QueryContextType.metricsql;
-  }
-}

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -191,7 +191,7 @@ const StepConfigurator: FC = () => {
             </p>
             <p>
               Read more about <Hyperlink
-                href="https://docs.victoriametrics.com/keyconcepts/#range-query"
+                href="https://docs.victoriametrics.com/keyConcepts.html#range-query"
                 text="Range"
               /> and <Hyperlink
                 href="https://docs.victoriametrics.com/keyconcepts/#instant-query"

app/vmui/packages/vmui/src/components/DownloadButton/DownloadButton.tsx

@@ -1,81 +0,0 @@
-import React, { FC, useCallback, useState } from "preact/compat";
-import Tooltip from "../Main/Tooltip/Tooltip";
-import Button from "../Main/Button/Button";
-import { DownloadIcon } from "../Main/Icons";
-import Popper from "../Main/Popper/Popper";
-import { useRef } from "react";
-import "./style.scss";
-import useBoolean from "../../hooks/useBoolean";
-
-interface DownloadButtonProps {
-  title: string;
-  downloadFormatOptions?: string[];
-  onDownload: (format?: string) => void;
-}
-
-const DownloadButton: FC<DownloadButtonProps> = ({ title, downloadFormatOptions, onDownload }) => {
-  const {
-    value: isPopupOpen,
-    setTrue: onOpenPopup,
-    setFalse: onClosePopup,
-  } = useBoolean(false);
-  const downloadButtonRef = useRef<HTMLDivElement>(null);
-  const onDownloadClick = useCallback(() => {
-    if (isPopupOpen) {
-      onClosePopup();
-      return;
-    }
-
-    if (downloadFormatOptions && downloadFormatOptions.length > 0) {
-      onOpenPopup();
-    } else {
-      onDownload();
-      onClosePopup();
-    }
-  }, [onDownload, onClosePopup, isPopupOpen, onOpenPopup]);
-
-  const onDownloadFormatClick = useCallback((event: Event) => {
-    const button = event.currentTarget as HTMLButtonElement;
-    onDownload(button.textContent ?? undefined);
-  }, [onDownload]);
-
-  return (
-    <>
-      <div ref={downloadButtonRef}>
-        <Tooltip
-          title={title}
-        >
-          <Button
-            variant="text"
-            startIcon={<DownloadIcon/>}
-            onClick={onDownloadClick}
-            ariaLabel={title}
-          />
-        </Tooltip>
-      </div>
-      {downloadFormatOptions && downloadFormatOptions.length > 0 && (
-        <Popper
-          open={isPopupOpen}
-          onClose={onClosePopup}
-          buttonRef={downloadButtonRef}
-          placement={"bottom-right"}
-        >
-          {downloadFormatOptions.map((option) =>
-            <div
-              key={option}
-              className={"vm-download-button__format-option"}
-            >
-              <Button
-                variant="text"
-                onClick={onDownloadFormatClick}
-                className={"vm-download-button__format-option-button"}
-              >
-                {option}
-              </Button>
-            </div>)}
-        </Popper>)}
-    </>
-  );
-};
-
-export default DownloadButton;

app/vmui/packages/vmui/src/components/DownloadButton/style.scss

@@ -1,15 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-download-button {
-  &__format-option {
-    padding: 4px;
-    &:first-child {
-      padding-bottom: 0;
-    }
-
-    &-button {
-      width: 100%;
-      justify-content: flex-start;
-    }
-  }
-}

app/vmui/packages/vmui/src/components/LogsConfigurators/GroupLogsConfigurators/GroupLogsConfigurators.tsx

@@ -20,7 +20,6 @@ import {
   WITHOUT_GROUPING
 } from "../../../constants/logs";
 import { getFromStorage, saveToStorage } from "../../../utils/storage";
-import LogParsingSwitches from "../../Configurators/LogsSettings/LogParsingSwitches";
 
 const {
   GROUP_BY,
@@ -226,8 +225,6 @@ const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
               </span>
             </div>
 
-            <LogParsingSwitches/>
-
             <div className="vm-group-logs-configurator-item">
               <Switch
                 value={noWrapLines}

app/vmui/packages/vmui/src/components/LogsConfigurators/GroupLogsConfigurators/style.scss

@@ -2,9 +2,9 @@
 
 .vm-group-logs-configurator {
   display: grid;
-  gap: calc($padding-global * 2);
+  gap: calc($padding-large * 2);
   padding: $padding-global 0;
-  max-width: 600px;
+  width: 600px;
 
   &-item {
     display: grid;

app/vmui/packages/vmui/src/components/Main/Button/Button.tsx

@@ -37,9 +37,10 @@ const Button: FC<ButtonProps> = ({
     "vm-button": true,
     [`vm-button_${variant}_${color}`]: true,
     [`vm-button_${size}`]: size,
-    "vm-button_icon_only": (startIcon || endIcon) && !children,
+    "vm-button_icon": (startIcon || endIcon) && !children,
     "vm-button_full-width": fullWidth,
-    "vm-button_with-icons": startIcon || endIcon,
+    "vm-button_with-icon": startIcon || endIcon,
+    "vm-button_disabled": disabled,
     [className || ""]: className
   });
 
@@ -51,7 +52,11 @@ const Button: FC<ButtonProps> = ({
       onClick={onClick}
       onMouseDown={onMouseDown}
     >
-      {startIcon}{children}{endIcon}
+      <>
+        {startIcon && <span className="vm-button__start-icon">{startIcon}</span>}
+        {children && <span>{children}</span>}
+        {endIcon && <span className="vm-button__end-icon">{endIcon}</span>}
+      </>
     </button>
   );
 };

app/vmui/packages/vmui/src/components/Main/Button/style.scss

@@ -3,133 +3,225 @@
 $button-radius: 6px;
 
 .vm-button {
-  display: inline-flex;
-  gap: 6px;
+  position: relative;
+  display: flex;
   align-items: center;
   justify-content: center;
   padding: 6px 14px;
   font-size: $font-size-small;
+  line-height: calc($font-size + 1px);
   font-weight: normal;
-  text-transform: capitalize;
-  color: $color-white;
-  background: transparent;
-  border: 1px solid transparent;
+  min-height: 31px;
   border-radius: $button-radius;
+  color: $color-white;
+  transform-style: preserve-3d;
   cursor: pointer;
+  text-transform: uppercase;
   user-select: none;
   white-space: nowrap;
-  transition: transform 0.2s, opacity 0.2s, background-color 0.2s;
 
-  &:focus-visible {
-    outline: 2px solid #006FEEFF;
-    outline-offset: 2px;
+  &:hover:after {
+    background-color: $color-hover-black;
   }
 
-  &:active {
-    transform: scale(0.97);
-    opacity: 0.9;
+  &:before,
+  &:after {
+    content: '';
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+    transition: background-color 200ms ease;
+    border-radius: $button-radius;
   }
 
-  &:disabled {
+  &:before {
+    transform: translateZ(-2px);
+  }
+
+  &:after {
+    background-color: transparent;
+    transform: translateZ(-1px);
+  }
+
+  &:active:after {
+    transform: scale(0.9);
+  }
+
+  span {
+    display: grid;
+    align-items: center;
+    justify-content: center;
+    transform: translateZ(1px);
+
+    svg {
+      width: calc($font-size + 1px);
+    }
+  }
+
+  &__start-icon {
+    margin-right: 6px;
+  }
+
+  &__end-icon {
+    margin-left: 6px;
+  }
+
+  &_disabled {
     opacity: 0.3;
-    pointer-events: none;
+    cursor: not-allowed;
   }
 
-  // Sizes
+  &_icon {
+    padding: 6px $padding-small;
+  }
+
+  &_icon &__start-icon,
+  &_icon &__end-icon {
+    margin: 0;
+  }
+
+  /* size SMALL */
   &_small {
-    padding: 4px 10px;
-    line-height: 14px;
+    padding: 4px 8px;
+    min-height: 25px;
 
-    svg {
-      width: 14px;
-    }
-  }
-
-  &_medium {
-    padding: 6px 14px;
-    line-height: 14px;
-
-    svg {
-      width: 16px;
-    }
-  }
-
-  &_large {
-    padding: 8px 20px;
-    font-size: $font-size-medium;
-
-    svg {
-      width: 18px;
-      line-height: 16px;
-    }
-  }
-
-  // Icon only
-  &_icon_only {
-    aspect-ratio: 1;
-  }
-
-  $icon-sizes: (
-    small: 4px,
-    medium: 6px,
-    large: 8px
-  );
-
-  @each $size, $padding in $icon-sizes {
-    &_#{$size}.vm-button_icon_only {
-      padding: $padding;
-    }
-  }
-
-  // Color variants
-  @mixin contained-button($name, $color, $text-color: $color-white) {
-    &_contained_#{$name} {
-      background-color: $color;
-      color: $text-color;
-
-      &:hover:not(:disabled) {
-        opacity: 0.8;
+    span {
+      svg {
+        width: 13px;
       }
     }
   }
 
-  @mixin outlined-button($name, $border-color, $text-color: $border-color) {
-    &_outlined_#{$name} {
-      background-color: transparent;
-      border-color: $border-color;
-      color: $text-color;
 
-      &:hover:not(:disabled) {
-        background-color: $color-hover-black;
-        opacity: 0.8;
-      }
+  /* variant CONTAINED */
+  &_contained_primary {
+    color: $color-primary-text;
+    background-color: $color-primary;
+
+    &:before {
+      background-color: $color-primary;
+    }
+
+    &:hover:after {
+      background-color: rgba($color-black, 0.2)
     }
   }
 
-  @mixin text-button($name, $text-color: $color-white) {
-    &_text_#{$name} {
-      background-color: transparent;
-      color: $text-color;
+  &_contained_secondary {
+    color: $color-secondary-text;
 
-      &:hover:not(:disabled) {
-        background-color: $color-hover-black;
-      }
+    &:before {
+      background-color: $color-secondary;
+    }
+
+    &:hover:after {
+      background-color: rgba($color-black, 0.2)
     }
   }
 
-  $button-colors: (
-    primary: $color-primary,
-    secondary: $color-secondary,
-    success: $color-success,
-    warning: $color-warning,
-    error: $color-error,
-    gray: $color-text-secondary,
-    white: $color-white
-  );
+  &_contained_success {
+    color: $color-success-text;
 
-  @each $name, $color in $button-colors {
-    @include contained-button($name, $color, if($name == white, $color-black, $color-white));
-    @include outlined-button($name, $color, if($name == white, $color-white, $color));
-    @include text-button($name, if($name == white, $color-white, $color));
+    &:before {
+      background-color: $color-success;
+    }
+
+    &:hover:after {
+      background-color: rgba($color-black, 0.2)
+    }
+  }
+
+  &_contained_error {
+    color: $color-error-text;
+
+    &:before {
+      background-color: $color-error;
+    }
+  }
+
+  &_contained_gray {
+    color: $color-text-secondary;
+
+    &:before {
+      background-color: $color-text-secondary;
+    }
+  }
+
+  &_contained_warning {
+    color: $color-warning;
+
+    &:before {
+      background-color: $color-warning;
+      opacity: 0.2;
+    }
+   }
+
+
+  /* variant TEXT */
+  &_text_primary {
+    color: $color-primary;
+  }
+
+  &_text_secondary {
+    color: $color-secondary;
+  }
+
+  &_text_success {
+    color: $color-success;
+  }
+
+  &_text_error {
+    color: $color-error;
+  }
+
+  &_text_gray {
+    color: $color-text-secondary;
+  }
+
+  &_text_white {
+    color: $color-white;
+  }
+
+  &_text_warning {
+    color: $color-warning;
+  }
+
+
+  /* variant OUTLINED */
+  &_outlined_primary {
+    border: 1px solid $color-primary;
+    color: $color-primary;
+  }
+
+  &_outlined_error {
+    border: 1px solid $color-error;
+    color: $color-error;
+  }
+
+  &_outlined_secondary {
+    border: 1px solid $color-secondary;
+    color: $color-secondary;
+  }
+
+  &_outlined_success {
+    border: 1px solid $color-success;
+    color: $color-success;
+  }
+
+  &_outlined_gray {
+    border: 1px solid $color-text-secondary;
+    color: $color-text-secondary;
+  }
+
+  &_outlined_white {
+    border: 1px solid $color-white;
+    color: $color-white;
+  }
+
+  &_outlined_warning {
+    border: 1px solid $color-warning;
+    color: $color-warning;
   }
 }

app/vmui/packages/vmui/src/components/QueryHistory/utils.test.ts

@@ -1,121 +0,0 @@
-import { describe, it, expect, vi, Mock, afterEach } from "vitest";
-import { getFromStorage, saveToStorage } from "../../utils/storage";
-import { getUpdatedHistory, setQueriesToStorage } from "./utils";
-import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../constants/graph";
-
-vi.mock("../../utils/storage", () => ({
-  getFromStorage: vi.fn(),
-  saveToStorage: vi.fn(),
-}));
-
-describe("utils", () => {
-  afterEach(() => {
-    vi.resetAllMocks();
-  });
-  describe("setQueriesToStorage", () => {
-    it("should not change QUERY_HISTORY ", () => {
-      const getFromStorageMock = getFromStorage as Mock;
-      const saveToStorageMock = saveToStorage as Mock;
-      getFromStorageMock.mockReturnValue(JSON.stringify({
-        "QUERY_HISTORY": [],
-      }));
-
-      setQueriesToStorage("LOGS_QUERY_HISTORY", []);
-      expect(saveToStorageMock).toHaveBeenCalledWith(
-        "LOGS_QUERY_HISTORY",
-        "{\"QUERY_HISTORY\":[[]]}"
-      );
-    });
-
-    it("should not change QUERY_HISTORY cause add the same query", () => {
-      const getFromStorageMock = getFromStorage as Mock;
-      const saveToStorageMock = saveToStorage as Mock;
-      getFromStorageMock.mockReturnValue(JSON.stringify({
-        "QUERY_HISTORY": [["first_query"]],
-      }));
-
-      setQueriesToStorage("LOGS_QUERY_HISTORY", [{ index: 0, values: ["first_query"] }]);
-      expect(saveToStorageMock).toHaveBeenCalledWith(
-        "LOGS_QUERY_HISTORY",
-        "{\"QUERY_HISTORY\":[[\"first_query\"]]}"
-      );
-    });
-
-    it("should add new query to the first position to QUERY_HISTORY", () => {
-      const getFromStorageMock = getFromStorage as Mock;
-      const saveToStorageMock = saveToStorage as Mock;
-      getFromStorageMock.mockReturnValue(JSON.stringify({
-        "QUERY_HISTORY": [["first_query"]],
-      }));
-
-      setQueriesToStorage("LOGS_QUERY_HISTORY", [{ index: 0, values: ["new_query"] }]);
-      expect(saveToStorageMock).toHaveBeenCalledWith(
-        "LOGS_QUERY_HISTORY",
-        "{\"QUERY_HISTORY\":[[\"new_query\",\"first_query\"]]}"
-      );
-    });
-
-    it("should limit the QUERY_HISTORY if add extra query", () => {
-      const getFromStorageMock = getFromStorage as Mock;
-      const saveToStorageMock = saveToStorage as Mock;
-      const maxQueries = MAX_QUERIES_HISTORY * MAX_QUERY_FIELDS;
-      const currentHistory = (new Array(maxQueries)).fill(1).map((_, i) => `${i}_query`);
-      getFromStorageMock.mockReturnValue(JSON.stringify({
-        "QUERY_HISTORY": [currentHistory],
-      }));
-
-      setQueriesToStorage("LOGS_QUERY_HISTORY", [{ index: 0, values: ["extra_query"] }]);
-
-      const calls = saveToStorageMock.mock.calls;
-      const firstCallArgs = calls[0];
-      expect(firstCallArgs[0]).toStrictEqual("LOGS_QUERY_HISTORY");
-      const savedQueries = JSON.parse(firstCallArgs[1]);
-      expect(savedQueries["QUERY_HISTORY"][0][0]).toStrictEqual("extra_query");
-      expect(savedQueries["QUERY_HISTORY"][0].length).toStrictEqual(maxQueries);
-    });
-  });
-
-  describe("getUpdatedHistory", () => {
-    it("should add new query to the end of array", () => {
-      const updatedHistory = getUpdatedHistory("new_query", {
-        index: 2,
-        values: ["first_query", "second_query"]
-      });
-      expect(updatedHistory).toStrictEqual({
-        index: 2,
-        values: [
-          "first_query",
-          "second_query",
-          "new_query",
-        ],
-      });
-    });
-
-    it("should not add new query if the last query is the same", () => {
-      const updatedHistory = getUpdatedHistory("new_query", {
-        index: 2,
-        values: ["first_query", "new_query"]
-      });
-      expect(updatedHistory).toStrictEqual({
-        index: 1,
-        values: [
-          "first_query",
-          "new_query",
-        ],
-      });
-    });
-
-    it("should remove the first query if the maximum number of query is reached", () => {
-      const maxQueries = MAX_QUERIES_HISTORY * MAX_QUERY_FIELDS;
-      const values = (new Array(maxQueries)).fill(1).map((_, i) => `${i}_query`);
-      const updatedHistory = getUpdatedHistory("new_query", {
-        index: 2,
-        values: values
-      });
-      expect(updatedHistory.index).toStrictEqual(maxQueries);
-      expect(updatedHistory.values.length).toStrictEqual(maxQueries);
-      expect(updatedHistory.values[0]).toStrictEqual("1_query");
-      expect(updatedHistory.values[updatedHistory.values.length - 1]).toStrictEqual("new_query");
-    });
-  });
-});

app/vmui/packages/vmui/src/components/QueryHistory/utils.ts

@@ -1,89 +0,0 @@
-import { getFromStorage, removeFromStorage, saveToStorage, StorageKeys } from "../../utils/storage";
-import { QueryHistoryType } from "../../state/query/reducer";
-import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../constants/graph";
-
-export type HistoryKey = Extract<StorageKeys, "LOGS_QUERY_HISTORY" | "METRICS_QUERY_HISTORY">;
-export type HistoryType = "QUERY_HISTORY" | "QUERY_FAVORITES";
-
-const getHistoryFromStorage = (key: HistoryKey) => {
-  const list = getFromStorage(key) as string;
-  const history: Record<HistoryType, string[][]> = list ? JSON.parse(list) : {};
-  return history;
-};
-
-const saveHistoryToStorage = (key: HistoryKey, historyType: HistoryType, history: string[][]) => {
-  const storageHistory = getHistoryFromStorage(key);
-  saveToStorage(key, JSON.stringify({
-    ...storageHistory,
-    [historyType]: history
-  }));
-};
-
-export const getQueriesFromStorage = (key: HistoryKey, historyType: HistoryType) => {
-  return getHistoryFromStorage(key)[historyType] || [];
-};
-
-export const setQueriesToStorage = (key: HistoryKey, history: QueryHistoryType[]) => {
-  // For localStorage, avoid splitting into query fields because when working from multiple tabs can cause confusion.
-  // For convenience, we maintain the original structure of `string[][]`
-  const lastValues = history.map(h => h.values[h.index]);
-  const storageHistory = getHistoryFromStorage(key);
-  const storageValues = storageHistory["QUERY_HISTORY"] || [];
-  if (!storageValues[0]) storageValues[0] = [];
-
-  const values = storageValues[0];
-  const TOTAL_LIMIT = MAX_QUERIES_HISTORY * MAX_QUERY_FIELDS;
-
-  lastValues.forEach((v) => {
-    const already = values.includes(v);
-    if (!already && v) values.unshift(v);
-    if (values.length > TOTAL_LIMIT) values.pop();
-  });
-
-  const newStorageHistory = {
-    ...storageHistory,
-    QUERY_HISTORY: [values]
-  };
-
-  saveToStorage(key, JSON.stringify(newStorageHistory));
-};
-
-export const setFavoriteQueriesToStorage = (key: HistoryKey, favoriteQueries: string[][]) => {
-  saveHistoryToStorage(key, "QUERY_FAVORITES", favoriteQueries);
-};
-
-export const clearQueryHistoryStorage = (key: HistoryKey, historyType: HistoryType) => {
-  const history = getHistoryFromStorage(key);
-  saveToStorage(key, JSON.stringify({
-    ...history,
-    [historyType]: [],
-  }));
-};
-
-export const getUpdatedHistory = (query: string, queryHistory?: QueryHistoryType): QueryHistoryType => {
-  const h = queryHistory || { values: [] };
-  const queryEqual = query === h.values[h.values.length - 1];
-  const newValues = !queryEqual && query ? [...h.values, query] : h.values;
-
-  // limit the history
-  if (newValues.length > MAX_QUERIES_HISTORY) newValues.shift();
-
-  return {
-    index: h.values.length - Number(queryEqual),
-    values: newValues
-  };
-};
-
-const migrateMetricsQueryHistoryToHistoryByKey = () => {
-  const migrateHistory = (type: HistoryType) => {
-    const queryList = getFromStorage(type) as string;
-    if (queryList) {
-      const queryHistory: string[][] = JSON.parse(queryList);
-      saveHistoryToStorage("METRICS_QUERY_HISTORY", type, queryHistory);
-      removeFromStorage([type]);
-    }
-  };
-  migrateHistory("QUERY_HISTORY");
-  migrateHistory("QUERY_FAVORITES");
-};
-migrateMetricsQueryHistoryToHistoryByKey();

app/vmui/packages/vmui/src/components/Table/TableSettings/style.scss

@@ -37,10 +37,6 @@
         overflow: auto;
         margin-bottom: $padding-global;
 
-        @media(max-width: 500px) {
-          width: 100vw;
-        }
-
         &__item {
           width: 100%;
           font-size: $font-size;

app/vmui/packages/vmui/src/components/Table/helpers.test.ts

@@ -1,48 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { descendingComparator } from "./helpers";
-import { getNanoTimestamp } from "../../utils/time"; // используем реальную реализацию
-
-describe("descendingComparator", () => {
-  it("returns 0 for equal numbers", () => {
-    const result = descendingComparator({ value: 42 }, { value: 42 }, "value");
-    expect(result).toBe(0);
-  });
-
-  it("sorts numbers descending", () => {
-    const result = descendingComparator({ value: 100 }, { value: 50 }, "value");
-    expect(result).toBeLessThan(0);
-  });
-
-  it("sorts null below any value", () => {
-    expect(descendingComparator({ value: null }, { value: 10 }, "value")).toBe(1);
-    expect(descendingComparator({ value: 10 }, { value: null }, "value")).toBe(-1);
-    expect(descendingComparator({ value: null }, { value: null }, "value")).toBe(0);
-  });
-
-  it("sorts strings descending", () => {
-    const result = descendingComparator({ name: "zzz" }, { name: "aaa" }, "name");
-    expect(result).toBe(-1);
-  });
-
-  it("sorts numeric strings as numbers when possible", () => {
-    const result = descendingComparator({ value: "200" }, { value: "50" }, "value");
-    expect(result).toBeLessThan(0);
-  });
-
-  it("sorts date strings via getNanoTimestamp", () => {
-    const a = { timestamp: "2024-01-01T00:00:00.200Z" };
-    const b = { timestamp: "2023-01-01T00:00:00.100Z" };
-
-    const nanoA = getNanoTimestamp(a.timestamp);
-    const nanoB = getNanoTimestamp(b.timestamp);
-    expect(nanoA).toBeGreaterThan(nanoB);
-
-    const result = descendingComparator(a, b, "timestamp");
-    expect(result).toBe(-1);
-  });
-
-  it("handles booleans and undefined safely", () => {
-    expect(descendingComparator({ value: true }, { value: false }, "value")).toBe(-1);
-    expect(descendingComparator({ value: undefined }, { value: false }, "value")).toBe(1);
-  });
-});