apptest: rename prometheus from stopwritequerirer

apptest: make linter happy

Makefile

@@ -545,7 +545,7 @@ test-full:
 test-full-386:
 	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
-integration-test: victoria-metrics vmagent vmalert vmauth vmctl
+integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore victoria-logs
 	go test ./apptest/... -skip="^TestCluster.*"
 
 benchmark:

app/victoria-logs/main.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -44,6 +45,8 @@ func main() {
 
 	vlstorage.Init()
 	vlselect.Init()
+
+	insertutil.SetLogRowsStorage(&vlstorage.Storage{})
 	vlinsert.Init()
 
 	go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{

app/vlinsert/datadog/datadog.go

@@ -11,7 +11,6 @@ import (
 	"github.com/valyala/fastjson"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -33,10 +32,10 @@ var parserPool fastjson.ParserPool
 // RequestHandler processes Datadog insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
-	case "/api/v1/validate":
+	case "/insert/datadog/api/v1/validate":
 		fmt.Fprintf(w, `{}`)
 		return true
-	case "/api/v2/logs":
+	case "/insert/datadog/api/v2/logs":
 		return datadogLogsIngestion(w, r)
 	default:
 		return false
@@ -74,7 +73,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
 		cp.IgnoreFields = *datadogIgnoreFields
 	}
 
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return true
 	}

app/vlinsert/elasticsearch/elasticsearch.go

@@ -11,7 +11,6 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -31,36 +30,38 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	// This header is needed for Logstash
 	w.Header().Set("X-Elastic-Product", "Elasticsearch")
 
-	if strings.HasPrefix(path, "/_ilm/policy") {
+	if strings.HasPrefix(path, "/insert/elasticsearch/_ilm/policy") {
 		// Return fake response for Elasticsearch ilm request.
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/_index_template") {
+	if strings.HasPrefix(path, "/insert/elasticsearch/_index_template") {
 		// Return fake response for Elasticsearch index template request.
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/_ingest") {
+	if strings.HasPrefix(path, "/insert/elasticsearch/_ingest") {
 		// Return fake response for Elasticsearch ingest pipeline request.
 		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/_nodes") {
+	if strings.HasPrefix(path, "/insert/elasticsearch/_nodes") {
 		// Return fake response for Elasticsearch nodes discovery request.
 		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
+	if strings.HasPrefix(path, "/insert/elasticsearch/logstash") || strings.HasPrefix(path, "/insert/elasticsearch/_logstash") {
 		// Return fake response for Logstash APIs requests.
 		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
 	switch path {
-	case "/", "":
+	// some clients may omit trailing slash
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
+	case "/insert/elasticsearch/", "/insert/elasticsearch":
 		switch r.Method {
 		case http.MethodGet:
 			// Return fake response for Elasticsearch ping request.
@@ -75,7 +76,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		}
 
 		return true
-	case "/_license":
+	case "/insert/elasticsearch/_license":
 		// Return fake response for Elasticsearch license request.
 		fmt.Fprintf(w, `{
 			"license": {
@@ -86,7 +87,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			}
 		}`)
 		return true
-	case "/_bulk":
+	case "/insert/elasticsearch/_bulk":
 		startTime := time.Now()
 		bulkRequestsTotal.Inc()
 
@@ -95,7 +96,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		if err := vlstorage.CanWriteData(); err != nil {
+		if err := insertutil.CanWriteData(); err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}

app/vlinsert/insertutil/common_params.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -36,6 +35,7 @@ type CommonParams struct {
 	DecolorizeFields []string
 	ExtraFields      []logstorage.Field
 
+	IsTimeFieldSet  bool
 	Debug           bool
 	DebugRequestURI string
 	DebugRemoteAddr string
@@ -49,8 +49,10 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		return nil, err
 	}
 
+	var isTimeFieldSet bool
 	timeFields := []string{"_time"}
 	if tfs := httputil.GetArray(r, "_time_field", "VL-Time-Field"); len(tfs) > 0 {
+		isTimeFieldSet = true
 		timeFields = tfs
 	}
 
@@ -86,9 +88,11 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 		IgnoreFields:     ignoreFields,
 		DecolorizeFields: decolorizeFields,
 		ExtraFields:      extraFields,
-		Debug:            debug,
-		DebugRequestURI:  debugRequestURI,
-		DebugRemoteAddr:  debugRemoteAddr,
+
+		IsTimeFieldSet:  isTimeFieldSet,
+		Debug:           debug,
+		DebugRequestURI: debugRequestURI,
+		DebugRemoteAddr: debugRemoteAddr,
 	}
 
 	return cp, nil
@@ -141,6 +145,29 @@ func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignore
 	return cp
 }
 
+// LogRowsStorage is an interface for ingesting logs into the storage.
+type LogRowsStorage interface {
+	// MustAddRows must add lr to the underlying storage.
+	MustAddRows(lr *logstorage.LogRows)
+
+	// CanWriteData must returns non-nil error if logs cannot be added to the underlying storage.
+	CanWriteData() error
+}
+
+var logRowsStorage LogRowsStorage
+
+// SetLogRowsStorage sets the storage for writing data to via LogMessageProcessor.
+//
+// This function must be called before using LogMessageProcessor and CanWriteData from this package.
+func SetLogRowsStorage(storage LogRowsStorage) {
+	logRowsStorage = storage
+}
+
+// CanWriteData returns non-nil error if data cannot be written to the underlying storage.
+func CanWriteData() error {
+	return logRowsStorage.CanWriteData()
+}
+
 // LogMessageProcessor is an interface for log message processors.
 type LogMessageProcessor interface {
 	// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
@@ -264,7 +291,7 @@ func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
 // flushLocked must be called under locked lmp.mu.
 func (lmp *logMessageProcessor) flushLocked() {
 	lmp.lastFlushTime = time.Now()
-	vlstorage.MustAddRows(lmp.lr)
+	logRowsStorage.MustAddRows(lmp.lr)
 	lmp.lr.ResetKeepSettings()
 }
 

app/vlinsert/insertutil/line_reader.go

@@ -56,6 +56,7 @@ func NewLineReader(name string, r io.Reader) *LineReader {
 // Check for Err in this case.
 func (lr *LineReader) NextLine() bool {
 	for {
+		lr.Line = nil
 		if lr.bufOffset >= len(lr.buf) {
 			if lr.err != nil || lr.eofReached {
 				return false

app/vlinsert/insertutil/line_reader_test.go

@@ -24,6 +24,9 @@ func TestLineReader_Success(t *testing.T) {
 		if lr.NextLine() {
 			t.Fatalf("expecting error on the second call to NextLine()")
 		}
+		if len(lr.Line) > 0 {
+			t.Fatalf("unexpected non-empty line after failed NextLine(): %q", lr.Line)
+		}
 		if !reflect.DeepEqual(lines, linesExpected) {
 			t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
 		}

app/vlinsert/insertutil/timestamp.go

@@ -38,7 +38,10 @@ func ExtractTimestampFromFields(timeFields []string, fields []logstorage.Field)
 }
 
 func parseTimestamp(s string) (int64, error) {
-	if s == "" || s == "0" {
+	// "-" is a nil timestamp value, if the syslog
+	// application is incapable of obtaining system time
+	// https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
+	if s == "" || s == "0" || s == "-" {
 		return time.Now().UnixNano(), nil
 	}
 	if len(s) <= len("YYYY") || s[len("YYYY")] != '-' {

app/vlinsert/insertutil/timestamp_test.go

@@ -133,6 +133,33 @@ func TestExtractTimestampFromFields_Success(t *testing.T) {
 	}, 1718773640000000000)
 }
 
+func TestExtractTimestampFromFields_Now(t *testing.T) {
+	f := func(timeField string, fields []logstorage.Field) {
+		t.Helper()
+
+		nsecs, err := ExtractTimestampFromFields([]string{timeField}, fields)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if nsecs < 1 {
+			t.Fatalf("expected generated timestamp, got error: %s", err)
+		}
+	}
+
+	// RFC5424 allows `-` for nil timestamp (log ingestion time)
+	f("time", []logstorage.Field{
+		{Name: "time", Value: "-"},
+	})
+
+	f("time", []logstorage.Field{
+		{Name: "time", Value: ""},
+	})
+
+	f("time", []logstorage.Field{
+		{Name: "time", Value: "0"},
+	})
+}
+
 func TestExtractTimestampFromFields_Error(t *testing.T) {
 	f := func(s string) {
 		t.Helper()

app/vlinsert/internalinsert/internalinsert.go

@@ -8,7 +8,6 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -40,7 +39,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}

app/vlinsert/journald/journald.go

@@ -3,29 +3,48 @@ package journald
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
-	"regexp"
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 // See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
-const journaldEntryMaxNameLen = 64
+const maxFieldNameLen = 64
 
-var allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
+func isValidJournaldFieldName(s string) bool {
+	if len(s) == 0 {
+		return false
+	}
+	c := s[0]
+	if !(c >= 'A' && c <= 'Z' || c == '_') {
+		return false
+	}
+
+	for i := 1; i < len(s); i++ {
+		c := s[i]
+		if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
+			return false
+		}
+	}
+	return true
+}
 
 var (
 	journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
@@ -36,9 +55,7 @@ var (
 		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
 	journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
 		"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
-	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
-
-	maxRequestSize = flagutil.NewBytes("journald.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single journald request")
+	journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include Journald fields with double underscore prefixes")
 )
 
 func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
@@ -53,11 +70,12 @@ func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
 		}
 		cp.TenantID = tenantID
 	}
-	if len(cp.TimeFields) == 0 {
+
+	if !cp.IsTimeFieldSet {
 		cp.TimeFields = []string{*journaldTimeField}
 	}
 	if len(cp.StreamFields) == 0 {
-		cp.StreamFields = *journaldStreamFields
+		cp.StreamFields = getStreamFields()
 	}
 	if len(cp.IgnoreFields) == 0 {
 		cp.IgnoreFields = *journaldIgnoreFields
@@ -66,10 +84,23 @@ func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
 	return cp, nil
 }
 
+func getStreamFields() []string {
+	if len(*journaldStreamFields) > 0 {
+		return *journaldStreamFields
+	}
+	return defaultStreamFields
+}
+
+var defaultStreamFields = []string{
+	"_MACHINE_ID",
+	"_HOSTNAME",
+	"_SYSTEMD_UNIT",
+}
+
 // RequestHandler processes Journald Export insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
-	case "/upload":
+	case "/insert/journald/upload":
 		if r.Header.Get("Content-Type") != "application/vnd.fdo.journal" {
 			httpserver.Errorf(w, r, "only application/vnd.fdo.journal encoding is supported for Journald")
 			return true
@@ -84,7 +115,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 // handleJournald parses Journal binary entries
 func handleJournald(r *http.Request, w http.ResponseWriter) {
 	startTime := time.Now()
-	requestsJournaldTotal.Inc()
+	requestsTotal.Inc()
 
 	cp, err := getCommonParams(r)
 	if err != nil {
@@ -93,19 +124,25 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
 		return
 	}
 
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		errorsTotal.Inc()
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
 
 	encoding := r.Header.Get("Content-Encoding")
-	err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
-		lmp := cp.NewLogMessageProcessor("journald", false)
-		err := parseJournaldRequest(data, lmp, cp)
-		lmp.MustClose()
-		return err
-	})
+	reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
+	if err != nil {
+		errorsTotal.Inc()
+		logger.Errorf("cannot decode journald request: %s", err)
+		return
+	}
+
+	lmp := cp.NewLogMessageProcessor("journald", true)
+	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
+	err = processStreamInternal(streamName, reader, lmp, cp)
+	protoparserutil.PutUncompressedReader(reader)
+	lmp.MustClose()
 	if err != nil {
 		errorsTotal.Inc()
 		httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
@@ -117,102 +154,180 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
 	// See https://github.com/systemd/systemd/pull/34822
 	w.Header().Set("Accept-Encoding", "zstd")
 
-	// update requestJournaldDuration only for successfully parsed requests
-	// There is no need in updating requestJournaldDuration for request errors,
+	// update requestDuration only for successfully parsed requests
+	// There is no need in updating requestDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.
-	requestJournaldDuration.UpdateDuration(startTime)
+	requestDuration.UpdateDuration(startTime)
 }
 
 var (
-	requestsJournaldTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
-	errorsTotal           = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
-
-	requestJournaldDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
+	requestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
+	errorsTotal     = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
+	requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
 )
 
+func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
+	wcr := writeconcurrencylimiter.GetReader(r)
+	defer writeconcurrencylimiter.PutReader(wcr)
+
+	lr := insertutil.NewLineReader("journald", wcr)
+
+	for {
+		err := readJournaldLogEntry(streamName, lr, lmp, cp)
+		wcr.DecConcurrency()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+			return fmt.Errorf("%s: %w", streamName, err)
+		}
+	}
+}
+
+type fieldsBuf struct {
+	fields []logstorage.Field
+
+	buf   []byte
+	name  []byte
+	value []byte
+}
+
+func (fb *fieldsBuf) reset() {
+	fb.fields = fb.fields[:0]
+	fb.buf = fb.buf[:0]
+	fb.name = fb.name[:0]
+	fb.value = fb.value[:0]
+}
+
+func (fb *fieldsBuf) addField(name, value string) {
+	bufLen := len(fb.buf)
+	fb.buf = append(fb.buf, name...)
+	nameCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
+
+	bufLen = len(fb.buf)
+	fb.buf = append(fb.buf, value...)
+	valueCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
+
+	fb.fields = append(fb.fields, logstorage.Field{
+		Name:  nameCopy,
+		Value: valueCopy,
+	})
+}
+
+func getFieldsBuf() *fieldsBuf {
+	fb := fieldsBufPool.Get()
+	if fb == nil {
+		return &fieldsBuf{}
+	}
+	return fb.(*fieldsBuf)
+}
+
+func putFieldsBuf(fb *fieldsBuf) {
+	fb.reset()
+	fieldsBufPool.Put(fb)
+}
+
+var fieldsBufPool sync.Pool
+
+// readJournaldLogEntry reads a single log entry in Journald format.
+//
 // See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
-func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
-	var fields []logstorage.Field
+func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
 	var ts int64
-	var size uint64
 	var name, value string
-	var line []byte
 
-	currentTimestamp := time.Now().UnixNano()
+	fb := getFieldsBuf()
+	defer putFieldsBuf(fb)
 
-	for len(data) > 0 {
-		idx := bytes.IndexByte(data, '\n')
-		switch {
-		case idx > 0:
-			// process fields
-			line = data[:idx]
-			data = data[idx+1:]
-		case idx == 0:
-			// next message or end of file
-			// double new line is a separator for the next message
-			if len(fields) > 0 {
+	if !lr.NextLine() {
+		if err := lr.Err(); err != nil {
+			return fmt.Errorf("cannot read the first field: %w", err)
+		}
+		return io.EOF
+	}
+
+	for {
+		line := lr.Line
+		if len(line) == 0 {
+			// The end of a single log entry. Write it to the storage
+			if len(fb.fields) > 0 {
 				if ts == 0 {
-					ts = currentTimestamp
+					ts = time.Now().UnixNano()
 				}
-				lmp.AddRow(ts, fields, nil)
-				fields = fields[:0]
+				lmp.AddRow(ts, fb.fields, nil)
 			}
-			// skip newline separator
-			data = data[1:]
-			continue
-		case idx < 0:
-			return fmt.Errorf("missing new line separator, unread data left=%d", len(data))
+			return nil
 		}
 
-		idx = bytes.IndexByte(line, '=')
-		// could b either e key=value\n pair
-		// or just  key\n
-		// with binary data at the buffer
-		if idx > 0 {
-			name = bytesutil.ToUnsafeString(line[:idx])
-			value = bytesutil.ToUnsafeString(line[idx+1:])
+		// line could be either "key=value\n" or "key\n<little_endian_size_64>value\n"
+		// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
+		if n := bytes.IndexByte(line, '='); n >= 0 {
+			// "key=value\n"
+			fb.name = append(fb.name[:0], line[:n]...)
+			name = bytesutil.ToUnsafeString(fb.name)
+
+			fb.value = append(fb.value[:0], line[n+1:]...)
+			value = bytesutil.ToUnsafeString(fb.value)
 		} else {
-			name = bytesutil.ToUnsafeString(line)
-			if len(data) == 0 {
-				return fmt.Errorf("unexpected zero data for binary field value of key=%s", name)
+			// "key\n<little_endian_size_64>value\n"
+			fb.name = append(fb.name[:0], line...)
+			name = bytesutil.ToUnsafeString(fb.name)
+
+			fb.value = fb.value[:0]
+			for len(fb.value) < 8 {
+				if !lr.NextLine() {
+					if err := lr.Err(); err != nil {
+						return fmt.Errorf("cannot read value size: %w", err)
+					}
+					return fmt.Errorf("unexpected end of stream while reading value size")
+				}
+				fb.value = append(fb.value, lr.Line...)
+				fb.value = append(fb.value, '\n')
 			}
-			// size of binary data encoded as le i64 at the begging
-			idx, err := binary.Decode(data, binary.LittleEndian, &size)
-			if err != nil {
-				return fmt.Errorf("failed to extract binary field %q value size: %w", name, err)
+			size := binary.LittleEndian.Uint64(fb.value[:8])
+
+			for size > uint64(len(fb.value[8:])) {
+				if !lr.NextLine() {
+					if err := lr.Err(); err != nil {
+						return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
+					}
+					return fmt.Errorf("unexpected end of stream while reading %q value with size %d bytes; read only %d bytes", fb.name, size, len(fb.value[8:]))
+				}
+				fb.value = append(fb.value, lr.Line...)
+				fb.value = append(fb.value, '\n')
 			}
-			// skip binary data size
-			data = data[idx:]
-			if size == 0 {
-				return fmt.Errorf("unexpected zero binary data size decoded %d", size)
+			value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
+			if uint64(len(value)) != size {
+				return fmt.Errorf("unexpected %q value size; got %d bytes; want %d bytes; value: %q", fb.name, len(value), size, value)
 			}
-			if int(size) > len(data) {
-				return fmt.Errorf("binary data size=%d cannot exceed size of the data at buffer=%d", size, len(data))
-			}
-			value = bytesutil.ToUnsafeString(data[:size])
-			data = data[int(size):]
-			// binary data must has new line separator for the new line or next field
-			if len(data) == 0 {
-				return fmt.Errorf("unexpected empty buffer after binary field=%s read", name)
-			}
-			lastB := data[0]
-			if lastB != '\n' {
-				return fmt.Errorf("expected new line separator after binary field=%s, got=%s", name, string(lastB))
-			}
-			data = data[1:]
 		}
-		if len(name) > journaldEntryMaxNameLen {
-			return fmt.Errorf("journald entry name should not exceed %d symbols, got: %q", journaldEntryMaxNameLen, name)
+
+		if !lr.NextLine() {
+			if err := lr.Err(); err != nil {
+				return fmt.Errorf("cannot read the next log field: %w", err)
+			}
+
+			// add the last log field below before the return
 		}
-		if !allowedJournaldEntryNameChars.MatchString(name) {
-			return fmt.Errorf("journald entry name should consist of `A-Z0-9_` characters and must start from non-digit symbol")
+
+		if len(name) > maxFieldNameLen {
+			logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
+			continue
 		}
+		if !isValidJournaldFieldName(name) {
+			logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
+			continue
+		}
+
 		if slices.Contains(cp.TimeFields, name) {
-			n, err := strconv.ParseInt(value, 10, 64)
+			t, err := strconv.ParseInt(value, 10, 64)
 			if err != nil {
-				return fmt.Errorf("failed to parse Journald timestamp, %w", err)
+				logger.Errorf("%s: cannot parse timestamp from the field %q: %w; using the current timestamp", streamName, name, err)
+				ts = 0
+			} else {
+				// Convert journald microsecond timestamp to nanoseconds
+				ts = t * 1e3
 			}
-			ts = n * 1e3
 			continue
 		}
 
@@ -220,18 +335,32 @@ func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *i
 			name = "_msg"
 		}
 
-		if *journaldIncludeEntryMetadata || !strings.HasPrefix(name, "__") {
-			fields = append(fields, logstorage.Field{
-				Name:  name,
-				Value: value,
-			})
+		if name == "PRIORITY" {
+			priority := journaldPriorityToLevel(value)
+			fb.addField("level", priority)
+		}
+
+		if !strings.HasPrefix(name, "__") || *journaldIncludeEntryMetadata {
+			fb.addField(name, value)
 		}
 	}
-	if len(fields) > 0 {
-		if ts == 0 {
-			ts = currentTimestamp
-		}
-		lmp.AddRow(ts, fields, nil)
-	}
-	return nil
+}
+
+func journaldPriorityToLevel(priority string) string {
+	// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
+	// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
+	switch priority {
+	case "0", "1", "2":
+		return "critical"
+	case "3":
+		return "error"
+	case "4":
+		return "warning"
+	case "5", "6":
+		return "info"
+	case "7":
+		return "debug"
+	default:
+		return priority
+	}
 }

app/vlinsert/journald/journald_test.go

@@ -1,20 +1,81 @@
 package journald
 
 import (
+	"bytes"
+	"net/http"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
 )
 
-func TestPushJournaldOk(t *testing.T) {
+func TestIsValidJournaldFieldName(t *testing.T) {
+	f := func(name string, resultExpected bool) {
+		t.Helper()
+
+		result := isValidJournaldFieldName(name)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
+		}
+	}
+
+	f("", false)
+	f("a", false)
+	f("1", false)
+	f("_", true)
+	f("X", true)
+	f("Xa", false)
+	f("X_343", true)
+	f("X_0123456789_AZ", true)
+	f("SDDFD sdf", false)
+}
+
+func TestGetCommonParams_TimeField(t *testing.T) {
+	f := func(timeFieldHeader, expectedTimeField string) {
+		t.Helper()
+
+		req, err := http.NewRequest("POST", "/insert/journald/upload", nil)
+		if err != nil {
+			t.Fatalf("unexpected error creating request: %s", err)
+		}
+
+		if timeFieldHeader != "" {
+			req.Header.Set("VL-Time-Field", timeFieldHeader)
+		}
+
+		cp, err := getCommonParams(req)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		if len(cp.TimeFields) != 1 || cp.TimeFields[0] != expectedTimeField {
+			t.Fatalf("unexpected TimeFields; got %v; want [%s]", cp.TimeFields, expectedTimeField)
+		}
+	}
+
+	// Test default behavior - when no custom time field is specified, journald uses __REALTIME_TIMESTAMP
+	f("", "__REALTIME_TIMESTAMP")
+
+	// Test custom time field - when a custom time field is specified via HTTP header, it's respected
+	f("custom_time", "custom_time")
+}
+
+func TestPushJournald_Success(t *testing.T) {
 	f := func(src string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
+
 		tlp := &insertutil.TestLogMessageProcessor{}
-		cp := &insertutil.CommonParams{
-			TimeFields: []string{"__REALTIME_TIMESTAMP"},
-			MsgFields:  []string{"MESSAGE"},
+
+		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
+		if err != nil {
+			t.Fatalf("cannot create request: %s", err)
 		}
-		if err := parseJournaldRequest([]byte(src), tlp, cp); err != nil {
+		cp, err := getCommonParams(r)
+		if err != nil {
+			t.Fatalf("cannot create commonParams: %s", err)
+		}
+
+		buf := bytes.NewBufferString(src)
+		if err := processStreamInternal("test", buf, tlp, cp); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 
@@ -22,16 +83,17 @@ func TestPushJournaldOk(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+
 	// Single event
-	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n",
+	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n",
 		[]int64{91723819283000},
 		"{\"_msg\":\"Test message\"}",
 	)
 
 	// Multiple events
-	f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
+	f("__REALTIME_TIMESTAMP=91723819283\nPRIORITY=3\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
 		[]int64{91723819283000, 91723819284000},
-		"{\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
+		"{\"level\":\"error\",\"PRIORITY\":\"3\",\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
 	)
 
 	// Parse binary data
@@ -39,30 +101,45 @@ func TestPushJournaldOk(t *testing.T) {
 		[]int64{1729698775704404000},
 		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
 	)
+
+	// Empty field name must be ignored
+	f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
+	f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
+
+	// field name starting with number must be ignored
+	f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
+
+	// field name exceeding 64 bytes limit must be ignored
+	f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n", nil, "")
+
+	// field name with invalid chars must be ignored
+	f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n", nil, "")
 }
 
 func TestPushJournald_Failure(t *testing.T) {
 	f := func(data string) {
 		t.Helper()
+
 		tlp := &insertutil.TestLogMessageProcessor{}
-		cp := &insertutil.CommonParams{
-			TimeFields: []string{"__REALTIME_TIMESTAMP"},
-			MsgFields:  []string{"MESSAGE"},
+
+		r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
+		if err != nil {
+			t.Fatalf("cannot create request: %s", err)
 		}
-		if err := parseJournaldRequest([]byte(data), tlp, cp); err == nil {
-			t.Fatalf("expected non nil error")
+		cp, err := getCommonParams(r)
+		if err != nil {
+			t.Fatalf("cannot create commonParams: %s", err)
+		}
+
+		buf := bytes.NewBufferString(data)
+		if err := processStreamInternal("test", buf, tlp, cp); err == nil {
+			t.Fatalf("expecting non-nil error")
 		}
 	}
-	// missing new line terminator for binary encoded message
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda2")
-	// missing new line terminator
-	f("__REALTIME_TIMESTAMP=91723819283\n=Test message")
-	// empty field name
-	f("__REALTIME_TIMESTAMP=91723819283\n=Test message\n")
-	// field name starting with number
-	f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n")
-	// field name exceeds 64 limit
-	f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n")
-	// Only allow A-Z0-9 and '_'
-	f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n")
+
+	// too short binary encoded message
+	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda")
+
+	// too long binary encoded message
+	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")
 }

app/vlinsert/journald/journald_timing_test.go

@@ -0,0 +1,62 @@
+package journald
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
+)
+
+func generateJournaldData(size int) []byte {
+	var buf []byte
+	timestamp := time.Now().UnixMicro()
+	binaryMsg := []byte("binary message data for performance test")
+	var sizeBuf [8]byte
+
+	for len(buf) < size {
+		timestamp++
+
+		var entry string
+		// Generate a mix of simple and binary messages
+		if timestamp%10 == 0 {
+			// Generate binary message
+			binary.LittleEndian.PutUint64(sizeBuf[:], uint64(len(binaryMsg)))
+			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE\n%s%s\n\n",
+				timestamp,
+				sizeBuf[:],
+				binaryMsg,
+			)
+		} else {
+			// Generate simple message
+			entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE=Performance test message %d\n\n", timestamp, timestamp)
+		}
+		buf = append(buf, entry...)
+	}
+	return buf
+}
+
+func BenchmarkPushJournaldPerformance(b *testing.B) {
+	cp := &insertutil.CommonParams{
+		TimeFields: []string{"__REALTIME_TIMESTAMP"},
+		MsgFields:  []string{"MESSAGE"},
+	}
+	const dataChunkSize = 1024 * 1024
+
+	data := generateJournaldData(dataChunkSize)
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(data)))
+	b.RunParallel(func(pb *testing.PB) {
+		r := &bytes.Reader{}
+		blp := &insertutil.BenchmarkLogMessageProcessor{}
+		for pb.Next() {
+			r.Reset(data)
+			if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
+				panic(fmt.Errorf("unexpected error: %w", err))
+			}
+		}
+	})
+}

app/vlinsert/jsonline/jsonline.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -33,7 +32,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}

app/vlinsert/loki/loki.go

@@ -16,10 +16,10 @@ var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whet
 // RequestHandler processes Loki insert requests
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
-	case "/api/v1/push":
+	case "/insert/loki/api/v1/push":
 		handleInsert(r, w)
 		return true
-	case "/ready":
+	case "/insert/loki/ready":
 		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte("ready"))

app/vlinsert/loki/loki_json.go

@@ -9,7 +9,6 @@ import (
 	"github.com/valyala/fastjson"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -30,7 +29,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
 	}
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}

app/vlinsert/loki/loki_protobuf.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -29,7 +28,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
 	}
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}

app/vlinsert/main.go

@@ -58,35 +58,29 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func insertHandler(w http.ResponseWriter, r *http.Request, path string) bool {
-	path = strings.TrimPrefix(path, "/insert")
-
 	switch path {
-	case "/jsonline":
+	case "/insert/jsonline":
 		jsonline.RequestHandler(w, r)
 		return true
-	case "/ready":
+	case "/insert/ready":
 		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(200)
 		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	}
 	switch {
-	case strings.HasPrefix(path, "/elasticsearch"):
-		// some clients may omit trailing slash
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
-		path = strings.TrimPrefix(path, "/elasticsearch")
+	// some clients may omit trailing slash at elasticsearch protocol.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
+	case strings.HasPrefix(path, "/insert/elasticsearch"):
 		return elasticsearch.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/loki/"):
-		path = strings.TrimPrefix(path, "/loki")
+
+	case strings.HasPrefix(path, "/insert/loki/"):
 		return loki.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/opentelemetry/"):
-		path = strings.TrimPrefix(path, "/opentelemetry")
+	case strings.HasPrefix(path, "/insert/opentelemetry/"):
 		return opentelemetry.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/journald/"):
-		path = strings.TrimPrefix(path, "/journald")
+	case strings.HasPrefix(path, "/insert/journald/"):
 		return journald.RequestHandler(path, w, r)
-	case strings.HasPrefix(path, "/datadog/"):
-		path = strings.TrimPrefix(path, "/datadog")
+	case strings.HasPrefix(path, "/insert/datadog/"):
 		return datadog.RequestHandler(path, w, r)
 	}
 

app/vlinsert/opentelemetry/opentelemetry.go

@@ -6,7 +6,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -22,7 +21,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 	switch path {
 	// use the same path as opentelemetry collector
 	// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
-	case "/v1/logs":
+	case "/insert/opentelemetry/v1/logs":
 		if r.Header.Get("Content-Type") == "application/json" {
 			httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 			return true
@@ -43,7 +42,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return
 	}
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		httpserver.Errorf(w, r, "%s", err)
 		return
 	}

app/vlinsert/syslog/syslog.go

@@ -17,7 +17,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -385,7 +384,7 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us
 
 // processStream parses a stream of syslog messages from r and ingests them into vlstorage.
 func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutil.CommonParams) error {
-	if err := vlstorage.CanWriteData(); err != nil {
+	if err := insertutil.CanWriteData(); err != nil {
 		return err
 	}
 

app/vlinsert/syslog/syslog_test.go

@@ -101,8 +101,8 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	currentYear := 2023
 	timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
 	resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
-{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
-{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
+{"priority":"165","facility_keyword":"local4","level":"info","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
+{"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
 	f(data, currentYear, timestampsExpected, resultExpected)
 }
 

app/vlselect/vmui/index.html

@@ -35,10 +35,10 @@
   <meta property="og:title" content="UI for VictoriaLogs">
   <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
   <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-BaRvaPfA.js"></script>
+  <script type="module" crossorigin src="./assets/index-DhqzKCNf.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vlstorage/main.go

@@ -253,8 +253,11 @@ func processForceFlush(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+// Storage implements insertutil.LogRowsStorage interface
+type Storage struct{}
+
 // CanWriteData returns non-nil error if it cannot write data to vlstorage
-func CanWriteData() error {
+func (*Storage) CanWriteData() error {
 	if localStorage == nil {
 		// The data can be always written in non-local mode.
 		return nil
@@ -273,7 +276,7 @@ func CanWriteData() error {
 // MustAddRows adds lr to vlstorage
 //
 // It is advised to call CanWriteData() before calling MustAddRows()
-func MustAddRows(lr *logstorage.LogRows) {
+func (*Storage) MustAddRows(lr *logstorage.LogRows) {
 	if localStorage != nil {
 		// Store lr in the local storage.
 		localStorage.MustAddRows(lr)

app/vmalert/rule/alerting.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"hash/fnv"
+	"math"
 	"sort"
 	"strings"
 	"sync"
@@ -335,7 +336,9 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 	var result []prompbmarshal.TimeSeries
 	holdAlertState := make(map[uint64]*notifier.Alert)
 	qFn := func(_ string) ([]datasource.Metric, error) {
-		return nil, fmt.Errorf("`query` template isn't supported in replay mode")
+		logger.Warnf("`query` template isn't supported in replay mode, mocked data is used")
+		//  mock query results to allow common used template {{ query <$expr> | first | value }}
+		return []datasource.Metric{{Timestamps: []int64{0}, Values: []float64{math.NaN()}}}, nil
 	}
 	for _, s := range res.Data {
 		ls, as, err := ar.expandTemplates(s, qFn, time.Time{})
@@ -413,7 +416,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
-	ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
+	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
 	qFn := func(query string) ([]datasource.Metric, error) {
 		res, _, err := ar.q.Query(ctx, query, ts)
 		return res.Data, err

app/vmalert/web.qtpl

@@ -162,7 +162,7 @@
                             <thead>
                                 <tr>
                                     <th scope="col" style="width: 60%">Rule</th>
-                                    <th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
+                                    <th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
                                     <th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
                                 </tr>
                             </thead>
@@ -594,7 +594,7 @@
             <thead>
                 <tr>
                     <th scope="col" title="The time when event was created">Updated at</th>
-                    <th scope="col" style="width: 10%" class="text-center" title="How many samples were returned">Samples</th>
+                    <th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
                     {% if seriesFetchedEnabled %}<th scope="col" style="width: 10%" class="text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
                     <th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th>
                     <th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>

app/vmalert/web.qtpl.go

@@ -524,7 +524,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []apiGr
                             <thead>
                                 <tr>
                                     <th scope="col" style="width: 60%">Rule</th>
-                                    <th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
+                                    <th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
                                     <th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
                                 </tr>
                             </thead>
@@ -1697,7 +1697,7 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule apiRule)
             <thead>
                 <tr>
                     <th scope="col" title="The time when event was created">Updated at</th>
-                    <th scope="col" style="width: 10%" class="text-center" title="How many samples were returned">Samples</th>
+                    <th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
                     `)
 //line app/vmalert/web.qtpl:598
 	if seriesFetchedEnabled {

app/vmctl/remote_read_test.go

@@ -1,351 +0,0 @@
-package main
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/prometheus/prometheus/prompb"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-)
-
-func TestRemoteRead(t *testing.T) {
-	barpool.Disable(true)
-	defer func() {
-		barpool.Disable(false)
-	}()
-	defer func() { isSilent = false }()
-
-	var testCases = []struct {
-		name             string
-		remoteReadConfig remoteread.Config
-		vmCfg            vm.Config
-		start            string
-		end              string
-		numOfSamples     int64
-		numOfSeries      int64
-		rrp              remoteReadProcessor
-		chunk            string
-		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
-		expectedSeries   []vm.TimeSeries
-	}{
-		{
-			name:             "step minute on minute time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
-			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
-			start:            "2022-11-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMinute,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{0, 0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{100, 100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{200, 200},
-				},
-			},
-		},
-		{
-			name:             "step month on month time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
-			vmCfg: vm.Config{
-				Addr:        "",
-				Concurrency: 1,
-				Transport:   httputil.NewTransport(false, "vmctl_test_read"),
-			},
-			start:            "2022-09-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMonth,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{200},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{200}},
-			},
-		},
-	}
-
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx := context.Background()
-			remoteReadServer := remote_read_integration.NewRemoteReadServer(t)
-			defer remoteReadServer.Close()
-			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
-			defer remoteWriteServer.Close()
-
-			tt.remoteReadConfig.Addr = remoteReadServer.URL()
-
-			rr, err := remoteread.NewClient(tt.remoteReadConfig)
-			if err != nil {
-				t.Fatalf("error create remote read client: %s", err)
-			}
-
-			start, err := time.Parse(time.RFC3339, tt.start)
-			if err != nil {
-				t.Fatalf("Error parse start time: %s", err)
-			}
-
-			end, err := time.Parse(time.RFC3339, tt.end)
-			if err != nil {
-				t.Fatalf("Error parse end time: %s", err)
-			}
-
-			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
-
-			remoteReadServer.SetRemoteReadSeries(rrs)
-			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
-
-			tt.vmCfg.Addr = remoteWriteServer.URL()
-
-			b, err := backoff.New(10, 1.8, time.Second*2)
-			if err != nil {
-				t.Fatalf("failed to create backoff: %s", err)
-			}
-			tt.vmCfg.Backoff = b
-
-			importer, err := vm.NewImporter(ctx, tt.vmCfg)
-			if err != nil {
-				t.Fatalf("failed to create VM importer: %s", err)
-			}
-			defer importer.Close()
-
-			rmp := remoteReadProcessor{
-				src: rr,
-				dst: importer,
-				filter: remoteReadFilter{
-					timeStart: &start,
-					timeEnd:   &end,
-					chunk:     tt.chunk,
-				},
-				cc:        1,
-				isVerbose: false,
-			}
-
-			err = rmp.run(ctx)
-			if err != nil {
-				t.Fatalf("failed to run remote read processor: %s", err)
-			}
-		})
-	}
-}
-
-func TestSteamRemoteRead(t *testing.T) {
-	barpool.Disable(true)
-	defer func() {
-		barpool.Disable(false)
-	}()
-	defer func() { isSilent = false }()
-
-	var testCases = []struct {
-		name             string
-		remoteReadConfig remoteread.Config
-		vmCfg            vm.Config
-		start            string
-		end              string
-		numOfSamples     int64
-		numOfSeries      int64
-		rrp              remoteReadProcessor
-		chunk            string
-		remoteReadSeries func(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries
-		expectedSeries   []vm.TimeSeries
-	}{
-		{
-			name:             "step minute on minute time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
-			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
-			start:            "2022-11-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMinute,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{0, 0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{100, 100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1669454585000, 1669454615000},
-					Values:     []float64{200, 200},
-				},
-			},
-		},
-		{
-			name:             "step month on month time range",
-			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*", UseStream: true},
-			vmCfg:            vm.Config{Addr: "", Concurrency: 1},
-			start:            "2022-09-26T11:23:05+02:00",
-			end:              "2022-11-26T11:24:05+02:00",
-			numOfSamples:     2,
-			numOfSeries:      3,
-			chunk:            stepper.StepMonth,
-			remoteReadSeries: remote_read_integration.GenerateRemoteReadSeries,
-			expectedSeries: []vm.TimeSeries{
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1664184185000},
-					Values:     []float64{200},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{0},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{100},
-				},
-				{
-					Name:       "vm_metric_1",
-					LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-					Timestamps: []int64{1666819415000},
-					Values:     []float64{200}},
-			},
-		},
-	}
-
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			ctx := context.Background()
-			remoteReadServer := remote_read_integration.NewRemoteReadStreamServer(t)
-			defer remoteReadServer.Close()
-			remoteWriteServer := remote_read_integration.NewRemoteWriteServer(t)
-			defer remoteWriteServer.Close()
-
-			tt.remoteReadConfig.Addr = remoteReadServer.URL()
-
-			rr, err := remoteread.NewClient(tt.remoteReadConfig)
-			if err != nil {
-				t.Fatalf("error create remote read client: %s", err)
-			}
-
-			start, err := time.Parse(time.RFC3339, tt.start)
-			if err != nil {
-				t.Fatalf("Error parse start time: %s", err)
-			}
-
-			end, err := time.Parse(time.RFC3339, tt.end)
-			if err != nil {
-				t.Fatalf("Error parse end time: %s", err)
-			}
-
-			rrs := tt.remoteReadSeries(start.Unix(), end.Unix(), tt.numOfSeries, tt.numOfSamples)
-
-			remoteReadServer.InitMockStorage(rrs)
-			remoteWriteServer.ExpectedSeries(tt.expectedSeries)
-
-			tt.vmCfg.Addr = remoteWriteServer.URL()
-
-			b, err := backoff.New(10, 1.8, time.Second*2)
-			if err != nil {
-				t.Fatalf("failed to create backoff: %s", err)
-			}
-
-			tt.vmCfg.Backoff = b
-			importer, err := vm.NewImporter(ctx, tt.vmCfg)
-			if err != nil {
-				t.Fatalf("failed to create VM importer: %s", err)
-			}
-			defer importer.Close()
-
-			rmp := remoteReadProcessor{
-				src: rr,
-				dst: importer,
-				filter: remoteReadFilter{
-					timeStart: &start,
-					timeEnd:   &end,
-					chunk:     tt.chunk,
-				},
-				cc:        1,
-				isVerbose: false,
-			}
-
-			err = rmp.run(ctx)
-			if err != nil {
-				t.Fatalf("failed to run remote read processor: %s", err)
-			}
-		})
-	}
-}

app/vmctl/testdata/servers_integration_test/remote_write_server.go

@@ -1,306 +0,0 @@
-package remote_read_integration
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"log"
-	"net/http"
-	"net/http/httptest"
-	"reflect"
-	"sort"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/native/stream"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
-	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/vmimport"
-)
-
-// LabelValues represents series from api/v1/series response
-type LabelValues map[string]string
-
-// Response represents response from api/v1/series
-type Response struct {
-	Status string        `json:"status"`
-	Series []LabelValues `json:"data"`
-}
-
-type MetricNamesResponse struct {
-	Status string   `json:"status"`
-	Data   []string `json:"data"`
-}
-
-// RemoteWriteServer represents fake remote write server with database
-type RemoteWriteServer struct {
-	server         *httptest.Server
-	series         []vm.TimeSeries
-	expectedSeries []vm.TimeSeries
-	tss            []vm.TimeSeries
-}
-
-// NewRemoteWriteServer prepares test remote write server
-func NewRemoteWriteServer(t *testing.T) *RemoteWriteServer {
-	rws := &RemoteWriteServer{series: make([]vm.TimeSeries, 0)}
-	mux := http.NewServeMux()
-
-	mux.Handle("/api/v1/import", rws.getWriteHandler(t))
-	mux.Handle("/health", rws.handlePing())
-	mux.Handle("/api/v1/series", rws.seriesHandler())
-	mux.Handle("/api/v1/label/__name__/values", rws.valuesHandler())
-	mux.Handle("/api/v1/export/native", rws.exportNativeHandler())
-	mux.Handle("/api/v1/import/native", rws.importNativeHandler(t))
-	rws.server = httptest.NewServer(mux)
-	return rws
-}
-
-// Close closes the server
-func (rws *RemoteWriteServer) Close() {
-	rws.server.Close()
-}
-
-// Series saves generated series for fake database
-func (rws *RemoteWriteServer) Series(series []vm.TimeSeries) {
-	rws.series = append(rws.series, series...)
-}
-
-// ExpectedSeries saves expected results to check in the handler
-func (rws *RemoteWriteServer) ExpectedSeries(series []vm.TimeSeries) {
-	rws.expectedSeries = append(rws.expectedSeries, series...)
-}
-
-func (rws *RemoteWriteServer) GetCollectedTimeSeries() []vm.TimeSeries {
-	return rws.tss
-}
-
-// URL returns server url
-func (rws *RemoteWriteServer) URL() string {
-	return rws.server.URL
-}
-
-func (rws *RemoteWriteServer) getWriteHandler(t *testing.T) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		scanner := bufio.NewScanner(r.Body)
-		var rows parser.Rows
-		for scanner.Scan() {
-
-			rows.Unmarshal(scanner.Text())
-			for _, row := range rows.Rows {
-				var labelPairs []vm.LabelPair
-				var ts vm.TimeSeries
-				nameValue := ""
-				for _, tag := range row.Tags {
-					if string(tag.Key) == "__name__" {
-						nameValue = string(tag.Value)
-						continue
-					}
-					labelPairs = append(labelPairs, vm.LabelPair{Name: string(tag.Key), Value: string(tag.Value)})
-				}
-
-				ts.Values = append(ts.Values, row.Values...)
-				ts.Timestamps = append(ts.Timestamps, row.Timestamps...)
-				ts.Name = nameValue
-				ts.LabelPairs = labelPairs
-				rws.tss = append(rws.tss, ts)
-			}
-			rows.Reset()
-		}
-
-		w.WriteHeader(http.StatusNoContent)
-		return
-	})
-}
-
-func (rws *RemoteWriteServer) handlePing() http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		_, _ = w.Write([]byte("OK"))
-	})
-}
-
-func (rws *RemoteWriteServer) seriesHandler() http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		var labelValues []LabelValues
-		for _, ser := range rws.series {
-			metricNames := make(LabelValues)
-			if ser.Name != "" {
-				metricNames["__name__"] = ser.Name
-			}
-			for _, p := range ser.LabelPairs {
-				metricNames[p.Name] = p.Value
-			}
-			labelValues = append(labelValues, metricNames)
-		}
-
-		resp := Response{
-			Status: "success",
-			Series: labelValues,
-		}
-
-		err := json.NewEncoder(w).Encode(resp)
-		if err != nil {
-			log.Printf("error send series: %s", err)
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-	})
-}
-
-func (rws *RemoteWriteServer) valuesHandler() http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		labelNames := make(map[string]struct{})
-		for _, ser := range rws.series {
-			if ser.Name != "" {
-				labelNames[ser.Name] = struct{}{}
-			}
-		}
-
-		metricNames := make([]string, 0, len(labelNames))
-		for k := range labelNames {
-			metricNames = append(metricNames, k)
-		}
-		resp := MetricNamesResponse{
-			Status: "success",
-			Data:   metricNames,
-		}
-
-		buf := bytes.NewBuffer(nil)
-		err := json.NewEncoder(buf).Encode(resp)
-		if err != nil {
-			log.Printf("error send series: %s", err)
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-
-		w.WriteHeader(http.StatusOK)
-		_, err = w.Write(buf.Bytes())
-		if err != nil {
-			log.Printf("error send series: %s", err)
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-		return
-	})
-}
-
-func (rws *RemoteWriteServer) exportNativeHandler() http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		now := time.Now()
-		err := prometheus.ExportNativeHandler(now, w, r)
-		if err != nil {
-			log.Printf("error export series via native protocol: %s", err)
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-
-		return
-	})
-}
-
-func (rws *RemoteWriteServer) importNativeHandler(t *testing.T) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		protoparserutil.StartUnmarshalWorkers()
-		defer protoparserutil.StopUnmarshalWorkers()
-
-		var gotTimeSeries []vm.TimeSeries
-		var mx sync.RWMutex
-
-		err := stream.Parse(r.Body, "", func(block *stream.Block) error {
-			mn := &block.MetricName
-			var timeseries vm.TimeSeries
-			timeseries.Name = string(mn.MetricGroup)
-			timeseries.Timestamps = append(timeseries.Timestamps, block.Timestamps...)
-			timeseries.Values = append(timeseries.Values, block.Values...)
-
-			for i := range mn.Tags {
-				tag := &mn.Tags[i]
-				timeseries.LabelPairs = append(timeseries.LabelPairs, vm.LabelPair{
-					Name:  string(tag.Key),
-					Value: string(tag.Value),
-				})
-			}
-
-			mx.Lock()
-			gotTimeSeries = append(gotTimeSeries, timeseries)
-			mx.Unlock()
-
-			return nil
-		})
-		if err != nil {
-			log.Printf("error parse stream blocks: %s", err)
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-
-		// got timeseries should be sorted
-		// because they are processed independently
-		sort.SliceStable(gotTimeSeries, func(i, j int) bool {
-			iv, jv := gotTimeSeries[i], gotTimeSeries[j]
-			switch {
-			case iv.Values[0] != jv.Values[0]:
-				return iv.Values[0] < jv.Values[0]
-			case iv.Timestamps[0] != jv.Timestamps[0]:
-				return iv.Timestamps[0] < jv.Timestamps[0]
-			default:
-				return iv.Name < jv.Name
-			}
-		})
-
-		if !reflect.DeepEqual(gotTimeSeries, rws.expectedSeries) {
-			w.WriteHeader(http.StatusInternalServerError)
-			t.Fatalf("datasets not equal, expected: %#v;\n got: %#v", rws.expectedSeries, gotTimeSeries)
-		}
-
-		w.WriteHeader(http.StatusNoContent)
-		return
-	})
-}
-
-// GenerateVNSeries generates test timeseries
-func GenerateVNSeries(start, end, numOfSeries, numOfSamples int64) []vm.TimeSeries {
-	var ts []vm.TimeSeries
-	j := 0
-	for i := 0; i < int(numOfSeries); i++ {
-		if i%3 == 0 {
-			j++
-		}
-
-		timeSeries := vm.TimeSeries{
-			Name: fmt.Sprintf("vm_metric_%d", j),
-			LabelPairs: []vm.LabelPair{
-				{Name: "job", Value: strconv.Itoa(i)},
-			},
-		}
-
-		ts = append(ts, timeSeries)
-	}
-
-	for i := range ts {
-		t, v := generateTimeStampsAndValues(i, start, end, numOfSamples)
-		ts[i].Timestamps = t
-		ts[i].Values = v
-	}
-
-	return ts
-}
-
-func generateTimeStampsAndValues(idx int, startTime, endTime, numOfSamples int64) ([]int64, []float64) {
-	delta := (endTime - startTime) / numOfSamples
-
-	var timestamps []int64
-	var values []float64
-	t := startTime
-	for t != endTime {
-		v := 100 * int64(idx)
-		timestamps = append(timestamps, t*1000)
-		values = append(values, float64(v))
-		t = t + delta
-	}
-
-	return timestamps, values
-}

app/vmctl/vm_native_test.go

@@ -1,268 +1,9 @@
 package main
 
 import (
-	"context"
-	"flag"
-	"fmt"
-	"log"
-	"net/http"
-	"os"
 	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
-	remote_read_integration "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/testdata/servers_integration_test"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
-const (
-	storagePath       = "TestStorage"
-	retentionPeriod   = "100y"
-	deleteSeriesLimit = 3e3
-)
-
-func TestVMNativeProcessorRun(t *testing.T) {
-	f := func(startStr, endStr string, numOfSeries, numOfSamples int, resultExpected []vm.TimeSeries) {
-		t.Helper()
-
-		src := remote_read_integration.NewRemoteWriteServer(t)
-		dst := remote_read_integration.NewRemoteWriteServer(t)
-
-		defer func() {
-			src.Close()
-			dst.Close()
-		}()
-
-		start, err := time.Parse(time.RFC3339, startStr)
-		if err != nil {
-			t.Fatalf("cannot parse start time: %s", err)
-		}
-
-		end, err := time.Parse(time.RFC3339, endStr)
-		if err != nil {
-			t.Fatalf("cannot parse end time: %s", err)
-		}
-
-		matchName := "__name__"
-		matchValue := ".*"
-		filter := native.Filter{
-			Match:     fmt.Sprintf("{%s=~%q}", matchName, matchValue),
-			TimeStart: startStr,
-			TimeEnd:   endStr,
-		}
-
-		rws := remote_read_integration.GenerateVNSeries(start.Unix(), end.Unix(), int64(numOfSeries), int64(numOfSamples))
-
-		src.Series(rws)
-		dst.ExpectedSeries(resultExpected)
-
-		if err := fillStorage(rws); err != nil {
-			t.Fatalf("cannot add series to storage: %s", err)
-		}
-
-		tr := httputil.NewTransport(false, "test_client")
-		tr.DisableKeepAlives = false
-
-		srcClient := &native.Client{
-			AuthCfg:     nil,
-			Addr:        src.URL(),
-			ExtraLabels: []string{},
-			HTTPClient: &http.Client{
-				Transport: tr,
-			},
-		}
-		dstClient := &native.Client{
-			AuthCfg:     nil,
-			Addr:        dst.URL(),
-			ExtraLabels: []string{},
-			HTTPClient: &http.Client{
-				Transport: tr,
-			},
-		}
-
-		isSilent = true
-		defer func() { isSilent = false }()
-
-		bf, err := backoff.New(10, 1.8, time.Second*2)
-		if err != nil {
-			t.Fatalf("cannot create backoff: %s", err)
-		}
-
-		p := &vmNativeProcessor{
-			filter:   filter,
-			dst:      dstClient,
-			src:      srcClient,
-			backoff:  bf,
-			cc:       1,
-			isNative: true,
-		}
-
-		ctx := context.Background()
-		if err := p.run(ctx); err != nil {
-			t.Fatalf("run() error: %s", err)
-		}
-		deleted, err := deleteSeries(matchName, matchValue)
-		if err != nil {
-			t.Fatalf("cannot delete series: %s", err)
-		}
-		if deleted != numOfSeries {
-			t.Fatalf("unexpected number of deleted series; got %d; want %d", deleted, numOfSeries)
-		}
-	}
-
-	processFlags()
-	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	defer func() {
-		vmstorage.Stop()
-		if err := os.RemoveAll(storagePath); err != nil {
-			log.Fatalf("cannot remove %q: %s", storagePath, err)
-		}
-	}()
-
-	barpool.Disable(true)
-	defer func() {
-		barpool.Disable(false)
-	}()
-
-	// step minute on minute time range
-	start := "2022-11-25T11:23:05+02:00"
-	end := "2022-11-27T11:24:05+02:00"
-	numOfSeries := 3
-	numOfSamples := 2
-	resultExpected := []vm.TimeSeries{
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-			Timestamps: []int64{1669368185000, 1669454615000},
-			Values:     []float64{0, 0},
-		},
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-			Timestamps: []int64{1669368185000, 1669454615000},
-			Values:     []float64{100, 100},
-		},
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-			Timestamps: []int64{1669368185000, 1669454615000},
-			Values:     []float64{200, 200},
-		},
-	}
-	f(start, end, numOfSeries, numOfSamples, resultExpected)
-
-	// step month on month time range
-	start = "2022-09-26T11:23:05+02:00"
-	end = "2022-11-26T11:24:05+02:00"
-	numOfSeries = 3
-	numOfSamples = 2
-	resultExpected = []vm.TimeSeries{
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-			Timestamps: []int64{1664184185000},
-			Values:     []float64{0},
-		},
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "0"}},
-			Timestamps: []int64{1666819415000},
-			Values:     []float64{0},
-		},
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-			Timestamps: []int64{1664184185000},
-			Values:     []float64{100},
-		},
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "1"}},
-			Timestamps: []int64{1666819415000},
-			Values:     []float64{100},
-		},
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-			Timestamps: []int64{1664184185000},
-			Values:     []float64{200},
-		},
-		{
-			Name:       "vm_metric_1",
-			LabelPairs: []vm.LabelPair{{Name: "job", Value: "2"}},
-			Timestamps: []int64{1666819415000},
-			Values:     []float64{200},
-		},
-	}
-	f(start, end, numOfSeries, numOfSamples, resultExpected)
-}
-
-func processFlags() {
-	flag.Parse()
-	for _, fv := range []struct {
-		flag  string
-		value string
-	}{
-		{flag: "storageDataPath", value: storagePath},
-		{flag: "retentionPeriod", value: retentionPeriod},
-	} {
-		// panics if flag doesn't exist
-		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
-			log.Fatalf("unable to set %q with value %q, err: %v", fv.flag, fv.value, err)
-		}
-	}
-}
-
-func fillStorage(series []vm.TimeSeries) error {
-	var mrs []storage.MetricRow
-	for _, series := range series {
-		var labels []prompbmarshal.Label
-		for _, lp := range series.LabelPairs {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  lp.Name,
-				Value: lp.Value,
-			})
-		}
-		if series.Name != "" {
-			labels = append(labels, prompbmarshal.Label{
-				Name:  "__name__",
-				Value: series.Name,
-			})
-		}
-		mr := storage.MetricRow{}
-		mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels)
-
-		timestamps := series.Timestamps
-		values := series.Values
-		for i, value := range values {
-			mr.Timestamp = timestamps[i]
-			mr.Value = value
-			mrs = append(mrs, mr)
-		}
-	}
-
-	if err := vmstorage.AddRows(mrs); err != nil {
-		return fmt.Errorf("unexpected error in AddRows: %s", err)
-	}
-	vmstorage.Storage.DebugFlush()
-	return nil
-}
-
-func deleteSeries(name, value string) (int, error) {
-	tfs := storage.NewTagFilters()
-	if err := tfs.Add([]byte(name), []byte(value), false, true); err != nil {
-		return 0, fmt.Errorf("unexpected error in TagFilters.Add: %w", err)
-	}
-	return vmstorage.DeleteSeries(nil, []*storage.TagFilters{tfs}, deleteSeriesLimit)
-}
-
 func TestBuildMatchWithFilter_Failure(t *testing.T) {
 	f := func(filter, metricName string) {
 		t.Helper()

app/vmselect/prometheus/prometheus.go

@@ -818,6 +818,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		LookbackDelta:       lookbackDelta,
 		RoundDigits:         getRoundDigits(r),
 		EnforcedTagFilterss: etfs,
+		CacheTagFilters:     etfs,
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
@@ -927,6 +928,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		LookbackDelta:       lookbackDelta,
 		RoundDigits:         getRoundDigits(r),
 		EnforcedTagFilterss: etfs,
+		CacheTagFilters:     etfs,
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},

app/vmselect/promql/eval.go

@@ -140,6 +140,13 @@ type EvalConfig struct {
 	// EnforcedTagFilterss may contain additional label filters to use in the query.
 	EnforcedTagFilterss [][]storage.TagFilter
 
+	// CacheTagFilters stores the original tag-filter sets and extra_label from the request.
+	// The slice is never modified after creation and is used only to build
+	// the query-cache key.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9001
+	CacheTagFilters [][]storage.TagFilter
+
 	// The callback, which returns the request URI during logging.
 	// The request URI isn't stored here because its' construction may take non-trivial amounts of CPU.
 	GetRequestURI func() string
@@ -166,6 +173,7 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.LookbackDelta = src.LookbackDelta
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
+	ec.CacheTagFilters = src.CacheTagFilters
 	ec.GetRequestURI = src.GetRequestURI
 	ec.QueryStats = src.QueryStats
 
@@ -1966,11 +1974,14 @@ func sumNoOverflow(a, b int64) int64 {
 }
 
 func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]float64, []int64) {
-	if *noStaleMarkers || funcName == "default_rollup" || funcName == "stale_samples_over_time" {
+	if *noStaleMarkers || funcName == "stale_samples_over_time" ||
+		funcName == "default_rollup" || funcName == "increase" || funcName == "rate" {
 		// Do not drop Prometheus staleness marks (aka stale NaNs) for default_rollup() function,
 		// since it uses them for Prometheus-style staleness detection.
 		// Do not drop staleness marks for stale_samples_over_time() function, since it needs
 		// to calculate the number of staleness markers.
+		// Do not drop staleness marks for increase() and rate() function, so they could stop
+		// returning results for stale series. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8891
 		return values, timestamps
 	}
 	// Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values.

app/vmselect/promql/rollup.go

@@ -71,7 +71,8 @@ var rollupFuncs = map[string]newRollupFunc{
 	"quantile_over_time":      newRollupQuantile,
 	"quantiles_over_time":     newRollupQuantiles,
 	"range_over_time":         newRollupFuncOneArg(rollupRange),
-	"rate":                    newRollupFuncOneArg(rollupDerivFast), // + rollupFuncsRemoveCounterResets
+	"rate":                    newRollupFuncOneArg(rollupDerivFast),           // + rollupFuncsRemoveCounterResets
+	"rate_prometheus":         newRollupFuncOneArg(rollupDerivFastPrometheus), // + rollupFuncsRemoveCounterResets
 	"rate_over_sum":           newRollupFuncOneArg(rollupRateOverSum),
 	"resets":                  newRollupFuncOneArg(rollupResets),
 	"rollup":                  newRollupFuncOneOrTwoArgs(rollupFake),
@@ -195,7 +196,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"zscore_over_time":        rollupZScoreOverTime,
 }
 
-// VictoriaMetrics can extends lookbehind window for these functions
+// VictoriaMetrics can extend lookbehind window for these functions
 // in order to make sure it contains enough points for returning non-empty results.
 //
 // This is needed for returning the expected non-empty graphs when zooming in the graph in Grafana,
@@ -225,6 +226,7 @@ var rollupFuncsRemoveCounterResets = map[string]bool{
 	"increase_pure":       true,
 	"irate":               true,
 	"rate":                true,
+	"rate_prometheus":     true,
 	"rollup_increase":     true,
 	"rollup_rate":         true,
 }
@@ -252,6 +254,7 @@ var rollupFuncsSamplesScannedPerCall = map[string]int{
 	"lifetime":            2,
 	"present_over_time":   1,
 	"rate":                2,
+	"rate_prometheus":     2,
 	"scrape_interval":     2,
 	"tfirst_over_time":    1,
 	"timestamp":           1,
@@ -913,15 +916,18 @@ func getMaxPrevInterval(scrapeInterval int64) int64 {
 	return scrapeInterval + scrapeInterval/8
 }
 
+// removeCounterResets removes resets for rollup functions over counters - see rollupFuncsRemoveCounterResets
+// it doesn't remove resets between samples with staleNaNs, or samples that exceed maxStalenessInterval
 func removeCounterResets(values []float64, timestamps []int64, maxStalenessInterval int64) {
-	// There is no need in handling NaNs here, since they are impossible
-	// on values from vmstorage.
 	if len(values) == 0 {
 		return
 	}
 	var correction float64
 	prevValue := values[0]
 	for i, v := range values {
+		if decimal.IsStaleNaN(v) {
+			continue
+		}
 		d := v - prevValue
 		if d < 0 {
 			if (-d * 8) < prevValue {
@@ -1853,8 +1859,13 @@ func rollupIncreasePure(rfa *rollupFuncArg) float64 {
 
 func rollupDelta(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
-	// before calling rollup funcs.
+	// before calling rollup funcs. Only StaleNaNs could remain in values - see dropStaleNaNs().
 	values := rfa.values
+	if len(values) > 0 && decimal.IsStaleNaN(values[len(values)-1]) {
+		// if last sample on interval is staleness marker then the selected series is expected
+		// to stop rendering immediately. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8891
+		return nan
+	}
 	prevValue := rfa.prevValue
 	if math.IsNaN(prevValue) {
 		if len(values) == 0 {
@@ -1938,10 +1949,23 @@ func rollupDerivSlow(rfa *rollupFuncArg) float64 {
 	return k
 }
 
+func rollupDerivFastPrometheus(rfa *rollupFuncArg) float64 {
+	delta := rollupDeltaPrometheus(rfa)
+	if math.IsNaN(delta) || rfa.window == 0 {
+		return nan
+	}
+	return delta / (float64(rfa.window) / 1e3)
+}
+
 func rollupDerivFast(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
-	// before calling rollup funcs.
+	// before calling rollup funcs. Only StaleNaNs could remain in values - see  - see dropStaleNaNs().
 	values := rfa.values
+	if len(values) > 0 && decimal.IsStaleNaN(values[len(values)-1]) {
+		// if last sample on interval is staleness marker then the selected series is expected
+		// to stop rendering immediately. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8891
+		return nan
+	}
 	timestamps := rfa.timestamps
 	prevValue := rfa.prevValue
 	prevTimestamp := rfa.prevTimestamp

app/vmselect/promql/rollup_result_cache.go

@@ -291,7 +291,7 @@ func (rrc *rollupResultCache) GetSeries(qt *querytracer.Tracer, ec *EvalConfig,
 	bb := bbPool.Get()
 	defer bbPool.Put(bb)
 
-	bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
+	bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
 	metainfoBuf := rrc.c.Get(nil, bb.B)
 	if len(metainfoBuf) == 0 {
 		qt.Printf("nothing found")
@@ -313,7 +313,7 @@ func (rrc *rollupResultCache) GetSeries(qt *querytracer.Tracer, ec *EvalConfig,
 	if !ok {
 		mi.RemoveKey(key)
 		metainfoBuf = mi.Marshal(metainfoBuf[:0])
-		bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
+		bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
 		rrc.c.Set(bb.B, metainfoBuf)
 		return nil, ec.Start
 	}
@@ -419,7 +419,7 @@ func (rrc *rollupResultCache) PutSeries(qt *querytracer.Tracer, ec *EvalConfig,
 	metainfoBuf := bbPool.Get()
 	defer bbPool.Put(metainfoBuf)
 
-	metainfoKey.B = marshalRollupResultCacheKeyForSeries(metainfoKey.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
+	metainfoKey.B = marshalRollupResultCacheKeyForSeries(metainfoKey.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
 	metainfoBuf.B = rrc.c.Get(metainfoBuf.B[:0], metainfoKey.B)
 	var mi rollupResultCacheMetainfo
 	if len(metainfoBuf.B) > 0 {

app/vmselect/promql/rollup_test.go

@@ -156,6 +156,14 @@ func TestRemoveCounterResets(t *testing.T) {
 	removeCounterResets(values, timestamps, 10)
 	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
 
+	// verify that staleNaNs are respected
+	// it is important to have counter reset in values below to trigger correction logic
+	values = []float64{2, 4, 2, decimal.StaleNaN}
+	timestamps = []int64{10, 20, 30, 40}
+	valuesExpected = []float64{2, 4, 6, decimal.StaleNaN}
+	removeCounterResets(values, timestamps, 10)
+	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
+
 	// verify results always increase monotonically with possible float operations precision error
 	values = []float64{34.094223, 2.7518, 2.140669, 0.044878, 1.887095, 2.546569, 2.490149, 0.045, 0.035684, 0.062454, 0.058296}
 	timestampsExpected = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
@@ -648,6 +656,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 	f("irate", 0)
 	f("outlier_iqr_over_time", nan)
 	f("rate", 2200)
+	f("rate_prometheus", 2200)
 	f("resets", 5)
 	f("range_over_time", 111)
 	f("avg_over_time", 47.083333333333336)
@@ -1525,16 +1534,31 @@ func testRowsEqual(t *testing.T, values []float64, timestamps []int64, valuesExp
 				i, ts, tsExpected, timestamps, timestampsExpected)
 		}
 		vExpected := valuesExpected[i]
+		if decimal.IsStaleNaN(v) {
+			if !decimal.IsStaleNaN(vExpected) {
+				t.Fatalf("unexpected stale NaN value at values[%d]; want %f\nvalues=\n%v\nvaluesExpected=\n%v",
+					i, vExpected, values, valuesExpected)
+			}
+			continue
+		}
+		// staleNaNBits == math.NaN(), but decimal.IsStaleNaN(math.NaN()) == false
+		// so we check for decimal.IsStaleNaN first.
+		if decimal.IsStaleNaN(vExpected) {
+			if !decimal.IsStaleNaN(v) {
+				t.Fatalf("unexpected value at values[%d]; got %f; want stale NaN\nvalues=\n%v\nvaluesExpected=\n%v",
+					i, v, values, valuesExpected)
+			}
+		}
 		if math.IsNaN(v) {
 			if !math.IsNaN(vExpected) {
-				t.Fatalf("unexpected nan value at values[%d]; want %f\nvalues=\n%v\nvaluesExpected=\n%v",
+				t.Fatalf("unexpected NaN value at values[%d]; want %f\nvalues=\n%v\nvaluesExpected=\n%v",
 					i, vExpected, values, valuesExpected)
 			}
 			continue
 		}
 		if math.IsNaN(vExpected) {
 			if !math.IsNaN(v) {
-				t.Fatalf("unexpected value at values[%d]; got %f; want nan\nvalues=\n%v\nvaluesExpected=\n%v",
+				t.Fatalf("unexpected value at values[%d]; got %f; want NaN\nvalues=\n%v\nvaluesExpected=\n%v",
 					i, v, values, valuesExpected)
 			}
 			continue
@@ -1608,6 +1632,33 @@ func TestRollupDelta(t *testing.T) {
 	f(100, nan, nan, nil, 0)
 }
 
+func TestRollupDerivFastPrometheus(t *testing.T) {
+	f := func(values []float64, window int64, resultExpected float64) {
+		t.Helper()
+		rfa := &rollupFuncArg{
+			values: values,
+			window: window,
+		}
+		result := rollupDerivFastPrometheus(rfa)
+		if math.IsNaN(result) {
+			if !math.IsNaN(resultExpected) {
+				t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
+			}
+			return
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
+		}
+	}
+	f(nil, 0, nan)
+	f(nil, 10, nan)
+	f([]float64{0, 10}, 0, nan)
+	f([]float64{10}, 10, nan)
+
+	f([]float64{0, 20}, 10e3, 2)
+	f([]float64{0, 10, 20}, 10e3, 2)
+}
+
 func TestRollupDeltaWithStaleness(t *testing.T) {
 	// there is a gap between samples in the dataset below
 	timestamps := []int64{0, 15000, 30000, 70000}
@@ -1746,6 +1797,28 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		timestampsExpected := []int64{0, 30e3, 60e3, 90e3}
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
+
+	// the last sample is stale NaN
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{0, 0, 0, 10, decimal.StaleNaN}
+	t.Run("last point is stale nan", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupDelta,
+			Start:              40001,
+			End:                40001,
+			Step:               50000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{nan}
+		timestampsExpected := []int64{40001}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
 }
 
 func TestRollupIncreasePureWithStaleness(t *testing.T) {
@@ -1860,3 +1933,48 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
+
+func TestRollupDerivFastWithStaleness(t *testing.T) {
+	timestamps := []int64{0, 10000, 20000, 30000, 40000}
+	values := []float64{0, 0, 0, 0, 10}
+	t.Run("no stale marker", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupDerivFast,
+			Start:              40001,
+			End:                40001,
+			Step:               50000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{0.25}
+		timestampsExpected := []int64{40001}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// the last sample is stale NaN
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{0, 0, 0, 10, decimal.StaleNaN}
+	t.Run("last point is stale nan", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupDerivFast,
+			Start:              40001,
+			End:                40001,
+			Step:               50000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{nan}
+		timestampsExpected := []int64{40001}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}

app/vmselect/vmui/index.html

@@ -36,10 +36,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-xmjGcv4-.js"></script>
+  <script type="module" crossorigin src="./assets/index-D-ssBbZq.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.24.3 AS build-web-stage
+FROM golang:1.24.4 AS build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.21.3
+FROM alpine:3.22.0
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/LogsQL/LogsQueryEditorAutocomplete.tsx

@@ -33,8 +33,12 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     const part = logicalParts.find(p => caretPosition[0] >= p.position[0] && caretPosition[0] <= p.position[1]);
     if (!part) return;
     const cursorStartPosition = caretPosition[0] - part.position[0];
+    const prevPart = logicalParts.find(p => p.id === part.id - 1);
+    const queryBeforeIncompleteFilter = prevPart ? value.substring(0, prevPart.position[1] + 1) : undefined;
     return {
       ...part,
+      queryBeforeIncompleteFilter,
+      query: value,
       ...getContextData(part, cursorStartPosition)
     };
   }, [logicalParts, caretPosition]);
@@ -50,6 +54,8 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
         return fieldValues;
       case ContextType.PipeName:
         return pipeList;
+      case ContextType.FilterOrPipeName:
+        return [...fieldNames, ...pipeList];
       default:
         return [];
     }
@@ -58,7 +64,7 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   const getUpdatedValue = (insertValue: string, logicalParts: LogicalPart[], id?: number) => {
     return logicalParts.reduce((acc, part) => {
       const value = part.id === id ? insertValue : part.value;
-      const separator = part.type === LogicalPartType.Pipe ? " | " : " ";
+      const separator = part.separator === "|" ? " | " : " ";
       return `${acc}${separator}${value}`;
     }, "").trim();
   };
@@ -70,7 +76,7 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
       modifiedInsert += ":";
     } else if (contextType === ContextType.FilterValue) {
       const insertWithQuotes = value.startsWith("_stream:") ? modifiedInsert : `${JSON.stringify(modifiedInsert)}`;
-      modifiedInsert = `${contextData?.filterName || ""}:${insertWithQuotes}`;
+      modifiedInsert = `${contextData?.filterName || ""}${contextData?.operator || ":"}${insertWithQuotes}`;
     }
 
     return modifiedInsert;
@@ -86,7 +92,13 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
 
     const insertValue = getModifyInsert(insert, contextType, value, item.type);
     const newValue = getUpdatedValue(insertValue, logicalParts, id);
-    const updatedPosition = (position[0] || 1) + insertValue.length + (item.type === ContextType.PipeName ? 1 : 0);
+    const logicalPart = logicalParts.find(p => p.id === id);
+    const getPositionCorrection = () => {
+      if (logicalPart?.type === LogicalPartType.FilterOrPipe) return 1;
+      if (item.type === ContextType.PipeName) return 1;
+      return 0;
+    };
+    const updatedPosition = (position[0] || 1) + insertValue.length + getPositionCorrection();
 
     onSelect(newValue, updatedPosition);
   }, [contextData, logicalParts]);

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/LogsQL/parser.ts

@@ -9,6 +9,7 @@ export const splitLogicalParts = (expr: string) => {
   const input = expr; //.replace(/\s*:\s*/g, ":");
   const parts: LogicalPart[] = [];
   let currentPart = "";
+  let separator: undefined | " " | "|" = undefined;
   let isPipePart = false;
 
   const quotes = ["'", "\"", "`"];
@@ -43,8 +44,9 @@ export const splitLogicalParts = (expr: string) => {
       isPipePart = true;
       const countStartSpaces = currentPart.match(/^ */)?.[0].length || 0;
       const countEndSpaces = currentPart.match(/ *$/)?.[0].length || 0;
-      pushPart(currentPart, true, [startIndex + countStartSpaces, i - countEndSpaces - 1], parts);
+      pushPart(currentPart, true, [startIndex + countStartSpaces, i - countEndSpaces - 1], parts, separator);
       currentPart = "";
+      separator = "|";
       startIndex = i + 1;
       continue;
     }
@@ -54,7 +56,8 @@ export const splitLogicalParts = (expr: string) => {
       const nextStr = input.slice(i).replace(/^\s*/, "");
       const prevStr = input.slice(0, i).replace(/\s*$/, "");
       if (!nextStr.startsWith(":") && !prevStr.endsWith(":")) {
-        pushPart(currentPart, false, [startIndex, i - 1], parts);
+        pushPart(currentPart, false, [startIndex, i - 1], parts, separator);
+        separator = " ";
         currentPart = "";
         startIndex = i + 1;
         continue;
@@ -65,26 +68,35 @@ export const splitLogicalParts = (expr: string) => {
   }
 
   // push the last part
-  pushPart(currentPart, isPipePart, [startIndex, input.length], parts);
+  pushPart(currentPart, isPipePart, [startIndex, input.length], parts, separator);
 
   return parts;
 };
 
-const pushPart = (currentPart: string, isPipePart: boolean, position: LogicalPartPosition, parts: LogicalPart[]) => {
+const pushPart = (currentPart: string, isPipePart: boolean, position: LogicalPartPosition, parts: LogicalPart[], separator: LogicalPart["separator"]) => {
   const trimmedPart = currentPart.trim();
   if (!trimmedPart) return;
   const isOperator = BUILDER_OPERATORS.includes(trimmedPart.toUpperCase());
+  const pipesTypes = [LogicalPartType.Pipe, LogicalPartType.FilterOrPipe];
+  const isPreviousPartPipe = parts.length > 0 && pipesTypes.includes(parts[parts.length - 1].type);
+
+  const getType = () => {
+    if (isPreviousPartPipe) return LogicalPartType.FilterOrPipe;
+    if (isPipePart) return LogicalPartType.Pipe;
+    if (isOperator) return LogicalPartType.Operator;
+    return LogicalPartType.Filter;
+  };
+
   parts.push({
     id: parts.length,
     value: trimmedPart,
     position,
-    type: isPipePart
-      ? LogicalPartType.Pipe
-      : isOperator ? LogicalPartType.Operator : LogicalPartType.Filter,
+    type: getType(),
+    separator,
   });
 };
 
-export const getContextData = (part: LogicalPart, cursorPos: number) => {
+export const getContextData = (part: LogicalPart, cursorPos: number): ContextData => {
   const valueBeforeCursor = part.value.substring(0, cursorPos);
   const valueAfterCursor = part.value.substring(cursorPos);
 
@@ -95,23 +107,91 @@ export const getContextData = (part: LogicalPart, cursorPos: number) => {
     contextType: ContextType.Unknown,
   };
 
-  if (part.type === LogicalPartType.Filter) {
-    const noColon = !valueBeforeCursor.includes(":") && !valueAfterCursor.includes(":");
-    if (noColon) {
-      metaData.contextType = ContextType.FilterUnknown;
-    } else if (valueBeforeCursor.includes(":")) {
-      const [filterName, filterValue] = valueBeforeCursor.split(":");
-      metaData.contextType = ContextType.FilterValue;
-      metaData.filterName = filterName;
-      metaData.valueContext = filterValue;
-    } else {
-      metaData.contextType = ContextType.FilterName;
-    }
-  } else if (part.type === LogicalPartType.Pipe) {
-    const valueStartWithPipe = PIPE_NAMES.some(p => part.value.startsWith(p));
-    metaData.contextType = valueStartWithPipe ? ContextType.PipeValue : ContextType.PipeName;
-  }
+  // Determine context type based on logical part type
+  determineContextType(part, valueBeforeCursor, valueAfterCursor, metaData);
 
+  // Clean up quotes in valueContext
   metaData.valueContext = metaData.valueContext.replace(/^["']|["']$/g, "");
+
   return metaData;
 };
+
+/** Helper function to determine if a string starts with any of the pipe names */
+const startsWithPipe = (value: string): boolean => {
+  return PIPE_NAMES.some(p => value.startsWith(p));
+};
+
+/** Helper function to check for colon presence */
+const hasNoColon = (before: string, after: string): boolean => {
+  return !before.includes(":") && !after.includes(":");
+};
+
+/** Helper function to extract filter name and update metadata for filter values */
+const handleFilterValue = (valueBeforeCursor: string, metaData: ContextData): void => {
+  const [filterName, ...filterValue] = valueBeforeCursor.split(":");
+  metaData.contextType = ContextType.FilterValue;
+  metaData.filterName = filterName;
+  const enhanceOperators = ["=", "-", "!", "~", "<", ">", "<=", ">="] as const;
+  const enhanceOperator = enhanceOperators.find(op => op === filterValue[0]);
+  if (enhanceOperator) {
+    metaData.valueContext = filterValue.slice(1).join(":");
+    metaData.operator = `:${enhanceOperator}`;
+  } else {
+    metaData.valueContext = filterValue.join(":");
+    metaData.operator = ":";
+  }
+};
+
+/** Function to determine context type based on part type and value */
+const determineContextType = (
+  part: LogicalPart,
+  valueBeforeCursor: string,
+  valueAfterCursor: string,
+  metaData: ContextData
+): void => {
+  switch (part.type) {
+    case LogicalPartType.Filter:
+      handleFilterType(valueBeforeCursor, valueAfterCursor, metaData);
+      break;
+
+    case LogicalPartType.Pipe:
+      metaData.contextType = startsWithPipe(part.value)
+        ? ContextType.PipeValue
+        : ContextType.PipeName;
+      break;
+
+    case LogicalPartType.FilterOrPipe:
+      handleFilterOrPipeType(part.value, valueBeforeCursor, metaData);
+      break;
+  }
+};
+
+/** Handle filter type context determination */
+const handleFilterType = (
+  valueBeforeCursor: string,
+  valueAfterCursor: string,
+  metaData: ContextData
+): void => {
+  if (hasNoColon(valueBeforeCursor, valueAfterCursor)) {
+    metaData.contextType = ContextType.FilterUnknown;
+  } else if (valueBeforeCursor.includes(":")) {
+    handleFilterValue(valueBeforeCursor, metaData);
+  } else {
+    metaData.contextType = ContextType.FilterName;
+  }
+};
+
+/** Handle FilterOrPipeType context determination */
+const handleFilterOrPipeType = (
+  value: string,
+  valueBeforeCursor: string,
+  metaData: ContextData
+): void => {
+  if (startsWithPipe(value)) {
+    metaData.contextType = ContextType.PipeValue;
+  } else if (valueBeforeCursor.includes(":")) {
+    handleFilterValue(valueBeforeCursor, metaData);
+  } else {
+    metaData.contextType = ContextType.FilterOrPipeName;
+  }
+};

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/LogsQL/types.ts

@@ -2,15 +2,19 @@ export enum LogicalPartType {
   Filter = "Filter",
   Pipe = "Pipe",
   Operator = "Operator",
+  FilterOrPipe = "FilterOrPipe",
 }
 
 export type LogicalPartPosition = [start: number, end: number];
 
+export type LogicalPartSeparator = " " | "|";
+
 export interface LogicalPart {
   id: number;
   value: string;
   type: LogicalPartType;
   position: LogicalPartPosition;
+  separator?: LogicalPartSeparator;
 }
 
 export interface ContextData {
@@ -19,6 +23,10 @@ export interface ContextData {
   contextType: ContextType;
   valueContext: string;
   filterName?: string;
+  query?: string;
+  queryBeforeIncompleteFilter?: string;
+  separator?: LogicalPartSeparator;
+  operator?: ":" | ":!" | ":-" | ":=" | ":~" | ":<" | ":>" | ":<=" | ":>=";
 }
 
 export enum ContextType {
@@ -28,4 +36,5 @@ export enum ContextType {
   PipeName = "Pipes",
   PipeValue = "PipeValue",
   Unknown = "Unknown",
+  FilterOrPipeName = "FilterOrPipeName",
 }

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/LogsQL/useFetchLogsQLOptions.tsx

@@ -10,11 +10,11 @@ import { AUTOCOMPLETE_LIMITS } from "../../../../constants/queryAutocomplete";
 import { LogsFiledValues } from "../../../../api/types";
 import { useLogsDispatch, useLogsState } from "../../../../state/logsPanel/LogsStateContext";
 import { useTenant } from "../../../../hooks/useTenant";
+import { generateQuery } from "./utils";
 
 type FetchDataArgs = {
   urlSuffix: string;
-  setter: Dispatch<SetStateAction<AutocompleteOptions[]>>
-  type: ContextType;
+  setter: (value: LogsFiledValues[]) => void;
   params?: URLSearchParams;
 }
 
@@ -24,7 +24,8 @@ const icons = {
   [ContextType.FilterValue]: <ValueIcon/>,
   [ContextType.PipeName]: <FunctionIcon/>,
   [ContextType.PipeValue]: <LabelIcon/>,
-  [ContextType.Unknown]: <ValueIcon/>
+  [ContextType.Unknown]: <ValueIcon/>,
+  [ContextType.FilterOrPipeName]: <FunctionIcon/>
 };
 
 export const useFetchLogsQLOptions = (contextData?: ContextData) => {
@@ -61,7 +62,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
     }));
   };
 
-  const fetchData = async ({ urlSuffix, setter, type, params }: FetchDataArgs) => {
+  const fetchData = async ({ urlSuffix, setter, params }: FetchDataArgs) => {
     abortControllerRef.current.abort();
     abortControllerRef.current = new AbortController();
     const { signal } = abortControllerRef.current;
@@ -73,7 +74,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
     try {
       const cachedData = autocompleteCache.get(key);
       if (cachedData) {
-        setter(processData(cachedData, type));
+        setter(cachedData);
         setLoading(false);
         return;
       }
@@ -86,7 +87,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
       if (response.ok) {
         const data = await response.json();
         const value = (data?.values || []) as LogsFiledValues[];
-        setter(value ? processData(value, type) : []);
+        setter(value || []);
         dispatch({ type: "SET_AUTOCOMPLETE_CACHE", payload: { key, value } });
       }
       setLoading(false);
@@ -101,7 +102,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
 
   // fetch field names
   useEffect(() => {
-    const validContexts = [ContextType.FilterName, ContextType.FilterUnknown];
+    const validContexts = [ContextType.FilterName, ContextType.FilterUnknown, ContextType.FilterOrPipeName];
     const isInvalidContext = !validContexts.includes(contextData?.contextType || ContextType.Unknown);
     if (!serverUrl || isInvalidContext) {
       return;
@@ -109,11 +110,14 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
 
     setFieldNames([]);
 
+    const setter = (filterNames: LogsFiledValues[]) => {
+      setFieldNames(processData(filterNames, ContextType.FilterName));
+    };
+
     fetchData({
       urlSuffix: "field_names",
-      setter: setFieldNames,
-      type: ContextType.FilterName,
-      params: getQueryParams({ query: "*" })
+      setter: setter,
+      params: getQueryParams({ query: contextData?.queryBeforeIncompleteFilter || "*" })
     });
 
     return () => abortControllerRef.current?.abort();
@@ -128,11 +132,14 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
 
     setFieldValues([]);
 
+    const setter = (filterValues: LogsFiledValues[]) => {
+      setFieldValues(processData(filterValues, ContextType.FilterValue));
+    };
+
     fetchData({
       urlSuffix: "field_values",
-      setter: setFieldValues,
-      type: ContextType.FilterValue,
-      params: getQueryParams({ query: "*", field: contextData.filterName })
+      setter: setter,
+      params: getQueryParams({ query: generateQuery(contextData), field: contextData.filterName })
     });
 
     return () => abortControllerRef.current?.abort();

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/LogsQL/util.test.ts

@@ -0,0 +1,131 @@
+import { expect } from "vitest";
+import { generateQuery } from "./utils";
+import { ContextType } from "./types";
+
+describe("utils", () => {
+  describe("_time", () => {
+    it("should return the trimmed value by `-`", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
+        contextType: ContextType.FilterValue,
+        filterName: "_time",
+        query: "_stream:{type=\"WatchEvent\"} _time:2025-04-1",
+        valueAfterCursor: "",
+        valueBeforeCursor: "_time=2025-04-1",
+        valueContext: "2025-04-1"
+      })).toStrictEqual("_stream:{type=\"WatchEvent\"} _time:2025-04");
+    });
+
+    it("should return the trimmed value by `:` if char `-` also exist in the query", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
+        contextType: ContextType.FilterValue,
+        filterName: "_time",
+        query: "_stream:{type=\"WatchEvent\"} _time:2025-04-10T23:45:5",
+        valueAfterCursor: "",
+        valueBeforeCursor: "_time=2025-04-10T23:45:5",
+        valueContext: "2025-04-10T23:45:5"
+      })).toStrictEqual("_stream:{type=\"WatchEvent\"} _time:2025-04-10T23:45");
+    });
+
+    it("should return default `*` instead of -time filter", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
+        contextType: ContextType.FilterValue,
+        filterName: "_time",
+        query: "_stream:{type=\"WatchEvent\"} _time:202",
+        valueAfterCursor: "",
+        valueBeforeCursor: "_time=202",
+        valueContext: "202"
+      })).toStrictEqual("_stream:{type=\"WatchEvent\"} *");
+    });
+  });
+
+  describe("_stream", () => {
+    it("should add regexp to filter value", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "",
+        contextType: ContextType.FilterValue,
+        filterName: "_stream",
+        query: "_stream:{type=\"WatchEve",
+        valueAfterCursor: "",
+        valueBeforeCursor: "_stream:{type=\"WatchEve",
+        valueContext: "{type=\"WatchEve"
+      })).toStrictEqual("_stream:{type=~\"WatchEve.*\"}");
+    });
+
+    it("should add regexp to filter value if cursor in the middle of value", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "",
+        contextType: ContextType.FilterValue,
+        filterName: "_stream",
+        query: "_stream:{type=\"WatchEve\"}",
+        valueAfterCursor: "",
+        valueBeforeCursor: "_stream:{type=\"WatchEve",
+        valueContext: "{type=\"WatchEve"
+      })).toStrictEqual("_stream:{type=~\"WatchEve.*\"}");
+    });
+
+    it("should return * if do not have value after =", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "",
+        contextType: ContextType.FilterValue,
+        filterName: "_stream",
+        query: "_stream:{type=",
+        valueAfterCursor: "",
+        valueBeforeCursor: "_stream:{type=",
+        valueContext: "{type="
+      })).toStrictEqual("*");
+    });
+  });
+
+  it("_msg", () => {
+    expect(generateQuery({
+      queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
+      contextType: ContextType.FilterValue,
+      filterName: "_msg",
+      query: "_stream:{type=\"WatchEvent\"} _msg:453",
+      valueAfterCursor: "",
+      valueBeforeCursor: "_msg:453",
+      valueContext: "453"
+    })).toStrictEqual("_stream:{type=\"WatchEvent\"} *");
+  });
+
+  it("_stream_id", () => {
+    expect(generateQuery({
+      queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
+      contextType: ContextType.FilterValue,
+      filterName: "_stream_id",
+      query: "_stream:{type=\"WatchEvent\"} _stream_id:453",
+      valueAfterCursor: "",
+      valueBeforeCursor: "_stream_id:453",
+      valueContext: "453"
+    })).toStrictEqual("_stream:{type=\"WatchEvent\"} *");
+  });
+
+  describe("other fields", () => {
+    it("should add prefix filter to other type of field names", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "",
+        contextType: ContextType.FilterValue,
+        filterName: "repo.name",
+        query: "repo.name:Victori",
+        valueAfterCursor: "",
+        valueBeforeCursor: "repo.name:Victori",
+        valueContext: "Victori"
+      })).toStrictEqual("repo.name:Victori*");
+    });
+
+    it("should add prefix filter to other type of field names with escaped via double quote", () => {
+      expect(generateQuery({
+        queryBeforeIncompleteFilter: "",
+        contextType: ContextType.FilterValue,
+        filterName: "repo.name",
+        query: "repo.name:\"Victori",
+        valueAfterCursor: "",
+        valueBeforeCursor: "repo.name:\"Victori",
+        valueContext: "Victori"
+      })).toStrictEqual("repo.name:Victori*");
+    });
+  });
+});

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/LogsQL/utils.ts

@@ -0,0 +1,61 @@
+import { ContextData } from "./types";
+
+const getStreamFieldQuery = (valueContext: string) => {
+  if (valueContext.includes("=")) {
+    const [fieldName, fieldValue] = valueContext.split("=");
+    if (fieldValue) {
+      return `_stream:${fieldName}=~${fieldValue}.*"}`;
+    }
+  }
+
+  return "*";
+};
+
+const getLastPartUntilDelimiter = (value: string, delimiter: string) => {
+  const lastIndexOfDelimiter = value.lastIndexOf(delimiter);
+  return lastIndexOfDelimiter !== -1 ? value.slice(0, lastIndexOfDelimiter) : "";
+};
+
+const getDateQuery = (contextData: ContextData) => {
+  let fieldValue = "";
+  if (contextData.valueContext.includes(":")) {
+    fieldValue = getLastPartUntilDelimiter(contextData.valueContext, ":");
+  } else if (contextData.valueContext.includes("-")) {
+    fieldValue = getLastPartUntilDelimiter(contextData.valueContext, "-");
+  }
+  return fieldValue ? `${contextData.filterName}:${fieldValue}` : "*";
+};
+
+/**
+ * Generates a query string based on the provided context data.
+ *
+ * The function processes the input based on the `filterName` property:
+ *
+ * - If `filterName` is `_msg` or `_stream_id`, the query cannot be generated specifically,
+ *   so a wildcard query (`"*"`) is returned.
+ *
+ * - If `filterName` is `_stream`, the query is generated using regexp (`{type=~"value.*"}`).
+ *
+ * - If `filterName` is `_time`, a simplified query is created by trimming the value up
+ *   to the first occurrence of a delimiter such as `-` or `:`.
+ *
+ * - For all other values of `filterName`, a prefix query is returned using
+ *   the `query` value with a `*` appended (e.g., `"value*"`).
+ *
+ * @param {ContextData} contextData - The context object containing query parameters and metadata.
+ * @returns {string} The generated query string.
+ */
+export const generateQuery = (contextData: ContextData): string => {
+  let fieldQuery = "";
+  if (!contextData.filterName || !contextData.query || ["_msg", "_stream_id"].includes(contextData.filterName)) {
+    fieldQuery = "*";
+  } else if ("_stream" === contextData.filterName) {
+    fieldQuery = getStreamFieldQuery(contextData.valueContext);
+  } else if ("_time" === contextData.filterName) {
+    fieldQuery = getDateQuery(contextData);
+  } else {
+    fieldQuery = `${contextData.filterName}:${contextData.valueContext}*`;
+  }
+
+  return contextData.queryBeforeIncompleteFilter ? `${contextData.queryBeforeIncompleteFilter}${contextData.separator ?? " "}${fieldQuery}` : fieldQuery;
+};

app/vmui/packages/vmui/src/components/LogsConfigurators/GroupLogsConfigurators/GroupLogsConfigurators.tsx

@@ -19,8 +19,8 @@ import {
   LOGS_URL_PARAMS,
   WITHOUT_GROUPING
 } from "../../../constants/logs";
-import { getFromStorage, saveToStorage } from "../../../utils/storage";
 import LogParsingSwitches from "../../Configurators/LogsSettings/LogParsingSwitches";
+import { useLocalStorageBoolean } from "../../../hooks/useLocalStorageBoolean";
 
 const {
   GROUP_BY,
@@ -48,7 +48,7 @@ const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
   const [dateFormat, setDateFormat] = useState(searchParams.get(DATE_FORMAT) || LOGS_DATE_FORMAT);
   const [errorFormat, setErrorFormat] = useState("");
 
-  const [disabledHovers, setDisabledHovers] = useState(!!getFromStorage("LOGS_DISABLED_HOVERS"));
+  const [disabledHovers, handleSetDisabledHovers] = useLocalStorageBoolean("LOGS_DISABLED_HOVERS");
 
   const isGroupChanged = groupBy !== LOGS_GROUP_BY;
   const isDisplayFieldsChanged = displayFields.length !== 1 || displayFields[0] !== LOGS_DISPLAY_FIELDS;
@@ -117,11 +117,6 @@ const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
     handleClose();
   };
 
-  const handleSetDisabledHovers = (value: boolean) => {
-    setDisabledHovers(value);
-    saveToStorage("LOGS_DISABLED_HOVERS", value);
-  };
-
   const tooltipContent = () => {
     if (!hasChanges) return title;
     return (

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -1,4 +1,3 @@
-import React from "react";
 import { getCssVariable } from "../../../utils/theme";
 
 export const LogoIcon = () => (
@@ -643,3 +642,17 @@ export const PauseIcon = () => (
     <path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z" />
   </svg>
 );
+
+export const ScrollToTopIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M8 12l4-4 4 4m-4-4v12"
+      strokeWidth="2"
+      stroke="currentColor"
+      fill="none"
+    />
+  </svg>
+);

app/vmui/packages/vmui/src/components/ScrollToTopButton/ScrollToTopButton.tsx

@@ -0,0 +1,59 @@
+import { FC, useEffect, useState } from "preact/compat";
+import Button from "../Main/Button/Button";
+import Tooltip from "../Main/Tooltip/Tooltip";
+import { ScrollToTopIcon } from "../Main/Icons";
+import classNames from "classnames";
+import "./style.scss";
+import { useCallback } from "react";
+
+interface ScrollToTopButtonProps {
+  className?: string;
+}
+
+const ScrollToTopButton: FC<ScrollToTopButtonProps> = ({ className }) => {
+  const [isVisible, setIsVisible] = useState(false);
+
+  const checkScrollPosition = () => {
+    const scrollPosition = window.pageYOffset || document.documentElement.scrollTop;
+    const visibleHeightThreshold = window.innerHeight;
+
+    setIsVisible(scrollPosition > visibleHeightThreshold);
+  };
+
+  const scrollToTop = useCallback(() => {
+    window.scrollTo({
+      top: 0,
+      behavior: "smooth"
+    });
+  }, []);
+
+  useEffect(() => {
+    window.addEventListener("scroll", checkScrollPosition);
+    checkScrollPosition();
+    
+    return () => {
+      window.removeEventListener("scroll", checkScrollPosition);
+    };
+  }, []);
+
+  return (
+    <div
+      className={classNames({
+        "vm-scroll-to-top-button": true,
+        "vm-scroll-to-top-button_visible": isVisible
+      }, className)}
+    >
+      <Tooltip title="Scroll to top">
+        <Button
+          variant="contained"
+          color="primary"
+          onClick={scrollToTop}
+          ariaLabel="Scroll to top"
+          startIcon={<ScrollToTopIcon />}
+        />
+      </Tooltip>
+    </div>
+  );
+};
+
+export default ScrollToTopButton;

app/vmui/packages/vmui/src/components/ScrollToTopButton/style.scss

@@ -0,0 +1,26 @@
+@use "src/styles/variables" as *;
+
+.vm-scroll-to-top-button {
+  position: fixed;
+  bottom: 20px;
+  right: 20px;
+  z-index: 4;
+  opacity: 0;
+  visibility: hidden;
+  transition: opacity 0.3s, visibility 0.3s;
+
+  &_visible {
+    opacity: 1;
+    visibility: visible;
+  }
+
+  .vm-button {
+    border-radius: 50%;
+    width: 40px;
+    height: 40px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
+  }
+}

app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.test.ts

@@ -0,0 +1,70 @@
+import { act, renderHook } from "@testing-library/preact";
+import { useLocalStorageBoolean } from "./useLocalStorageBoolean";
+import * as storageUtils from "../utils/storage";
+import { Mock } from "vitest";
+import { StorageKeys } from "../utils/storage";
+
+vi.mock("../utils/storage");
+
+const testStorageKey = "TEST_STORAGE_KEY" as StorageKeys;
+
+describe("useLocalStorageBoolean", () => {
+  const { getFromStorage, saveToStorage } = storageUtils;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("initializes with the value from localStorage", () => {
+    const mockGetFromStorage = getFromStorage as Mock;
+    mockGetFromStorage.mockReturnValueOnce(true);
+
+    const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
+
+    expect(result.current[0]).toBe(true);
+    expect(getFromStorage).toHaveBeenCalledWith(testStorageKey);
+  });
+
+  it("updates localStorage and state when setter is called", () => {
+    const mockGetFromStorage = getFromStorage as Mock;
+    mockGetFromStorage.mockReturnValueOnce(false);
+
+    const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
+
+    act(() => {
+      result.current[1](true);
+    });
+
+    expect(saveToStorage).toHaveBeenCalledWith(testStorageKey, true);
+    expect(result.current[0]).toBe(false);
+  });
+
+  it("reacts to changes in localStorage by storage events", () => {
+    const mockGetFromStorage = getFromStorage as Mock;
+    mockGetFromStorage.mockReturnValueOnce(false);
+
+    const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
+
+    // Simulate a storage event
+    act(() => {
+      mockGetFromStorage.mockReturnValueOnce(true);
+      window.dispatchEvent(new StorageEvent("storage", { key: testStorageKey, newValue: "true" }));
+    });
+
+    expect(result.current[0]).toBe(true);
+  });
+
+  it("does not update state if the localStorage value remains the same", () => {
+    const mockGetFromStorage = getFromStorage as Mock;
+    mockGetFromStorage.mockReturnValueOnce(false);
+
+    const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
+
+    act(() => {
+      mockGetFromStorage.mockReturnValueOnce(false);
+      window.dispatchEvent(new StorageEvent("storage", { key: testStorageKey, newValue: "false" }));
+    });
+
+    expect(result.current[0]).toBe(false);
+  });
+});

app/vmui/packages/vmui/src/hooks/useLocalStorageBoolean.ts

@@ -0,0 +1,31 @@
+import { useMemo, useState } from "preact/compat";
+import { getFromStorage, saveToStorage, StorageKeys } from "../utils/storage";
+import useEventListener from "./useEventListener";
+import { useCallback } from "react";
+
+/**
+ * A custom hook that synchronizes a boolean state with a value stored in localStorage.
+ *
+ * @param {StorageKeys} key - The key used to access the corresponding value in localStorage.
+ * @returns {[boolean, function]} A tuple containing the current boolean value from localStorage and a setter function to update the value in localStorage.
+ *
+ * The hook listens to the "storage" event to automatically update the state when the localStorage value changes.
+ */
+export const useLocalStorageBoolean = (key: StorageKeys): [boolean, (value: boolean) => void] => {
+  const [value, setValue] = useState(!!getFromStorage(key));
+
+  const handleUpdateStorage = useCallback(() => {
+    const newValue = !!getFromStorage(key);
+    if (newValue !== value) {
+      setValue(newValue);
+    }
+  }, [key, value]);
+
+  const setNewValue = useCallback((newValue: boolean) => {
+    saveToStorage(key, newValue);
+  }, [key]);
+
+  useEventListener("storage", handleUpdateStorage);
+
+  return useMemo(() => [value, setNewValue], [value, setNewValue]);
+};

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/LiveTailingView/LiveTailingSettings.tsx

@@ -19,8 +19,8 @@ interface LiveTailingSettingsProps {
   handleResumeLiveTailing: () => void;
   pauseLiveTailing: () => void;
   clearLogs: () => void;
-  isCompactTailingNumber: boolean;
-  handleSetCompactTailing: (value: boolean) => void;
+  isRawJsonView: boolean;
+  onRawJsonViewChange: (value: boolean) => void;
 }
 
 const LiveTailingSettings: FC<LiveTailingSettingsProps> = ({
@@ -32,8 +32,8 @@ const LiveTailingSettings: FC<LiveTailingSettingsProps> = ({
   handleResumeLiveTailing,
   pauseLiveTailing,
   clearLogs,
-  isCompactTailingNumber,
-  handleSetCompactTailing
+  isRawJsonView,
+  onRawJsonViewChange
 }) => {
   const settingButtonRef = useRef<HTMLDivElement>(null);
   const { value: isSettingsOpen, setFalse: closeSettings, setTrue: openSettings } = useBoolean(false);
@@ -106,12 +106,12 @@ const LiveTailingSettings: FC<LiveTailingSettingsProps> = ({
           <div className="vm-live-tailing-view__settings-modal">
             <div className={"vm-live-tailing-view__settings-modal-item"}>
               <Switch
-                label={"Expandable Properties View"}
-                value={isCompactTailingNumber}
-                onChange={handleSetCompactTailing}
+                label={"Raw JSON View"}
+                value={isRawJsonView}
+                onChange={onRawJsonViewChange}
               />
               <span className="vm-group-logs-configurator-item__info">
-                Switches log display to expandable properties view with additional visualization settings. Please note: when processing large volumes of data, it may increase system response time.
+                When this option is enabled, logs will be displayed in raw JSON format. This improves performance and uses less CPU and memory.
               </span>
             </div>
           </div>

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/LiveTailingView/LiveTailingView.tsx

@@ -12,11 +12,13 @@ import GroupLogsItem from "../../../GroupLogs/GroupLogsItem";
 import LiveTailingSettings from "./LiveTailingSettings";
 import Alert from "../../../../../components/Main/Alert/Alert";
 import { isDecreasing } from "../../../../../utils/array";
+import { useLocalStorageBoolean } from "../../../../../hooks/useLocalStorageBoolean";
+import ScrollToTopButton from "../../../../../components/ScrollToTopButton/ScrollToTopButton";
 
 const SCROLL_THRESHOLD = 100;
 const scrollToBottom = () => window.scrollTo({
   top: document.documentElement.scrollHeight,
-  behavior: "instant"
+  behavior: "smooth"
 });
 const throttledScrollToBottom = throttle(scrollToBottom, 200);
 
@@ -28,8 +30,7 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
   const { setSearchParamsFromKeys } = useSearchParamsFromObject();
   const [rowsPerPage, setRowsPerPage] = useStateSearchParams(100, "rows_per_page");
   const [query, _setQuery] = useStateSearchParams("*", "query");
-  const [isCompactTailingStr] = useStateSearchParams(0, "compact_tailing");
-  const isCompactTailingNumber = Boolean(Number(isCompactTailingStr));
+  const [isRawJsonView, setIsRawJsonView] = useLocalStorageBoolean("RAW_JSON_LIVE_VIEW");
   const {
     logs,
     isPaused,
@@ -54,10 +55,6 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
     setSearchParamsFromKeys({ rows_per_page: limit });
   }, [setRowsPerPage, setSearchParamsFromKeys]);
 
-  const handleSetCompactTailing = useCallback((value: boolean) => {
-    setSearchParamsFromKeys({ compact_tailing: Number(value) });
-  }, [setSearchParamsFromKeys]);
-
   useEffect(() => {
     startLiveTailing();
     return () => stopLiveTailing();
@@ -111,9 +108,10 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
         handleResumeLiveTailing={handleResumeLiveTailing}
         pauseLiveTailing={pauseLiveTailing}
         clearLogs={clearLogs}
-        isCompactTailingNumber={isCompactTailingNumber}
-        handleSetCompactTailing={handleSetCompactTailing}
+        isRawJsonView={isRawJsonView}
+        onRawJsonViewChange={setIsRawJsonView}
       />
+      <ScrollToTopButton />
       <div
         ref={containerRef}
         className="vm-live-tailing-view__container"
@@ -122,28 +120,31 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
           ? (<div className="vm-live-tailing-view__empty">Waiting for logs...</div>)
           : (<div className="vm-live-tailing-view__logs">
             {logs.map(({ _log_id, ...log }, idx) =>
-              isCompactTailingNumber
-                ? (
-                  <GroupLogsItem
-                    key={_log_id}
-                    log={log}
-                    onItemClick={pauseLiveTailing}
-                    hideGroupButton={true}
-                    displayFields={displayFields}
-                  />
-                ) : (
-                  <pre
-                    key={idx}
-                    className="vm-live-tailing-view__log-row"
-                  >
-                    {JSON.stringify(log)}
-                  </pre>
-                )
+              isRawJsonView ? (
+                <pre
+                  key={idx}
+                  className="vm-live-tailing-view__log-row"
+                  onMouseDown={pauseLiveTailing}
+                >
+                  {JSON.stringify(log)}
+                </pre>
+              ) : (
+                <GroupLogsItem
+                  key={_log_id}
+                  log={log}
+                  onItemClick={pauseLiveTailing}
+                  hideGroupButton={true}
+                  displayFields={displayFields}
+                />
+              )
             )}
           </div>
           )}
       </div>
-      {isLimitedLogsPerUpdate && (<Alert variant="warning">Too many logs per second detected. Large volumes of log data are difficult to process and may impact performance. We recommend adding filters to your query for better analysis and system performance.</Alert>)}
+      {isLimitedLogsPerUpdate && (
+        <Alert variant="warning">Too many logs per second detected. Large volumes of log data are difficult to process
+          and may impact performance. We recommend adding filters to your query for better analysis and system
+          performance.</Alert>)}
     </>
   );
 };

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/LiveTailingView/style.scss

@@ -34,9 +34,10 @@
     width: 100%;
     height: 100%;
     overflow: auto;
-    min-height: 200px;
+    min-height: calc(100vh - 120px);
     font-family: $font-family-monospace;
     padding-bottom: $padding-medium;
+    transition: min-height 0.3s ease;
   }
 
   &__empty {

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/LiveTailingView/useLiveTailingLogs.ts

@@ -4,19 +4,8 @@ import { Logs } from "../../../../../api/types";
 import { useAppState } from "../../../../../state/common/StateContext";
 import useBoolean from "../../../../../hooks/useBoolean";
 import { useTenant } from "../../../../../hooks/useTenant";
+import { LogFlowAnalyzer } from "./utils";
 
-/**
- * Defines the maximum number of consecutive times logs can be fetched above the threshold
- * before showing a warning notification, and vice versa:
- * - If logs are fetched above a threshold this many times in a row -> show warning
- * - If warning is shown, it won't disappear until logs are fetched below a threshold
- *   this many times in a row
- *
- * This threshold helps optimize log display performance when dealing with large volumes of logs.
- * If the threshold is consistently exceeded, users will be prompted to add filters to their query
- * for better system performance and more focused log analysis.
- */
-const MAX_ATTEMPTS_FETCH_LOGS_PER_SECOND = 5;
 /**
  * Defines the log's threshold, after which will be shown a warning notification
  */
@@ -56,7 +45,7 @@ const createStreamProcessor = (
     } catch (e) {
       if (e instanceof Error && e.name !== "AbortError") {
         console.error("Stream processing error:", e);
-        setError(String(e));
+        restartTailing();
       }
     } finally {
       clearInterval(connectionCheckInterval);
@@ -64,31 +53,6 @@ const createStreamProcessor = (
   };
 };
 
-const updateLimitModeTracking = (
-  linesCount: number,
-  attemptsFetchLimitRef: React.MutableRefObject<number>,
-  attemptsFetchLowRef: React.MutableRefObject<number>,
-  isLimitedLogsPerUpdate: boolean,
-) => {
-  if (linesCount > LOGS_THRESHOLD) {
-    attemptsFetchLimitRef.current++;
-    attemptsFetchLowRef.current = 0;
-  } else {
-    attemptsFetchLowRef.current++;
-    attemptsFetchLimitRef.current = 0;
-  }
-
-  if (attemptsFetchLimitRef.current > MAX_ATTEMPTS_FETCH_LOGS_PER_SECOND) {
-    return true;
-  }
-
-  if (attemptsFetchLowRef.current > MAX_ATTEMPTS_FETCH_LOGS_PER_SECOND) {
-    return false;
-  }
-
-  return isLimitedLogsPerUpdate;
-};
-
 const parseLogLines = (lines: string[], counterRef: React.MutableRefObject<bigint>): Logs[] => {
   return lines
     .map(line => {
@@ -108,27 +72,22 @@ interface ProcessBufferedLogsParams {
   lines: string[];
   limit: number;
   counterRef: React.MutableRefObject<bigint>;
-  attemptsFetchLimitRef: React.MutableRefObject<number>;
-  attemptsFetchLowRef: React.MutableRefObject<number>;
   setIsLimitedLogsPerUpdate: (isLimited: boolean) => void;
   setLogs: React.Dispatch<React.SetStateAction<Logs[]>>;
   bufferLinesRef: React.MutableRefObject<string[]>;
-  isLimitedLogsPerUpdate: boolean;
+  logFlowAnalyzerRef?: React.MutableRefObject<LogFlowAnalyzer>;
 }
 
 const processBufferedLogs = ({
   lines,
   limit,
   counterRef,
-  attemptsFetchLimitRef,
-  attemptsFetchLowRef,
   setIsLimitedLogsPerUpdate,
   setLogs,
   bufferLinesRef,
-  isLimitedLogsPerUpdate
+  logFlowAnalyzerRef
 }: ProcessBufferedLogsParams) => {
-
-  const isLimitLogsMode = updateLimitModeTracking(lines.length, attemptsFetchLimitRef, attemptsFetchLowRef, isLimitedLogsPerUpdate);
+  const isLimitLogsMode = logFlowAnalyzerRef?.current?.update(lines.length) === "high";
   const limitedLines = isLimitLogsMode && lines.length > LOGS_THRESHOLD ? lines.slice(-LOGS_THRESHOLD) : lines;
   const newLogs = parseLogLines(limitedLines, counterRef);
 
@@ -155,8 +114,7 @@ export const useLiveTailingLogs = (query: string, limit: number) => {
   const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
   const bufferRef = useRef<string>("");
   const bufferLinesRef = useRef<string[]>([]);
-  const attemptsFetchLimitLogsPerSecondCountRef = useRef<number>(0);
-  const attemptsFetchLowLogsPerSecondCountRef = useRef<number>(0);
+  const logFlowAnalyzerRef = useRef(new LogFlowAnalyzer());
 
   const stopLiveTailing = useCallback(() => {
     if (readerRef.current) {
@@ -239,12 +197,10 @@ export const useLiveTailingLogs = (query: string, limit: number) => {
         lines,
         limit,
         counterRef,
-        attemptsFetchLimitRef: attemptsFetchLimitLogsPerSecondCountRef,
-        attemptsFetchLowRef: attemptsFetchLowLogsPerSecondCountRef,
         setIsLimitedLogsPerUpdate,
-        isLimitedLogsPerUpdate,
         setLogs,
-        bufferLinesRef
+        bufferLinesRef,
+        logFlowAnalyzerRef
       });
     }, PROCESSING_INTERVAL_MS);
 

app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/views/LiveTailingView/utils.ts

@@ -0,0 +1,45 @@
+export class LogFlowAnalyzer {
+  private threshold: number;
+  private windowSize: number;
+  private minHighCount: number;
+  private minNormalCount: number;
+  private window: number[];
+  private state: "normal" | "high";
+
+  /**
+   * @param {number} threshold - The threshold value used for state evaluation. Defaults to 200.
+   * @param {number} windowSize - The size of the window used for tracking data. Defaults to 10.
+   * @param {number} minHighCount - The minimum number of high occurrences needed for state transition. Defaults to 6.
+   * @param {number} minNormalCount - The minimum number of normal occurrences needed for state reset. Defaults to 2.
+   * @return {void}
+   */
+  constructor(threshold: number = 200, windowSize: number = 10, minHighCount: number = 6, minNormalCount: number = 2) {
+    this.threshold = threshold;
+    this.windowSize = windowSize;
+    this.minHighCount = minHighCount;
+    this.minNormalCount = minNormalCount;
+    this.window = [];
+    this.state = "normal";
+  }
+
+  update(logCount: number): "normal" | "high" {
+    this.window.push(logCount);
+    if (this.window.length > this.windowSize) {
+      this.window.shift();
+    }
+
+    const highCount = this.window.filter((x) => x > this.threshold).length;
+
+    if (this.state === "normal") {
+      if (highCount >= this.minHighCount) {
+        this.state = "high";
+      }
+    } else if (this.state === "high") {
+      if (highCount < this.minNormalCount) {
+        this.state = "normal";
+      }
+    }
+
+    return this.state;
+  }
+}

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsFields.tsx

@@ -1,10 +1,9 @@
-import React, { FC, useMemo, useState } from "preact/compat";
+import { FC, useMemo } from "preact/compat";
 import { Logs } from "../../../api/types";
 import "./style.scss";
 import classNames from "classnames";
 import GroupLogsFieldRow from "./GroupLogsFieldRow";
-import useEventListener from "../../../hooks/useEventListener";
-import { getFromStorage } from "../../../utils/storage";
+import { useLocalStorageBoolean } from "../../../hooks/useLocalStorageBoolean";
 
 interface Props {
   log: Logs;
@@ -17,16 +16,7 @@ const GroupLogsFields: FC<Props> = ({ log, hideGroupButton }) => {
       .sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
   }, [log]);
 
-  const [disabledHovers, setDisabledHovers] = useState(!!getFromStorage("LOGS_DISABLED_HOVERS"));
-
-  const handleUpdateStage = () => {
-    const newValDisabledHovers = !!getFromStorage("LOGS_DISABLED_HOVERS");
-    if (newValDisabledHovers !== disabledHovers) {
-      setDisabledHovers(newValDisabledHovers);
-    }
-  };
-
-  useEventListener("storage", handleUpdateStage);
+  const [disabledHovers] = useLocalStorageBoolean("LOGS_DISABLED_HOVERS");
 
   return (
     <div

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx

@@ -1,8 +1,8 @@
-import React, { FC, memo, useMemo, useState } from "preact/compat";
+import React, { FC, memo, useMemo } from "preact/compat";
 import { Logs } from "../../../api/types";
 import "./style.scss";
 import useBoolean from "../../../hooks/useBoolean";
-import { ArrowDownIcon } from "../../../components/Main/Icons";
+import { ArrowDownIcon, CopyIcon } from "../../../components/Main/Icons";
 import classNames from "classnames";
 import { useLogsState } from "../../../state/logsPanel/LogsStateContext";
 import dayjs from "dayjs";
@@ -10,10 +10,13 @@ import { useTimeState } from "../../../state/time/TimeStateContext";
 import { marked } from "marked";
 import { useSearchParams } from "react-router-dom";
 import { LOGS_DATE_FORMAT, LOGS_URL_PARAMS } from "../../../constants/logs";
-import useEventListener from "../../../hooks/useEventListener";
-import { getFromStorage } from "../../../utils/storage";
 import { parseAnsiToHtml } from "../../../utils/ansiParser";
 import GroupLogsFields from "./GroupLogsFields";
+import { useLocalStorageBoolean } from "../../../hooks/useLocalStorageBoolean";
+import Button from "../../../components/Main/Button/Button";
+import Tooltip from "../../../components/Main/Tooltip/Tooltip";
+import { useCallback, useEffect, useState } from "react";
+import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
 
 interface Props {
   log: Logs;
@@ -27,6 +30,8 @@ const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"], onItemClick,
     value: isOpenFields,
     toggle: toggleOpenFields,
   } = useBoolean(false);
+  const [copied, setCopied] = useState<boolean>(false);
+  const copyToClipboard = useCopyToClipboard();
 
   const [searchParams] = useSearchParams();
   const { markdownParsing, ansiParsing } = useLogsState();
@@ -68,21 +73,29 @@ const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"], onItemClick,
     return values;
   }, [log, hasFields, displayFields, ansiParsing]);
 
-  const [disabledHovers, setDisabledHovers] = useState(!!getFromStorage("LOGS_DISABLED_HOVERS"));
-
-  const handleUpdateStage = () => {
-    const newValDisabledHovers = !!getFromStorage("LOGS_DISABLED_HOVERS");
-    if (newValDisabledHovers !== disabledHovers) {
-      setDisabledHovers(newValDisabledHovers);
-    }
-  };
+  const [disabledHovers] = useLocalStorageBoolean("LOGS_DISABLED_HOVERS");
 
   const handleClick = () => {
     toggleOpenFields();
     onItemClick?.(log);
   };
 
-  useEventListener("storage", handleUpdateStage);
+  const handleCopy = useCallback(async (e: Event) => {
+    e.stopPropagation();
+    if (copied) return;
+    try {
+      await copyToClipboard(JSON.stringify(log, null, 2));
+      setCopied(true);
+    } catch (e) {
+      console.error(e);
+    }
+  }, [copied, copyToClipboard]);
+
+  useEffect(() => {
+    if (copied === null) return;
+    const timeout = setTimeout(() => setCopied(false), 2000);
+    return () => clearTimeout(timeout);
+  }, [copied]);
 
   return (
     <div className="vm-group-logs-row">
@@ -93,6 +106,17 @@ const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"], onItemClick,
         })}
         onClick={handleClick}
       >
+        <Tooltip title={copied ? "Copied" : "Copy to clipboard"}>
+          <Button
+            className="vm-group-logs-row-content__copy-row"
+            variant="text"
+            color="gray"
+            size="small"
+            startIcon={<CopyIcon/>}
+            onClick={handleCopy}
+            ariaLabel="copy to clipboard"
+          />
+        </Tooltip>
         {hasFields && (
           <div
             className={classNames({

app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss

@@ -132,7 +132,7 @@ $font-size-logs: var(--font-size-logs, $font-size-small);
 
     &-content {
       display: flex;
-      padding: 2px 0;
+      padding: 2px 24px 2px 0;
       cursor: pointer;
 
       &_interactive {
@@ -140,8 +140,23 @@ $font-size-logs: var(--font-size-logs, $font-size-small);
         will-change: background-color;
       }
 
+      &__copy-row {
+        position: absolute;
+        top: 0;
+        right: 0;
+        z-index: 1;
+        visibility: hidden;
+
+        &.vm-button {
+          padding: 2px;
+        }
+      }
+
       &_interactive:hover {
         background-color: $color-hover-black;
+        .vm-group-logs-row-content__copy-row {
+          visibility: visible;
+        }
       }
 
       &__arrow {

app/vmui/packages/vmui/src/pages/ExploreLogs/hooks/useFetchLogHits.ts

@@ -74,7 +74,7 @@ export const useFetchLogHits = (server: string, query: string) => {
       }
     }
     setIsLoading(prev => ({ ...prev, [id]: false }));
-  }, [url, query]);
+  }, [url, query, tenant]);
 
   useEffect(() => {
     return () => {

app/vmui/packages/vmui/src/utils/storage.ts

@@ -14,11 +14,13 @@ export type StorageKeys = "AUTOCOMPLETE"
   | "THEME"
   | "LOGS_LIMIT"
   | "LOGS_MARKDOWN"
+  | "LOGS_ANSI"
   | "LOGS_DISABLED_HOVERS"
   | "EXPLORE_METRICS_TIPS"
   | "LOGS_QUERY_HISTORY"
   | "METRICS_QUERY_HISTORY"
   | "SERVER_URL"
+  | "RAW_JSON_LIVE_VIEW"
   | DeprecatedStorageKeys;
 
 

apptest/app.go

@@ -22,6 +22,8 @@ var (
 	vminsertAddrRE              = regexp.MustCompile(`accepting vminsert conns at (.*:\d{1,5})$`)
 	vminsertClusterNativeAddrRE = regexp.MustCompile(`started TCP clusternative server at "(.*:\d{1,5})"`)
 	vmselectAddrRE              = regexp.MustCompile(`accepting vmselect conns at (.*:\d{1,5})$`)
+
+	logsStorageDataPathRE = regexp.MustCompile(`opening storage at -storageDataPath=(.*)`)
 )
 
 // app represents an instance of some VictoriaMetrics server (such as vmstorage,

apptest/model.go

@@ -1,8 +1,11 @@
 package apptest
 
 import (
+	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"math"
 	"net/url"
 	"slices"
@@ -15,22 +18,22 @@ import (
 	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
-// PrometheusQuerier contains methods available to Prometheus-like HTTP API for Querying
-type PrometheusQuerier interface {
-	PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
-	PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
-	PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse
-	PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse
-	PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
+// APIQuerier contains methods available to Prometheus-like HTTP API for Querying
+type APIQuerier interface {
+	APIV1Export(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
+	APIV1Query(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
+	APIV1QueryRange(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse
+	APIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *APIV1SeriesResponse
+	APIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
 }
 
-// Writer contains methods for writing new data
-type Writer interface {
-	// Prometheus APIs
-	PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
-	PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
-	PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts)
-	PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts)
+// APIWriter contains methods for writing new data
+type APIWriter interface {
+	// Prometheus-like APIs
+	APIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts)
+	APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
+	APIV1ImportCSV(t *testing.T, records []string, opts QueryOpts)
+	APIV1ImportNative(t *testing.T, data []byte, opts QueryOpts)
 
 	// Graphit APIs
 	GraphiteWrite(t *testing.T, records []string, opts QueryOpts)
@@ -51,11 +54,11 @@ type StorageMerger interface {
 	ForceMerge(t *testing.T)
 }
 
-// PrometheusWriteQuerier encompasses the methods for writing, flushing and
+// WriteQuerier encompasses the methods for writing, flushing and
 // querying the data.
-type PrometheusWriteQuerier interface {
-	Writer
-	PrometheusQuerier
+type WriteQuerier interface {
+	APIWriter
+	APIQuerier
 	StorageFlusher
 	StorageMerger
 }
@@ -111,9 +114,33 @@ func (qos *QueryOpts) getTenant() string {
 	return qos.Tenant
 }
 
-// PrometheusAPIV1QueryResponse is an inmemory representation of the
+// QueryOptsLogs contains various params used for VictoriaLogs querying or ingesting data
+type QueryOptsLogs struct {
+	MessageField string
+	StreamFields string
+	TimeField    string
+}
+
+func (qos *QueryOptsLogs) asURLValues() url.Values {
+	uv := make(url.Values)
+	addNonEmpty := func(name string, values ...string) {
+		for _, value := range values {
+			if len(value) == 0 {
+				continue
+			}
+			uv.Add(name, value)
+		}
+	}
+	addNonEmpty("_time_field", qos.TimeField)
+	addNonEmpty("_stream_fields", qos.StreamFields)
+	addNonEmpty("_msg_field", qos.MessageField)
+
+	return uv
+}
+
+// APIV1QueryResponse is an inmemory representation of the
 // /prometheus/api/v1/query or /prometheus/api/v1/query_range response.
-type PrometheusAPIV1QueryResponse struct {
+type APIV1QueryResponse struct {
 	Status    string
 	Data      *QueryData
 	ErrorType string
@@ -121,12 +148,12 @@ type PrometheusAPIV1QueryResponse struct {
 	IsPartial bool
 }
 
-// NewPrometheusAPIV1QueryResponse is a test helper function that creates a new
-// instance of PrometheusAPIV1QueryResponse by unmarshalling a json string.
-func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1QueryResponse {
+// NewAPIV1QueryResponse is a test helper function that creates a new
+// instance of APIV1QueryResponse by unmarshalling a json string.
+func NewAPIV1QueryResponse(t *testing.T, s string) *APIV1QueryResponse {
 	t.Helper()
 
-	res := &PrometheusAPIV1QueryResponse{}
+	res := &APIV1QueryResponse{}
 	if err := json.Unmarshal([]byte(s), res); err != nil {
 		t.Fatalf("could not unmarshal query response data=\n%s\n: %v", string(s), err)
 	}
@@ -134,7 +161,7 @@ func NewPrometheusAPIV1QueryResponse(t *testing.T, s string) *PrometheusAPIV1Que
 }
 
 // Sort performs data.Result sort by metric labels
-func (pqr *PrometheusAPIV1QueryResponse) Sort() {
+func (pqr *APIV1QueryResponse) Sort() {
 	if pqr.Data == nil {
 		return
 	}
@@ -230,9 +257,9 @@ func (s *Sample) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
-// PrometheusAPIV1SeriesResponse is an inmemory representation of the
+// APIV1SeriesResponse is an inmemory representation of the
 // /prometheus/api/v1/series response.
-type PrometheusAPIV1SeriesResponse struct {
+type APIV1SeriesResponse struct {
 	Status    string
 	IsPartial bool
 	Data      []map[string]string
@@ -241,12 +268,12 @@ type PrometheusAPIV1SeriesResponse struct {
 	Error     string
 }
 
-// NewPrometheusAPIV1SeriesResponse is a test helper function that creates a new
-// instance of PrometheusAPIV1SeriesResponse by unmarshalling a json string.
-func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1SeriesResponse {
+// NewAPIV1SeriesResponse is a test helper function that creates a new
+// instance of APIV1SeriesResponse by unmarshalling a json string.
+func NewAPIV1SeriesResponse(t *testing.T, s string) *APIV1SeriesResponse {
 	t.Helper()
 
-	res := &PrometheusAPIV1SeriesResponse{}
+	res := &APIV1SeriesResponse{}
 	if err := json.Unmarshal([]byte(s), res); err != nil {
 		t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", string(s), err)
 	}
@@ -254,7 +281,7 @@ func NewPrometheusAPIV1SeriesResponse(t *testing.T, s string) *PrometheusAPIV1Se
 }
 
 // Sort sorts the response data.
-func (r *PrometheusAPIV1SeriesResponse) Sort() *PrometheusAPIV1SeriesResponse {
+func (r *APIV1SeriesResponse) Sort() *APIV1SeriesResponse {
 	str := func(m map[string]string) string {
 		s := []string{}
 		for k, v := range m {
@@ -368,6 +395,13 @@ type TSDBStatusResponse struct {
 	Data      TSDBStatusResponseData
 }
 
+// AdminTenantsResponse is an in-memory representation of the json response
+// returned by the /api/v1/admin/tenants endpoint.
+type AdminTenantsResponse struct {
+	Status string
+	Data   []string
+}
+
 // Sort performs sorting of stats entries
 func (tsr *TSDBStatusResponse) Sort() {
 	sortTSDBStatusResponseEntries(tsr.Data.SeriesCountByLabelName)
@@ -410,3 +444,44 @@ func sortTSDBStatusResponseEntries(entries []TSDBStatusResponseEntry) {
 		return left.Count < right.Count
 	})
 }
+
+// LogsQLQueryResponse is an in-memory representation of the
+// /select/logsql/query response.
+type LogsQLQueryResponse struct {
+	LogLines []string
+}
+
+// NewLogsQLQueryResponse is a test helper function that creates a new
+// instance of LogsQLQueryResponse by unmarshalling a json string.
+func NewLogsQLQueryResponse(t *testing.T, s string) *LogsQLQueryResponse {
+	t.Helper()
+	res := &LogsQLQueryResponse{}
+	if len(s) == 0 {
+		return res
+	}
+	bs := bytes.NewBufferString(s)
+	for {
+		logLine, err := bs.ReadString('\n')
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				if len(logLine) > 0 {
+					t.Fatalf("BUG: unexpected non-empty line=%q with io.EOF", logLine)
+				}
+				break
+			}
+			t.Fatalf("BUG: cannot read logline from buffer: %s", err)
+		}
+		var lv map[string]any
+		if err := json.Unmarshal([]byte(logLine), &lv); err != nil {
+			t.Fatalf("cannot parse log line=%q: %s", logLine, err)
+		}
+		delete(lv, "_stream_id")
+		normalizedLine, err := json.Marshal(lv)
+		if err != nil {
+			t.Fatalf("cannot marshal parsed logline=%q: %s", logLine, err)
+		}
+		res.LogLines = append(res.LogLines, string(normalizedLine))
+	}
+
+	return res
+}

apptest/testcase.go

@@ -146,7 +146,7 @@ func (tc *TestCase) MustStartVmagent(instance string, flags []string, promScrape
 // Vmcluster represents a typical cluster setup: several vmstorage replicas, one
 // vminsert, and one vmselect.
 //
-// Both Vmsingle and Vmcluster implement the PrometheusWriteQuerier used in
+// Both Vmsingle and Vmcluster implement the WriteQuerier used in
 // business logic tests to abstract out the infrasture.
 //
 // This type is not suitable for infrastructure tests where custom cluster
@@ -189,6 +189,36 @@ func (tc *TestCase) MustStartVmauth(instance string, flags []string, configFileY
 	return app
 }
 
+// MustStartVmbackup is a test helper that starts an instance of vmbackup
+// and waits until the app exits. It fails the test if the app fails to start or
+// exits with non zero code.
+func (tc *TestCase) MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string) {
+	tc.t.Helper()
+
+	if err := StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst); err != nil {
+		tc.t.Fatalf("vmbackup %q failed to start or exited with non-zero code: %v", instance, err)
+	}
+
+	// Do not add the process to the list of running apps using
+	// tc.addApp(instance, app), because the method blocks until the process
+	// exits.
+}
+
+// MustStartVmrestore is a test helper that starts an instance of vmrestore
+// and waits until the app exits. It fails the test if the app fails to start or
+// exits with non zero code.
+func (tc *TestCase) MustStartVmrestore(instance, src, storageDataPath string) {
+	tc.t.Helper()
+
+	if err := StartVmrestore(instance, src, storageDataPath); err != nil {
+		tc.t.Fatalf("vmrestore %q failed to start or exited with non-zero code: %v", instance, err)
+	}
+
+	// Do not add the process to the list of running apps using
+	// tc.addApp(instance, app), because the method blocks until the process
+	// exits.
+}
+
 // MustStartDefaultCluster starts a typical cluster configuration with default
 // flags.
 func (tc *TestCase) MustStartDefaultCluster() *Vmcluster {
@@ -254,15 +284,13 @@ func (tc *TestCase) MustStartCluster(opts *ClusterOptions) *Vmcluster {
 }
 
 // MustStartVmctl is a test helper function that starts an instance of vmctl
-func (tc *TestCase) MustStartVmctl(instance string, flags []string) *Vmctl {
+func (tc *TestCase) MustStartVmctl(instance string, flags []string) {
 	tc.t.Helper()
 
-	app, err := StartVmctl(instance, flags)
+	err := StartVmctl(instance, flags)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
-	tc.addApp(instance, app)
-	return app
 }
 
 func (tc *TestCase) addApp(instance string, app Stopper) {
@@ -281,8 +309,8 @@ func (tc *TestCase) StopApp(instance string) {
 	}
 }
 
-// StopPrometheusWriteQuerier stop all apps that are a part of the pwq.
-func (tc *TestCase) StopPrometheusWriteQuerier(pwq PrometheusWriteQuerier) {
+// StopWriteQuerier stop all apps that are a part of the pwq.
+func (tc *TestCase) StopWriteQuerier(pwq WriteQuerier) {
 	tc.t.Helper()
 	switch t := pwq.(type) {
 	case *Vmsingle:
@@ -381,3 +409,27 @@ func (tc *TestCase) Assert(opts *AssertOptions) {
 		tc.t.Error(msg)
 	}
 }
+
+// MustStartDefaultVlsingle is a test helper function that starts an instance of
+// vlsingle with defaults suitable for most tests.
+func (tc *TestCase) MustStartDefaultVlsingle() *Vlsingle {
+	tc.t.Helper()
+
+	return tc.MustStartVlsingle("vlsingle", []string{
+		"-storageDataPath=" + tc.Dir() + "/vlsingle",
+		"-retentionPeriod=100y",
+	})
+}
+
+// MustStartVlsingle is a test helper function that starts an instance of
+// vlsingle and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVlsingle(instance string, flags []string) *Vlsingle {
+	tc.t.Helper()
+
+	app, err := StartVlsingle(instance, flags, tc.cli)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}

apptest/tests/backup_restore_test.go

@@ -0,0 +1,239 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+type testBackupRestoreOpts struct {
+	startSUT           func() at.WriteQuerier
+	stopSUT            func()
+	storageDataPaths   []string
+	snapshotCreateURLs func(at.WriteQuerier) []string
+}
+
+func TestSingleBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testBackupRestoreOpts{
+		startSUT: func() at.WriteQuerier {
+			return tc.MustStartVmsingle("vmsingle", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopSUT: func() {
+			tc.StopApp("vmsingle")
+		},
+		storageDataPaths: []string{
+			storageDataPath,
+		},
+		snapshotCreateURLs: func(sut at.WriteQuerier) []string {
+			return []string{
+				sut.(*at.Vmsingle).SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testBackupRestore(tc, opts)
+}
+
+func TestClusterBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testBackupRestoreOpts{
+		startSUT: func() at.WriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1")
+			tc.StopApp("vmstorage2")
+		},
+		storageDataPaths: []string{
+			storage1DataPath,
+			storage2DataPath,
+		},
+		snapshotCreateURLs: func(sut at.WriteQuerier) []string {
+			c := sut.(*at.Vmcluster)
+			return []string{
+				c.Vmstorages[0].SnapshotCreateURL(),
+				c.Vmstorages[1].SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testBackupRestore(tc, opts)
+}
+
+func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
+	t := tc.T()
+
+	const msecPerMinute = 60 * 1000
+	genData := func(count int, prefix string, start int64) (recs []string, wantSeries []map[string]string, wantQueryResults []*at.QueryResult) {
+		recs = make([]string, count)
+		wantSeries = make([]map[string]string, count)
+		wantQueryResults = make([]*at.QueryResult, count)
+		for i := range count {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			value := float64(i)
+			timestamp := start + int64(i)*msecPerMinute
+
+			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
+			wantSeries[i] = map[string]string{"__name__": name}
+			wantQueryResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, wantSeries, wantQueryResults
+	}
+
+	backupBaseDir, err := filepath.Abs(filepath.Join(tc.Dir(), "backups"))
+	if err != nil {
+		t.Fatalf("could not get absolute path for the backup base dir")
+	}
+
+	// assertSeries retrieves set of all metric names from the storage and
+	// compares it with the expected set.
+	assertSeries := func(app at.APIQuerier, query string, start, end int64, want []map[string]string) {
+		t.Helper()
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.APIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.APIV1SeriesResponse{
+				Status: "success",
+				Data:   want,
+			},
+			FailNow: true,
+		})
+	}
+
+	// assertSeries retrieves all data from the storage and compares it with the
+	// expected result.
+	assertQueryResults := func(app at.APIQuerier, query string, start, end int64, want []*at.QueryResult) {
+		t.Helper()
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.APIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  "60s",
+				})
+			},
+			Want: &at.APIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     want,
+				},
+			},
+			FailNow: true,
+			Retries: 300,
+		})
+	}
+
+	createBackup := func(sut at.WriteQuerier, name string) {
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmbackup-%s-%s", name, replica)
+			snapshotCreateURL := opts.snapshotCreateURLs(sut)[i]
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, backupPath)
+		}
+	}
+
+	restoreFromBackup := func(name string) {
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmrestore-%s-%s", name, replica)
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmrestore(instance, backupPath, storageDataPath)
+		}
+	}
+
+	// Use the same number of metrics and time range for all the data ingestions
+	// below.
+	const numMetrics = 1000
+	// With 1000 metrics (one per minute), the time range spans 2 months.
+	end := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
+	start := end - numMetrics*msecPerMinute
+
+	// Verify backup/restore:
+	//
+	// - Start vmsingle with empty storage data dir.
+	// - Ingest first batch or records (batch1) and ensure they can be queried.
+	// - Create batch1 backup
+	// - Ingest second batch of records (batch2) and ensure the queries return
+	//   (batch1 + batch2) data.
+	// - Stop vmsingle
+	// - Restore batch1 from backup
+	// - Start vmsingle
+	// - Ensure that the queries return batch1 data only.
+
+	batch1Data, wantBatch1Series, wantBatch1QueryResults := genData(numMetrics, "batch1", start)
+	batch2Data, wantBatch2Series, wantBatch2QueryResults := genData(numMetrics, "batch2", start)
+	wantBatch12Series := slices.Concat(wantBatch1Series, wantBatch2Series)
+	wantBatch12QueryResults := slices.Concat(wantBatch1QueryResults, wantBatch2QueryResults)
+
+	sut := opts.startSUT()
+
+	sut.APIV1ImportPrometheus(t, batch1Data, at.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+	createBackup(sut, "batch1")
+
+	sut.APIV1ImportPrometheus(t, batch2Data, at.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12Series)
+	assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12QueryResults)
+	createBackup(sut, "batch12")
+
+	opts.stopSUT()
+
+	restoreFromBackup("batch1")
+
+	sut = opts.startSUT()
+
+	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+}

apptest/tests/dedup_test.go

@@ -5,12 +5,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 )
 
 func TestSingleDeduplication_dedulicationIsOff(t *testing.T) {
@@ -80,7 +81,7 @@ func TestClusterDeduplication_deduplicationIsOn(t *testing.T) {
 }
 
 // See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication
-func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplicationIsOn bool) {
+func testDeduplication(tc *at.TestCase, sut at.WriteQuerier, deduplicationIsOn bool) {
 	t := tc.T()
 
 	firstDayOfThisMonth := func() time.Time {
@@ -134,11 +135,11 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
 		},
 	}
 
-	sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
+	sut.APIV1Write(t, data, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 	sut.ForceMerge(t)
 
-	wantDuplicates := &at.PrometheusAPIV1QueryResponse{
+	wantDuplicates := &at.APIV1QueryResponse{
 		Status: "success",
 		Data: &at.QueryData{
 			ResultType: "matrix",
@@ -166,7 +167,7 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
 			},
 		},
 	}
-	wantDeduped := &at.PrometheusAPIV1QueryResponse{
+	wantDeduped := &at.APIV1QueryResponse{
 		Status: "success",
 		Data: &at.QueryData{
 			ResultType: "matrix",
@@ -207,7 +208,7 @@ func testDeduplication(tc *at.TestCase, sut at.PrometheusWriteQuerier, deduplica
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected response",
 		Got: func() any {
-			got := sut.PrometheusAPIV1Export(t, `{__name__=~"metric.*"}`, apptest.QueryOpts{
+			got := sut.APIV1Export(t, `{__name__=~"metric.*"}`, apptest.QueryOpts{
 				ReduceMemUsage: "1",
 				Start:          fmt.Sprintf("%d", start.UnixMilli()),
 				End:            fmt.Sprintf("%d", end.UnixMilli()),

apptest/tests/export_import_test.go

@@ -33,9 +33,9 @@ func TestClusterExportImportNative(t *testing.T) {
 
 // testExportImportNative test export and import in VictoriaMetrics’ native format.
 // see: https://docs.victoriametrics.com/#how-to-import-data-in-native-format
-func testExportImportNative(t *testing.T, sut at.PrometheusWriteQuerier) {
+func testExportImportNative(t *testing.T, sut at.WriteQuerier) {
 	// create test data
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`native_export_import 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 	}, at.QueryOpts{
 		ExtraLabels: []string{"el1=elv1", "el2=elv2"},
@@ -43,27 +43,27 @@ func testExportImportNative(t *testing.T, sut at.PrometheusWriteQuerier) {
 	sut.ForceFlush(t)
 
 	// export test data via native export API
-	exportResult := sut.PrometheusAPIV1ExportNative(t, "native_export_import", at.QueryOpts{
+	exportResult := sut.APIV1ExportNative(t, "native_export_import", at.QueryOpts{
 		Start: "2024-02-05T08:50:00.700Z",
 		End:   "2024-02-05T09:00:00.700Z",
 	})
 
 	// re-import test data via native import API
-	sut.PrometheusAPIV1ImportNative(t, exportResult, at.QueryOpts{})
+	sut.APIV1ImportNative(t, exportResult, at.QueryOpts{})
 	sut.ForceFlush(t)
 
 	// check query result
-	got := sut.PrometheusAPIV1QueryRange(t, "native_export_import", at.QueryOpts{
+	got := sut.APIV1QueryRange(t, "native_export_import", at.QueryOpts{
 		Start: "2024-02-05T08:57:36.700Z",
 		End:   "2024-02-05T08:57:36.700Z",
 		Step:  "60s",
 	})
 
 	cmpOptions := []cmp.Option{
-		cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 		cmpopts.EquateNaNs(),
 	}
-	want := at.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "native_export_import", "el1": "elv1", "el2":"elv2"}, "values": []}]}}`)
+	want := at.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "native_export_import", "el1": "elv1", "el2":"elv2"}, "values": []}]}}`)
 	want.Data.Result[0].Samples = []*at.Sample{
 		at.NewSample(t, "2024-02-05T08:57:36.700Z", 10),
 	}

apptest/tests/ingestprotocols_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -21,7 +22,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
 		wantMetrics []map[string]string
 		wantSamples []*at.Sample
 	}
-	f := func(sut at.PrometheusQuerier, opts *opts) {
+	f := func(sut at.APIQuerier, opts *opts) {
 		t.Helper()
 		wantResult := []*at.QueryResult{}
 		for idx, wm := range opts.wantMetrics {
@@ -34,16 +35,16 @@ func TestSingleIngestionProtocols(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /export query response",
 			Got: func() any {
-				got := sut.PrometheusAPIV1Export(t, opts.query, at.QueryOpts{
+				got := sut.APIV1Export(t, opts.query, at.QueryOpts{
 					Start: "2024-02-05T08:50:00.700Z",
 					End:   "2024-02-05T09:00:00.700Z",
 				})
 				got.Sort()
 				return got
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
+			Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+				cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 			},
 		})
 	}
@@ -108,7 +109,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
 	})
 
 	// CSV import
-	sut.PrometheusAPIV1ImportCSV(t, []string{
+	sut.APIV1ImportCSV(t, []string{
 		`GOOG,1.23,4.56,NYSE,1707123457`,
 		`MSFT,23,56,NASDAQ,1707123457`,
 	}, at.QueryOpts{
@@ -157,7 +158,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
 	})
 
 	// prometheus text exposition format
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`importprometheus_series 10 1707123456700`,                               // 2024-02-05T08:57:36.700Z
 		`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
 	}, at.QueryOpts{
@@ -226,7 +227,7 @@ func TestSingleIngestionProtocols(t *testing.T) {
 			},
 		},
 	}
-	sut.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
+	sut.APIV1Write(t, pbData, at.QueryOpts{})
 	sut.ForceFlush(t)
 	f(sut, &opts{
 		query: `{__name__=~"prometheusrw.+"}`,
@@ -281,22 +282,22 @@ func TestClusterIngestionProtocols(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /export query response",
 			Got: func() any {
-				got := vmselect.PrometheusAPIV1Export(t, opts.query, at.QueryOpts{
+				got := vmselect.APIV1Export(t, opts.query, at.QueryOpts{
 					Start: "2024-02-05T08:50:00.700Z",
 					End:   "2024-02-05T09:00:00.700Z",
 				})
 				got.Sort()
 				return got
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
+			Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+				cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 			},
 		})
 	}
 
 	// prometheus text exposition format
-	vminsert.PrometheusAPIV1ImportPrometheus(t, []string{
+	vminsert.APIV1ImportPrometheus(t, []string{
 		`importprometheus_series 10 1707123456700`,                               // 2024-02-05T08:57:36.700Z
 		`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
 	}, at.QueryOpts{
@@ -357,7 +358,7 @@ func TestClusterIngestionProtocols(t *testing.T) {
 	})
 
 	// CSV import
-	vminsert.PrometheusAPIV1ImportCSV(t, []string{
+	vminsert.APIV1ImportCSV(t, []string{
 		`GOOG,1.23,4.56,NYSE,1707123457`, // 2024-02-05T08:57:37.000Z
 		`MSFT,23,56,NASDAQ,1707123457`,   // 2024-02-05T08:57:37.000Z
 	}, at.QueryOpts{
@@ -473,7 +474,7 @@ func TestClusterIngestionProtocols(t *testing.T) {
 			},
 		},
 	}
-	vminsert.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
+	vminsert.APIV1Write(t, pbData, at.QueryOpts{})
 	vmstorage.ForceFlush(t)
 	f(&opts{
 		query: `{__name__=~"prometheusrw.+"}`,
@@ -493,3 +494,65 @@ func TestClusterIngestionProtocols(t *testing.T) {
 		},
 	})
 }
+
+func TestVlsingleIngestionProtocols(t *testing.T) {
+	os.RemoveAll(t.Name())
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+	sut := tc.MustStartDefaultVlsingle()
+	type opts struct {
+		query        string
+		wantLogLines []string
+	}
+
+	f := func(opts *opts) {
+		t.Helper()
+		sut.ForceFlush(t)
+		got := sut.LogsQLQuery(t, opts.query, at.QueryOptsLogs{})
+		assertLogsQLResponseEqual(t, got, &at.LogsQLQueryResponse{LogLines: opts.wantLogLines})
+	}
+	// json line ingest
+	sut.JSONLineWrite(t, []string{
+		`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
+		`{"_msg":"ingest jsonline","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
+	}, at.QueryOptsLogs{})
+	f(&opts{
+		query: "ingest jsonline",
+		wantLogLines: []string{
+			`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
+			`{"_msg":"ingest jsonline","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
+		},
+	})
+	// native format ingest
+	sut.NativeWrite(t, []logstorage.InsertRow{
+		{
+			StreamTagsCanonical: canonicalStreamTagsFromSet(map[string]string{"foo": "bar"}),
+			Timestamp:           1749141697409000000, // 2025-06-05T:18:41:37.000000Z
+			Fields: []logstorage.Field{
+				{
+					Name:  "_msg",
+					Value: "ingest native",
+				},
+				{
+					Name:  "qwe",
+					Value: "rty",
+				},
+			},
+		},
+	}, at.QueryOpts{})
+	f(&opts{
+		query: "ingest native",
+		wantLogLines: []string{
+			`{"_msg":"ingest native","_time":"2025-06-05T16:41:37.409Z", "_stream":"{foo=\"bar\"}", "qwe": "rty"}`,
+		},
+	})
+
+}
+
+func canonicalStreamTagsFromSet(set map[string]string) string {
+	var st logstorage.StreamTags
+	for key, value := range set {
+		st.Add(key, value)
+	}
+	return string(st.MarshalCanonical(nil))
+}

apptest/tests/key_concepts_test.go

@@ -1,6 +1,10 @@
 package tests
 
 import (
+	"encoding/json"
+	"os"
+	"sort"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -51,10 +55,10 @@ func TestClusterKeyConceptsQueryData(t *testing.T) {
 }
 
 // testKeyConceptsQueryData verifies cases from https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data
-func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
+func testKeyConceptsQueryData(t *testing.T, sut at.WriteQuerier) {
 
 	// Insert example data from documentation.
-	sut.PrometheusAPIV1ImportPrometheus(t, docData, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, docData, at.QueryOpts{})
 	sut.ForceFlush(t)
 
 	testInstantQuery(t, sut)
@@ -65,14 +69,14 @@ func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
 // testInstantQuery verifies the statements made in the `Instant query` section
 // of the VictoriaMetrics documentation. See:
 // https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query
-func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
+func testInstantQuery(t *testing.T, q at.APIQuerier) {
 	// Get the value of the foo_bar time series at 2022-05-10T08:03:00Z with the
 	// step of 5m and timeout 5s. There is no sample at exactly this timestamp.
 	// Therefore, VictoriaMetrics will search for the nearest sample within the
 	// [time-5m..time] interval.
-	got := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
-	want := at.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
-	opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+	got := q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
+	want := at.NewAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
+	opt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
 	if diff := cmp.Diff(want, got, opt); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -82,7 +86,7 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
 	// Therefore, VictoriaMetrics will search for the nearest sample within the
 	// [time-1m..time] interval. Since the nearest sample is 2m away and the
 	// step is 1m, then the VictoriaMetrics must return empty response.
-	got = q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
+	got = q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
 	if len(got.Data.Result) > 0 {
 		t.Errorf("unexpected response: got non-empty result, want empty result:\n%v", got)
 	}
@@ -91,14 +95,14 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
 // testRangeQuery verifies the statements made in the `Range query` section of
 // the VictoriaMetrics documentation. See:
 // https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query
-func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
+func testRangeQuery(t *testing.T, q at.APIQuerier) {
 	f := func(start, end, step string, wantSamples []*at.Sample) {
 		t.Helper()
 
-		got := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{Start: start, End: end, Step: step})
-		want := at.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
+		got := q.APIV1QueryRange(t, "foo_bar", at.QueryOpts{Start: start, End: end, Step: step})
+		want := at.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
 		want.Data.Result[0].Samples = wantSamples
-		opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+		opt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
 		if diff := cmp.Diff(want, got, opt); diff != "" {
 			t.Errorf("unexpected response (-want, +got):\n%s", diff)
 		}
@@ -164,11 +168,11 @@ func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
 // will not produce ephemeral points.
 //
 // See: https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query
-func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.PrometheusQuerier) {
+func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.APIQuerier) {
 	f := func(timestamp string, want *at.Sample) {
 		t.Helper()
 
-		gotInstant := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: timestamp, Step: "1m"})
+		gotInstant := q.APIV1Query(t, "foo_bar", at.QueryOpts{Time: timestamp, Step: "1m"})
 		if want == nil {
 			if got, want := len(gotInstant.Data.Result), 0; got != want {
 				t.Errorf("unexpected instant result size: got %d, want %d", got, want)
@@ -181,7 +185,7 @@ func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.Prometheu
 		}
 	}
 
-	rangeRes := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{
+	rangeRes := q.APIV1QueryRange(t, "foo_bar", at.QueryOpts{
 		Start: "2022-05-10T07:59:00.000Z",
 		End:   "2022-05-10T08:17:00.000Z",
 		Step:  "1m",
@@ -227,7 +231,7 @@ func TestClusterMillisecondPrecisionInInstantQueries(t *testing.T) {
 	testMillisecondPrecisionInInstantQueries(tc, sut)
 }
 
-func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	type opts struct {
@@ -238,7 +242,7 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
 		wantSample  *at.Sample
 		wantSamples []*at.Sample
 	}
-	f := func(sut at.PrometheusQuerier, opts *opts) {
+	f := func(sut at.APIQuerier, opts *opts) {
 		t.Helper()
 		wantResult := []*at.QueryResult{}
 		if opts.wantMetric != nil && (opts.wantSample != nil || len(opts.wantSamples) > 0) {
@@ -251,19 +255,19 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query response",
 			Got: func() any {
-				return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
+				return sut.APIV1Query(t, opts.query, at.QueryOpts{
 					Time: opts.qtime,
 					Step: opts.step,
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
+			Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+				cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 			},
 		})
 	}
 
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`series1{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 		`series1{label="foo"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
 	}, at.QueryOpts{})
@@ -322,7 +326,7 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
 
 	// Insert samples with different dates. The difference in ms between the two
 	// timestamps is 4236579304.
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`series2{label="foo"} 10 1638564958042`, // 2021-12-03T20:55:58.042Z
 		`series2{label="foo"} 20 1642801537346`, // 2022-01-21T21:45:37.346Z
 	}, at.QueryOpts{})
@@ -367,3 +371,131 @@ func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.Prometheus
 		wantSample: &at.Sample{Timestamp: 1642801537346, Value: 2},
 	})
 }
+
+// TestVlsingleKeyConcepts verifies cases from https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model
+// for vl-single.
+func TestVlsingleKeyConcepts(t *testing.T) {
+	os.RemoveAll(t.Name())
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+	sut := tc.MustStartDefaultVlsingle()
+
+	type opts struct {
+		ingestRecords   []string
+		ingestQueryArgs at.QueryOptsLogs
+		wantResponse    *at.LogsQLQueryResponse
+		query           string
+		selectQueryArgs at.QueryOptsLogs
+	}
+
+	f := func(opts *opts) {
+		t.Helper()
+		sut.JSONLineWrite(t, opts.ingestRecords, opts.ingestQueryArgs)
+		sut.ForceFlush(t)
+		got := sut.LogsQLQuery(t, opts.query, opts.selectQueryArgs)
+		assertLogsQLResponseEqual(t, got, opts.wantResponse)
+	}
+
+	// nested objects flatten
+	f(&opts{
+		ingestRecords: []string{
+			`{"_msg":"case 1","_time": "2025-06-05T14:30:19.088007Z", "host": {"name": "foobar","os": {"version": "1.2.3"}}}`,
+			`{"_msg":"case 1","_time": "2025-06-05T14:30:19.088007Z", "tags": ["foo", "bar"], "offset": 12345, "is_error": false}`,
+		},
+		wantResponse: &at.LogsQLQueryResponse{
+			LogLines: []string{
+				`{"_msg":"case 1","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","host.name":"foobar","host.os.version":"1.2.3"}`,
+				`{"_msg":"case 1","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","is_error":"false","offset":"12345","tags":"[\"foo\",\"bar\"]"}`,
+			},
+		},
+		query: "case 1",
+	})
+
+	// obtain _msg value from non-default field
+	f(&opts{
+		ingestRecords: []string{
+			`{"my_msg":"case 2","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
+			`{"my_msg_other":"case 2","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
+		},
+		ingestQueryArgs: at.QueryOptsLogs{
+			MessageField: "my_msg,my_msg_other",
+		},
+		query: "case 2",
+		wantResponse: &at.LogsQLQueryResponse{
+			LogLines: []string{
+				`{"_msg":"case 2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
+				`{"_msg":"case 2","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
+			},
+		},
+	})
+
+	// populate stream fields
+	f(&opts{
+		ingestRecords: []string{
+			`{"my_msg":"case 3","_time": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
+			`{"my_msg":"case 3","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
+			`{"my_msg":"case 3","_time": "2025-06-05T14:30:19.088007Z", "bar":"foo","foo":"bar","baz":"bar"}`,
+		},
+		ingestQueryArgs: at.QueryOptsLogs{
+			MessageField: "my_msg",
+			StreamFields: "foo,bar,baz",
+		},
+		wantResponse: &at.LogsQLQueryResponse{
+			LogLines: []string{
+				`{"_msg":"case 3","_stream":"{foo=\"bar\"}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
+				`{"_msg":"case 3","_stream":"{bar=\"foo\"}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
+				`{"_msg":"case 3","_stream":"{bar=\"foo\",baz=\"bar\",foo=\"bar\"}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo","foo":"bar","baz":"bar"}`,
+			},
+		},
+		query: "case 3",
+	})
+
+	// obtain _time value from non-default field
+	f(&opts{
+		ingestRecords: []string{
+			`{"_msg":"case 4","my_time_field": "2025-06-05T14:30:19.088007Z", "foo":"bar"}`,
+			`{"_msg":"case 4","my_other_time_field": "2025-06-05T14:30:19.088007Z", "bar":"foo"}`,
+		},
+		ingestQueryArgs: at.QueryOptsLogs{
+			TimeField: "my_time_field,my_other_time_field",
+		},
+		wantResponse: &at.LogsQLQueryResponse{
+			LogLines: []string{
+				`{"_msg":"case 4","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","foo":"bar"}`,
+				`{"_msg":"case 4","_stream":"{}","_time":"2025-06-05T14:30:19.088007Z","bar":"foo"}`,
+			},
+		},
+		query: "case 4",
+	})
+
+}
+
+func assertLogsQLResponseEqual(t *testing.T, got, want *at.LogsQLQueryResponse) {
+	t.Helper()
+	sort.Strings(got.LogLines)
+	sort.Strings(want.LogLines)
+	if len(got.LogLines) != len(want.LogLines) {
+		t.Errorf("unexpected response len: -%d: +%d\n%s", len(want.LogLines), len(got.LogLines), strings.Join(got.LogLines, "\n"))
+		return
+	}
+	for i := range len(want.LogLines) {
+		gotLine, wantLine := got.LogLines[i], want.LogLines[i]
+		var gotLineJSON map[string]any
+		var wantLineJSON map[string]any
+		if err := json.Unmarshal([]byte(gotLine), &gotLineJSON); err != nil {
+			t.Errorf("cannot parse got line=%q: %s", gotLine, err)
+			return
+		}
+		if err := json.Unmarshal([]byte(wantLine), &wantLineJSON); err != nil {
+			t.Errorf("cannot parse want line=%q: %s", wantLine, err)
+			return
+		}
+		// stream_id is always unique, remove it from comparison
+		delete(gotLineJSON, "_stream_id")
+		delete(wantLineJSON, "_stream_id")
+		if diff := cmp.Diff(gotLineJSON, wantLineJSON); diff != "" {
+			t.Errorf("unexpected response (-want, +got):\n%s\n%s\n%s", diff, wantLine, gotLine)
+			return
+		}
+	}
+}

apptest/tests/maxingestionrate_test.go

@@ -16,7 +16,7 @@ func TestSingleMaxIngestionRateIncrementsMetric(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
 	sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=1"})
-	sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
 	if got := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"); got <= 0 {
 		t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
 	}
@@ -26,7 +26,7 @@ func TestSingleMaxIngestionRateDoesNotIncrementMetric(t *testing.T) {
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
 	sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=15"})
-	sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
 	if got, want := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"), 0.0; got != want {
 		t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
 	}

apptest/tests/metric_names_stats_test.go

@@ -37,7 +37,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 	}
 	tsdbMetricNameEntryCmpOpts := cmpopts.IgnoreFields(apptest.TSDBStatusResponseMetricNameEntry{}, "LastRequestTimestamp")
 
-	sut.PrometheusAPIV1ImportPrometheus(t, dataSet, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, dataSet, at.QueryOpts{})
 	sut.ForceFlush(t)
 
 	// verify ingest request correctly registered
@@ -55,7 +55,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 	}
 
 	// verify query request correctly registered
-	sut.PrometheusAPIV1Query(t, `{__name__!=""}`, at.QueryOpts{Time: ingestDateTime})
+	sut.APIV1Query(t, `{__name__!=""}`, at.QueryOpts{Time: ingestDateTime})
 	expected = apptest.MetricNamesStatsResponse{
 		Records: []at.MetricNamesStatsRecord{
 			{MetricName: largeMetricName, QueryRequestsCount: 1},
@@ -97,7 +97,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 	}
 
 	// perform query request for single metric and check counter increase
-	sut.PrometheusAPIV1Query(t, `metric_name_2`, at.QueryOpts{Time: ingestDateTime})
+	sut.APIV1Query(t, `metric_name_2`, at.QueryOpts{Time: ingestDateTime})
 	expected = apptest.MetricNamesStatsResponse{
 		Records: []at.MetricNamesStatsRecord{
 			{MetricName: largeMetricName, QueryRequestsCount: 1},
@@ -187,7 +187,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 	// ingest per tenant data and verify it with search
 	tenantIDs := []string{"1:1", "1:15", "15:15"}
 	for _, tenantID := range tenantIDs {
-		vminsert.PrometheusAPIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{Tenant: tenantID})
+		vminsert.APIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{Tenant: tenantID})
 		vmstorage1.ForceFlush(t)
 		vmstorage2.ForceFlush(t)
 
@@ -206,7 +206,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 		}
 
 		// verify query request registered correctly
-		vmselect.PrometheusAPIV1Query(t, `{__name__!=""}`, apptest.QueryOpts{
+		vmselect.APIV1Query(t, `{__name__!=""}`, apptest.QueryOpts{
 			Tenant: tenantID, Time: ingestDateTime,
 		})
 

apptest/tests/metricsql_test.go

@@ -46,7 +46,7 @@ func TestClusterInstantQuery(t *testing.T) {
 	testQueryRangeWithAtModifier(t, sut)
 }
 
-func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQuerier) {
+func testInstantQueryWithUTFNames(t *testing.T, sut apptest.WriteQuerier) {
 	data := []pb.TimeSeries{
 		{
 			Labels: []pb.Label{
@@ -59,18 +59,18 @@ func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQueri
 		},
 	}
 
-	sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
+	sut.APIV1Write(t, data, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 
-	var got, want *apptest.PrometheusAPIV1QueryResponse
+	var got, want *apptest.APIV1QueryResponse
 	cmpOptions := []cmp.Option{
-		cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 		cmpopts.EquateNaNs(),
 	}
 
-	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "3fooµ¥", "3👋tfにちは": "漢©®€£"}}]}}`)
+	want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "3fooµ¥", "3👋tfにちは": "漢©®€£"}}]}}`)
 	fn := func(query string) {
-		got = sut.PrometheusAPIV1Query(t, query, apptest.QueryOpts{
+		got = sut.APIV1Query(t, query, apptest.QueryOpts{
 			Step: "5m",
 			Time: "2024-01-01T00:01:00.000Z",
 		})
@@ -112,24 +112,24 @@ var staleNaNsData = func() []pb.TimeSeries {
 	}
 }()
 
-func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.PrometheusWriteQuerier) {
+func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.WriteQuerier) {
 
-	sut.PrometheusAPIV1Write(t, staleNaNsData, apptest.QueryOpts{})
+	sut.APIV1Write(t, staleNaNsData, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 
-	var got, want *apptest.PrometheusAPIV1QueryResponse
+	var got, want *apptest.APIV1QueryResponse
 	cmpOptions := []cmp.Option{
-		cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+		cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 		cmpopts.EquateNaNs(),
 	}
 
 	// Verify that instant query returns the first point.
 
-	got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
+	got = sut.APIV1Query(t, "metric", apptest.QueryOpts{
 		Step: "5m",
 		Time: "2024-01-01T00:01:00.000Z",
 	})
-	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
+	want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}}]}}`)
 	want.Data.Result[0].Sample = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
 	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
@@ -137,11 +137,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
 
 	// Verify that instant query does not return stale NaN.
 
-	got = sut.PrometheusAPIV1Query(t, "metric", apptest.QueryOpts{
+	got = sut.APIV1Query(t, "metric", apptest.QueryOpts{
 		Step: "5m",
 		Time: "2024-01-01T00:02:00.000Z",
 	})
-	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": []}}`)
+	want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": []}}`)
 	// Empty response, stale NaN is not included into response
 	if diff := cmp.Diff(want, got, cmpOptions...); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
@@ -151,11 +151,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
 	// while it must not.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5806
 
-	got = sut.PrometheusAPIV1Query(t, "metric[2m]", apptest.QueryOpts{
+	got = sut.APIV1Query(t, "metric[2m]", apptest.QueryOpts{
 		Step: "5m",
 		Time: "2024-01-01T00:02:00.000Z",
 	})
-	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
+	want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
 	s := make([]*apptest.Sample, 2)
 	s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
 	s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
@@ -166,11 +166,11 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
 
 	// Verify that exported data contains stale NaN.
 
-	got = sut.PrometheusAPIV1Export(t, `{__name__="metric"}`, apptest.QueryOpts{
+	got = sut.APIV1Export(t, `{__name__="metric"}`, apptest.QueryOpts{
 		Start: "2024-01-01T00:01:00.000Z",
 		End:   "2024-01-01T00:02:00.000Z",
 	})
-	want = apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
+	want = apptest.NewAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "metric"}, "values": []}]}}`)
 	s = make([]*apptest.Sample, 2)
 	s[0] = apptest.NewSample(t, "2024-01-01T00:01:00Z", 1)
 	s[1] = apptest.NewSample(t, "2024-01-01T00:02:00Z", decimal.StaleNaN)
@@ -184,7 +184,7 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
 // See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8444
 // However, conversion of math.NaN to int64 could behave differently depending on platform and Go version.
 // Hence, this test could succeed for some platforms even if fix is rolled back.
-func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQuerier) {
+func testQueryRangeWithAtModifier(t *testing.T, sut apptest.WriteQuerier) {
 	data := []pb.TimeSeries{
 		{
 			Labels: []pb.Label{
@@ -204,10 +204,10 @@ func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQueri
 		},
 	}
 
-	sut.PrometheusAPIV1Write(t, data, apptest.QueryOpts{})
+	sut.APIV1Write(t, data, apptest.QueryOpts{})
 	sut.ForceFlush(t)
 
-	resp := sut.PrometheusAPIV1QueryRange(t, `vector(1) @ up`, apptest.QueryOpts{
+	resp := sut.APIV1QueryRange(t, `vector(1) @ up`, apptest.QueryOpts{
 		Start: "2025-01-01T00:00:00Z",
 		End:   "2025-01-01T00:02:00Z",
 		Step:  "10s",
@@ -217,7 +217,7 @@ func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQueri
 		t.Fatalf("unexpected status: %q", resp.Status)
 	}
 
-	resp = sut.PrometheusAPIV1QueryRange(t, `vector(1) @ metricNaN`, apptest.QueryOpts{
+	resp = sut.APIV1QueryRange(t, `vector(1) @ metricNaN`, apptest.QueryOpts{
 		Start: "2025-01-01T00:00:00Z",
 		End:   "2025-01-01T00:02:00Z",
 		Step:  "10s",

apptest/tests/multilevel_test.go

@@ -37,7 +37,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
 
 	const numMetrics = 1000
 	records := make([]string, numMetrics)
-	want := &apptest.PrometheusAPIV1SeriesResponse{
+	want := &apptest.APIV1SeriesResponse{
 		Status:    "success",
 		IsPartial: false,
 		Data:      make([]map[string]string, numMetrics),
@@ -49,7 +49,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
 	}
 	want.Sort()
 	qopts := apptest.QueryOpts{Tenant: "0"}
-	vminsert.PrometheusAPIV1ImportPrometheus(t, records, qopts)
+	vminsert.APIV1ImportPrometheus(t, records, qopts)
 	vmstorage.ForceFlush(t)
 
 	// Retrieve all time series and verify that both vmselect (L1) and
@@ -60,7 +60,7 @@ func TestClusterMultilevelSelect(t *testing.T) {
 		tc.Assert(&apptest.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, qopts)
+				res := app.APIV1Series(t, `{__name__=~".*"}`, qopts)
 				res.Sort()
 				return res
 			},

apptest/tests/multitenant_test.go

@@ -13,8 +13,8 @@ import (
 func TestClusterMultiTenantSelect(t *testing.T) {
 	os.RemoveAll(t.Name())
 
-	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
-	cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
+	cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
+	cmpSROpt := cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Status", "IsPartial")
 
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
@@ -37,12 +37,12 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	}
 
 	// test for empty tenants request
-	got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
+	got := vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "multitenant",
 		Step:   "5m",
 		Time:   "2022-05-10T08:03:00.000Z",
 	})
-	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
+	want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
 	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -51,12 +51,12 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	tenantIDs := []string{"1:1", "1:15"}
 	instantCT := "2022-05-10T08:05:00.000Z"
 	for _, tenantID := range tenantIDs {
-		vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
+		vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
 		vmstorage.ForceFlush(t)
-		got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
+		got := vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
 			Tenant: tenantID, Time: instantCT,
 		})
-		want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
+		want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
 		if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
 			t.Errorf("unexpected response (-want, +got):\n%s", diff)
 		}
@@ -64,7 +64,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	// verify all tenants searchable with multitenant APIs
 
 	//  /api/v1/query
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+	want = apptest.NewAPIV1QueryResponse(t,
 		`{"data":
        {"result":[
           {"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id": "1"},"value":[1652169900,"3"]},
@@ -73,7 +73,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
        }
      }`,
 	)
-	got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
+	got = vmselect.APIV1Query(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "multitenant",
 		Time:   instantCT,
 	})
@@ -83,14 +83,14 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 
 	// /api/v1/query_range aggregated by tenant labels
 	query := "sum(foo_bar) by(vm_account_id,vm_project_id)"
-	got = vmselect.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
+	got = vmselect.APIV1QueryRange(t, query, apptest.QueryOpts{
 		Tenant: "multitenant",
 		Start:  "2022-05-10T07:59:00.000Z",
 		End:    "2022-05-10T08:05:00.000Z",
 		Step:   "1m",
 	})
 
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+	want = apptest.NewAPIV1QueryResponse(t,
 		`{"data": 
         {"result": [
           {"metric": {"vm_account_id": "1","vm_project_id":"1"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
@@ -104,7 +104,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 
 	// verify /api/v1/series response
 
-	wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
+	wantSR := apptest.NewAPIV1SeriesResponse(t,
 		`{"data": [
         {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
         {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
@@ -112,7 +112,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
      }`)
 	wantSR.Sort()
 
-	gotSR := vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+	gotSR := vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "multitenant",
 		Start:  "2022-05-10T08:03:00.000Z",
 	})
@@ -129,11 +129,11 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 		`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
 	}
 
-	vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
+	vminsert.APIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
 	vmstorage.ForceFlush(t)
 
 	//  /api/v1/query with query filters
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+	want = apptest.NewAPIV1QueryResponse(t,
 		`{"data":
         {"result":[
 	         {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"value":[1652169900,"1"]},
@@ -142,7 +142,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
         }
     }`,
 	)
-	got = vmselect.PrometheusAPIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
+	got = vmselect.APIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
 		Time:   instantCT,
 		Tenant: "multitenant",
 	})
@@ -152,14 +152,14 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 
 	// /api/v1/series with extra_filters
 
-	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+	wantSR = apptest.NewAPIV1SeriesResponse(t,
 		`{"data": [
 	       {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"15"},
 	       {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
 	             ]
 	   }`)
 	wantSR.Sort()
-	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+	gotSR = vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
 		Start:        "2022-05-10T08:00:00.000Z",
 		End:          "2022-05-10T08:30:00.000Z",
 		ExtraFilters: []string{`{vm_project_id="15"}`},
@@ -175,7 +175,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	vmselect.DeleteSeries(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "5:15",
 	})
-	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+	wantSR = apptest.NewAPIV1SeriesResponse(t,
 		`{"data": [
         {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
         {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
@@ -185,7 +185,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
      }`)
 	wantSR.Sort()
 
-	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
+	gotSR = vmselect.APIV1Series(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "multitenant",
 		Start:  "2022-05-10T08:03:00.000Z",
 	})
@@ -199,7 +199,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 		Tenant: "multitenant",
 	})
 
-	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
+	wantSR = apptest.NewAPIV1SeriesResponse(t,
 		`{"data": [
         {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
         {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
@@ -207,7 +207,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
      }`)
 	wantSR.Sort()
 
-	gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{
+	gotSR = vmselect.APIV1Series(t, `foo_bar`, apptest.QueryOpts{
 		Tenant: "multitenant",
 		Start:  "2022-05-10T08:03:00.000Z",
 	})

apptest/tests/per_day_index_test.go

@@ -11,7 +11,7 @@ func TestSingleSearchWithDisabledPerDayIndex(t *testing.T) {
 	tc := at.NewTestCase(t)
 	defer tc.Stop()
 
-	testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier {
+	testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.WriteQuerier {
 		return tc.MustStartVmsingle("vmsingle-"+name, []string{
 			"-storageDataPath=" + tc.Dir() + "/vmsingle",
 			"-retentionPeriod=100y",
@@ -25,7 +25,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
 	tc := at.NewTestCase(t)
 	defer tc.Stop()
 
-	testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier {
+	testSearchWithDisabledPerDayIndex(tc, func(name string, disablePerDayIndex bool) at.WriteQuerier {
 		// Using static ports for vmstorage because random ports may cause
 		// changes in how data is sharded.
 		vmstorage1 := tc.MustStartVmstorage("vmstorage1-"+name, []string{
@@ -59,7 +59,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
 	})
 }
 
-type startSUTFunc func(name string, disablePerDayIndex bool) at.PrometheusWriteQuerier
+type startSUTFunc func(name string, disablePerDayIndex bool) at.WriteQuerier
 
 // testDisablePerDayIndex_Search shows what search results to expect when data
 // is first inserted with per-day index enabled and then with per-day index
@@ -78,17 +78,17 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
 		wantSeries       []map[string]string
 		wantQueryResults []*at.QueryResult
 	}
-	assertSearchResults := func(sut at.PrometheusQuerier, opts *opts) {
+	assertSearchResults := func(sut at.APIQuerier, opts *opts) {
 		t.Helper()
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				return sut.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
+				return sut.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
 					Start: opts.start,
 					End:   opts.end,
 				}).Sort()
 			},
-			Want: &at.PrometheusAPIV1SeriesResponse{
+			Want: &at.APIV1SeriesResponse{
 				Status: "success",
 				Data:   opts.wantSeries,
 			},
@@ -96,13 +96,13 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query_range response",
 			Got: func() any {
-				return sut.PrometheusAPIV1QueryRange(t, `{__name__=~".*"}`, at.QueryOpts{
+				return sut.APIV1QueryRange(t, `{__name__=~".*"}`, at.QueryOpts{
 					Start: opts.start,
 					End:   opts.end,
 					Step:  "1d",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "matrix",
@@ -116,7 +116,7 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
 	// is searchable.
 	sut := start("with-per-day-index", false)
 	sample1 := []string{"metric1 111 1704067200000"} // 2024-01-01T00:00:00Z
-	sut.PrometheusAPIV1ImportPrometheus(t, sample1, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, sample1, at.QueryOpts{})
 	sut.ForceFlush(t)
 	assertSearchResults(sut, &opts{
 		start:      "2024-01-01T00:00:00Z",
@@ -132,10 +132,10 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
 
 	// Restart vmsingle with disabled per-day index, insert sample2, and confirm
 	// that both sample1 and sample2 is searchable.
-	tc.StopPrometheusWriteQuerier(sut)
+	tc.StopWriteQuerier(sut)
 	sut = start("without-per-day-index", true)
 	sample2 := []string{"metric2 222 1704067200000"} // 2024-01-01T00:00:00Z
-	sut.PrometheusAPIV1ImportPrometheus(t, sample2, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, sample2, at.QueryOpts{})
 	sut.ForceFlush(t)
 	assertSearchResults(sut, &opts{
 		start: "2024-01-01T00:00:00Z",
@@ -165,9 +165,9 @@ func testSearchWithDisabledPerDayIndex(tc *at.TestCase, start startSUTFunc) {
 	// - sample2 is not searchable when the time range is <= 40 days
 	// - sample2 becomes searchable when the time range is > 40 days
 	sample3 := []string{"metric1 333 1705708800000"} // 2024-01-20T00:00:00Z
-	sut.PrometheusAPIV1ImportPrometheus(t, sample3, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, sample3, at.QueryOpts{})
 	sut.ForceFlush(t)
-	tc.StopPrometheusWriteQuerier(sut)
+	tc.StopWriteQuerier(sut)
 	sut = start("with-per-day-index2", false)
 
 	// Time range is 1 day (Jan 1st) <= 40 days
@@ -295,14 +295,14 @@ func testClusterActiveTimeseriesMetric(t *testing.T, disablePerDayIndex bool) {
 	})
 }
 
-func testActiveTimeseriesMetric(tc *at.TestCase, sut at.PrometheusWriteQuerier, getActiveTimeseries func() int) {
+func testActiveTimeseriesMetric(tc *at.TestCase, sut at.WriteQuerier, getActiveTimeseries func() int) {
 	t := tc.T()
 	const numSamples = 1000
 	samples := make([]string, numSamples)
 	for i := range numSamples {
 		samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
 	}
-	sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
 	sut.ForceFlush(t)
 	tc.Assert(&at.AssertOptions{
 		Msg: `unexpected vm_cache_entries{type="storage/hour_metric_ids"} metric value`,

apptest/tests/prometheus_mock_storage.go

@@ -0,0 +1,62 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/remote"
+)
+
+// PrometheusMockStorage is a mock implementation of the Prometheus remote read storage interface.
+type PrometheusMockStorage struct {
+	query *prompb.Query
+	store []*prompb.TimeSeries
+	b     labels.ScratchBuilder
+}
+
+// NewPrometheusMockStorage creates a new PrometheusMockStorage with the provided series.
+func NewPrometheusMockStorage(series []*prompb.TimeSeries) *PrometheusMockStorage {
+	return &PrometheusMockStorage{store: series}
+}
+
+// Read implements the storage.Storage interface for reading time series data.
+func (ms *PrometheusMockStorage) Read(_ context.Context, query *prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
+	if ms.query != nil {
+		return nil, fmt.Errorf("expected only one call to remote client got: %v", query)
+	}
+	ms.query = query
+
+	matchers, err := remote.FromLabelMatchers(query.Matchers)
+	if err != nil {
+		return nil, err
+	}
+
+	q := &prompb.QueryResult{}
+	for _, s := range ms.store {
+		l := s.ToLabels(&ms.b, nil)
+		var notMatch bool
+
+		for _, m := range matchers {
+			if v := l.Get(m.Name); v != "" {
+				if !m.Matches(v) {
+					notMatch = true
+					break
+				}
+			}
+		}
+
+		if !notMatch {
+			q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples})
+		}
+	}
+
+	return remote.FromQueryResult(sortSeries, q), nil
+}
+
+// Reset resets the PrometheusMockStorage, clearing any stored query and series.
+func (ms *PrometheusMockStorage) Reset() {
+	ms.query = nil
+}

apptest/tests/query_resource_limit_test.go

@@ -13,7 +13,7 @@ import (
 func TestClusterMaxUniqueTimeseries(t *testing.T) {
 	os.RemoveAll(t.Name())
 
-	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+	cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
 
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
@@ -54,14 +54,14 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
 	// write data to two tenants
 	tenantIDs := []string{"0:0", "1:15"}
 	for _, tenantID := range tenantIDs {
-		vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
+		vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{Tenant: tenantID})
 		vmstorage.ForceFlush(t)
 	}
 
 	instantCT := "2022-05-10T08:05:00.000Z"
 
 	// success - `/api/v1/query`
-	want := apptest.NewPrometheusAPIV1QueryResponse(t,
+	want := apptest.NewAPIV1QueryResponse(t,
 		`{"data":
        {"result":[
           {"metric":{"__name__":"foo_bar1","instance":"a"},"value":[1652169900,"1"]}
@@ -69,7 +69,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
        }
      }`,
 	)
-	queryRes := vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar1", apptest.QueryOpts{
+	queryRes := vmselectSmallLimit.APIV1Query(t, "foo_bar1", apptest.QueryOpts{
 		Time: instantCT,
 	})
 	if diff := cmp.Diff(want, queryRes, cmpOpt); diff != "" {
@@ -78,7 +78,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
 
 	// success - multitenant `/api/v1/query`
 	// query is split into two queries for each tenant, so the final result can exceed the limit.
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+	want = apptest.NewAPIV1QueryResponse(t,
 		`{"data":
        {"result":[
           {"metric":{"__name__":"foo_bar1","instance":"a","vm_account_id":"0","vm_project_id":"0"},"value":[1652169900,"1"]},
@@ -87,7 +87,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
        }
      }`,
 	)
-	queryRes = vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar1", apptest.QueryOpts{
+	queryRes = vmselectSmallLimit.APIV1Query(t, "foo_bar1", apptest.QueryOpts{
 		Time:   instantCT,
 		Tenant: "multitenant",
 	})
@@ -96,7 +96,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
 	}
 
 	// fail - `/api/v1/query`, exceed vmselect `maxUniqueTimeseries`
-	queryRes = vmselectSmallLimit.PrometheusAPIV1Query(t, "foo_bar2", apptest.QueryOpts{
+	queryRes = vmselectSmallLimit.APIV1Query(t, "foo_bar2", apptest.QueryOpts{
 		Time: instantCT,
 	})
 	if queryRes.ErrorType != "422" {
@@ -104,7 +104,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
 	}
 
 	// fail - `/api/v1/query`, exceed vmstorage `maxUniqueTimeseries`
-	queryRes = vmselectNoLimit.PrometheusAPIV1Query(t, "foo_bar3", apptest.QueryOpts{
+	queryRes = vmselectNoLimit.APIV1Query(t, "foo_bar3", apptest.QueryOpts{
 		Time: instantCT,
 	})
 	if queryRes.ErrorType != "422" {
@@ -112,7 +112,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
 	}
 
 	// fail - `/api/v1/query`, vmselect `maxUniqueTimeseries` cannot exceed vmstorage `maxUniqueTimeseries`
-	queryRes = vmselectBigLimit.PrometheusAPIV1Query(t, "foo_bar3", apptest.QueryOpts{
+	queryRes = vmselectBigLimit.APIV1Query(t, "foo_bar3", apptest.QueryOpts{
 		Time: instantCT,
 	})
 	if queryRes.ErrorType != "422" {
@@ -123,7 +123,7 @@ func TestClusterMaxUniqueTimeseries(t *testing.T) {
 func TestClusterMaxSeries(t *testing.T) {
 	os.RemoveAll(t.Name())
 
-	cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
+	cmpSROpt := cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Status", "IsPartial")
 
 	tc := apptest.NewTestCase(t)
 	defer tc.Stop()
@@ -153,18 +153,18 @@ func TestClusterMaxSeries(t *testing.T) {
 	}
 
 	// write data
-	vminsert.PrometheusAPIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{})
+	vminsert.APIV1ImportPrometheus(t, commonSamples, apptest.QueryOpts{})
 	vmstorage.ForceFlush(t)
 
 	// success - `/api/v1/series`, vmselect `maxLabelsAPISeries` can exceed vmstorage `maxLabelsAPISeries``
-	wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
+	wantSR := apptest.NewAPIV1SeriesResponse(t,
 		`{"data": [
 			{"__name__":"foo_bar3","instance":"a"},
 			{"__name__":"foo_bar3","instance":"b"},
 			{"__name__":"foo_bar3","instance":"c"}
 			]
 		 }`)
-	seriesRes := vmselectBigLimit.PrometheusAPIV1Series(t, "foo_bar3", apptest.QueryOpts{
+	seriesRes := vmselectBigLimit.APIV1Series(t, "foo_bar3", apptest.QueryOpts{
 		Start: "2022-05-10T08:03:00.000Z",
 	})
 	if diff := cmp.Diff(wantSR.Sort(), seriesRes.Sort(), cmpSROpt); diff != "" {
@@ -172,7 +172,7 @@ func TestClusterMaxSeries(t *testing.T) {
 	}
 
 	// fail - `/api/v1/series`, exceed vmselect `maxSeries`
-	seriesRes1 := vmselectSmallLimit.PrometheusAPIV1Series(t, "foo_bar3", apptest.QueryOpts{
+	seriesRes1 := vmselectSmallLimit.APIV1Series(t, "foo_bar3", apptest.QueryOpts{
 		Start: "2022-05-10T08:03:00.000Z",
 	})
 	if seriesRes1.ErrorType != "422" {

apptest/tests/relabeling_test.go

@@ -57,7 +57,7 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
 		wantMetrics []map[string]string
 		wantSamples []*at.Sample
 	}
-	f := func(sut at.PrometheusQuerier, opts *opts) {
+	f := func(sut at.APIQuerier, opts *opts) {
 		t.Helper()
 		wantResult := []*at.QueryResult{}
 		for idx, wm := range opts.wantMetrics {
@@ -70,19 +70,19 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query response",
 			Got: func() any {
-				return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
+				return sut.APIV1Query(t, opts.query, at.QueryOpts{
 					Time: opts.qtime,
 					Step: opts.step,
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
+			Want: &at.APIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+				cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 			},
 		})
 	}
 
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`importprometheus_series{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 		`must_drop_series{label="foo"} 20 1707123456800`,        // 2024-02-05T08:57:36.800Z
 	}, at.QueryOpts{})
@@ -180,7 +180,7 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
 			},
 		},
 	}
-	sut.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
+	sut.APIV1Write(t, pbData, at.QueryOpts{})
 	sut.ForceFlush(t)
 	f(sut, &opts{
 		query: `{label="foo2"}[120ms]`,

apptest/tests/remote_read_server.go

@@ -1,4 +1,4 @@
-package remote_read_integration
+package tests
 
 import (
 	"context"
@@ -12,32 +12,29 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
-	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
-	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/storage/remote"
-	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/chunks"
-	"github.com/prometheus/prometheus/util/annotations"
 )
 
 const (
 	maxBytesInFrame = 1024 * 1024
 )
 
+// RemoteReadServer is a mock server that implements the Prometheus remote read protocol.
 type RemoteReadServer struct {
 	server  *httptest.Server
-	series  []*prompb.TimeSeries
-	storage *MockStorage
+	storage *PrometheusMockStorage
 }
 
 // NewRemoteReadServer creates a remote read server. It exposes a single endpoint and responds with the
 // passed series based on the request to the read endpoint. It returns a server which should be closed after
 // being used.
-func NewRemoteReadServer(t *testing.T) *RemoteReadServer {
+func NewRemoteReadServer(t *testing.T, series []*prompb.TimeSeries) *RemoteReadServer {
+	mockStorage := NewPrometheusMockStorage(series)
 	rrs := &RemoteReadServer{
-		series: make([]*prompb.TimeSeries, 0),
+		storage: mockStorage,
 	}
 	rrs.server = httptest.NewServer(rrs.getReadHandler(t))
 	return rrs
@@ -48,14 +45,11 @@ func (rrs *RemoteReadServer) Close() {
 	rrs.server.Close()
 }
 
-func (rrs *RemoteReadServer) URL() string {
+// HTTPAddr returns the HTTP address of the server.
+func (rrs *RemoteReadServer) HTTPAddr() string {
 	return rrs.server.URL
 }
 
-func (rrs *RemoteReadServer) SetRemoteReadSeries(series []*prompb.TimeSeries) {
-	rrs.series = append(rrs.series, series...)
-}
-
 func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if !validateReadHeaders(t, r) {
@@ -84,8 +78,8 @@ func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
 		for i, r := range req.Queries {
 			startTs := r.StartTimestampMs
 			endTs := r.EndTimestampMs
-			ts := make([]*prompb.TimeSeries, len(rrs.series))
-			for i, s := range rrs.series {
+			ts := make([]*prompb.TimeSeries, len(rrs.storage.store))
+			for i, s := range rrs.storage.store {
 				var samples []prompb.Sample
 				for _, sample := range s.Samples {
 					if sample.Timestamp >= startTs && sample.Timestamp < endTs {
@@ -119,18 +113,18 @@ func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
 	})
 }
 
-func NewRemoteReadStreamServer(t *testing.T) *RemoteReadServer {
+// NewRemoteReadStreamServer creates a remote read server that supports streaming responses.
+// passed series based on the request to the read endpoint. It returns a server which should be closed after
+// being used.
+func NewRemoteReadStreamServer(t *testing.T, series []*prompb.TimeSeries) *RemoteReadServer {
+	mockStorage := NewPrometheusMockStorage(series)
 	rrs := &RemoteReadServer{
-		series: make([]*prompb.TimeSeries, 0),
+		storage: mockStorage,
 	}
 	rrs.server = httptest.NewServer(rrs.getStreamReadHandler(t))
 	return rrs
 }
 
-func (rrs *RemoteReadServer) InitMockStorage(series []*prompb.TimeSeries) {
-	rrs.storage = NewMockStorage(series)
-}
-
 func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if !validateStreamReadHeaders(t, r) {
@@ -180,10 +174,10 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 			for ss.Next() {
 				series := ss.At()
 				iter = series.Iterator(iter)
-				labels := remote.MergeLabels(labelsToLabelsProto(series.Labels()), nil)
+				lbls := remote.MergeLabels(labelsToLabelsProto(series.Labels()), nil)
 
 				frameBytesLeft := maxBytesInFrame
-				for _, lb := range labels {
+				for _, lb := range lbls {
 					frameBytesLeft -= lb.Size()
 				}
 
@@ -213,7 +207,7 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 
 					resp := &prompb.ChunkedReadResponse{
 						ChunkedSeries: []*prompb.ChunkedSeries{
-							{Labels: labels, Chunks: chks},
+							{Labels: lbls, Chunks: chks},
 						},
 						QueryIndex: int64(idx),
 					}
@@ -280,6 +274,7 @@ func validateStreamReadHeaders(t *testing.T, r *http.Request) bool {
 	return true
 }
 
+// GenerateRemoteReadSeries generates a set of remote read series with the given parameters.
 func GenerateRemoteReadSeries(start, end, numOfSeries, numOfSamples int64) []*prompb.TimeSeries {
 	var ts []*prompb.TimeSeries
 	j := 0
@@ -322,141 +317,6 @@ func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64)
 	return samples
 }
 
-type MockStorage struct {
-	query *prompb.Query
-	store []*prompb.TimeSeries
-}
-
-func NewMockStorage(series []*prompb.TimeSeries) *MockStorage {
-	return &MockStorage{store: series}
-}
-
-func (ms *MockStorage) Read(_ context.Context, query *prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
-	if sortSeries {
-		return nil, fmt.Errorf("unexpected sortSeries=true")
-	}
-	if ms.query != nil {
-		return nil, fmt.Errorf("expected only one call to remote client got: %v", query)
-	}
-	ms.query = query
-
-	tss := make([]*prompb.TimeSeries, 0, len(ms.store))
-	for _, s := range ms.store {
-		var samples []prompb.Sample
-		for _, sample := range s.Samples {
-			if sample.Timestamp >= query.StartTimestampMs && sample.Timestamp < query.EndTimestampMs {
-				samples = append(samples, sample)
-			}
-		}
-		var series prompb.TimeSeries
-		if len(samples) > 0 {
-			series.Labels = s.Labels
-			series.Samples = samples
-		}
-
-		tss = append(tss, &series)
-	}
-	return &mockSeriesSet{
-		tss: tss,
-	}, nil
-}
-
-func (ms *MockStorage) Reset() {
-	ms.query = nil
-}
-
-type mockSeriesSet struct {
-	tss  []*prompb.TimeSeries
-	next int
-}
-
-func (ss *mockSeriesSet) Next() bool {
-	if ss.next >= len(ss.tss) {
-		return false
-	}
-	ss.next++
-	return true
-}
-
-func (ss *mockSeriesSet) At() storage.Series {
-	return &mockSeries{
-		s: ss.tss[ss.next-1],
-	}
-}
-
-func (ss *mockSeriesSet) Err() error {
-	return nil
-}
-
-func (ss *mockSeriesSet) Warnings() annotations.Annotations {
-	return nil
-}
-
-type mockSeries struct {
-	s *prompb.TimeSeries
-}
-
-func (s *mockSeries) Labels() labels.Labels {
-	a := make(labels.Labels, len(s.s.Labels))
-	for i, label := range s.s.Labels {
-		a[i] = labels.Label{
-			Name:  label.Name,
-			Value: label.Value,
-		}
-	}
-	return a
-}
-
-func (s *mockSeries) Iterator(chunkenc.Iterator) chunkenc.Iterator {
-	return &mockSamplesIterator{
-		samples: s.s.Samples,
-	}
-}
-
-type mockSamplesIterator struct {
-	samples []prompb.Sample
-	next    int
-}
-
-func (si *mockSamplesIterator) Next() chunkenc.ValueType {
-	if si.next >= len(si.samples) {
-		return chunkenc.ValNone
-	}
-	si.next++
-	return chunkenc.ValFloat
-}
-
-func (si *mockSamplesIterator) Seek(t int64) chunkenc.ValueType {
-	for i := range si.samples {
-		if si.samples[i].Timestamp >= t {
-			si.next = i + 1
-			return chunkenc.ValFloat
-		}
-	}
-	return chunkenc.ValNone
-}
-
-func (si *mockSamplesIterator) At() (int64, float64) {
-	s := si.samples[si.next-1]
-	return s.Timestamp, s.Value
-}
-
-func (si *mockSamplesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
-	panic("BUG: mustn't be called")
-}
-
-func (si *mockSamplesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
-	panic("BUG: mustn't be called")
-}
-
-func (si *mockSamplesIterator) AtT() int64 {
-	return si.samples[si.next-1].Timestamp
-}
-
-func (si *mockSamplesIterator) Err() error {
-	return nil
-}
-
 func labelsToLabelsProto(labels labels.Labels) []prompb.Label {
 	result := make([]prompb.Label, 0, len(labels))
 	for _, l := range labels {

apptest/tests/replication_test.go

@@ -7,10 +7,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
-	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
 type clusterWithReplication struct {
@@ -94,7 +95,7 @@ func TestClusterReplication_DataIsWrittenSeveralTimes(t *testing.T) {
 	for i := range numRecs {
 		recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
 	}
-	c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
+	c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
 	tc.ForceFlush(c.vmstorages...)
 
 	// Verify that each storage node has metrics and that total metric count across
@@ -151,7 +152,7 @@ func TestClusterReplication_Deduplication(t *testing.T) {
 			ts = ts.Add(1 * time.Minute)
 		}
 	}
-	c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
+	c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
 	tc.ForceFlush(c.vmstorages...)
 
 	// Check /api/v1/series response.
@@ -164,12 +165,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
+				return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-31T00:00:00Z",
 				}).Sort()
 			},
-			Want: &at.PrometheusAPIV1SeriesResponse{
+			Want: &at.APIV1SeriesResponse{
 				Status:    "success",
 				IsPartial: false,
 				Data: []map[string]string{
@@ -194,12 +195,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query response",
 			Got: func() any {
-				return app.PrometheusAPIV1Query(t, "metric_1", at.QueryOpts{
+				return app.APIV1Query(t, "metric_1", at.QueryOpts{
 					Time: "2024-01-01T00:05:00Z",
 					Step: "5m",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "vector",
@@ -236,12 +237,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query response",
 			Got: func() any {
-				return app.PrometheusAPIV1Query(t, "metric_1[5m]", at.QueryOpts{
+				return app.APIV1Query(t, "metric_1[5m]", at.QueryOpts{
 					Time: "2024-01-01T00:05:00Z",
 					Step: "5m",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "matrix",
@@ -273,13 +274,13 @@ func TestClusterReplication_Deduplication(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query_range response",
 			Got: func() any {
-				return app.PrometheusAPIV1QueryRange(t, "metric_1", at.QueryOpts{
+				return app.APIV1QueryRange(t, "metric_1", at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-01T00:10:00Z",
 					Step:  "5m",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "matrix",
@@ -309,12 +310,12 @@ func TestClusterReplication_Deduplication(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/export response",
 			Got: func() any {
-				return app.PrometheusAPIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
+				return app.APIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-01T00:03:00Z",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "matrix",
@@ -360,7 +361,7 @@ func TestClusterReplication_PartialResponse(t *testing.T) {
 	for i := range numRecs {
 		recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
 	}
-	c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
+	c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
 	tc.ForceFlush(c.vmstorages...)
 
 	// Verify partial vs full response.
@@ -370,14 +371,14 @@ func TestClusterReplication_PartialResponse(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
+				return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
 			},
-			Want: &at.PrometheusAPIV1SeriesResponse{
+			Want: &at.APIV1SeriesResponse{
 				Status:    "success",
 				IsPartial: wantPartial,
 			},
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
+				cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Data"),
 			},
 		})
 	}
@@ -437,7 +438,7 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
 
 	const numRecs = 1000
 	recs := make([]string, numRecs)
-	wantSeries := &at.PrometheusAPIV1SeriesResponse{
+	wantSeries := &at.APIV1SeriesResponse{
 		Status: "success",
 		Data:   make([]map[string]string, numRecs),
 	}
@@ -447,7 +448,7 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
 		wantSeries.Data[i] = map[string]string{"__name__": name}
 	}
 	wantSeries.Sort()
-	c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
+	c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
 	tc.ForceFlush(c.vmstorages...)
 
 	// Verify skipping slow replicas by counting the number of skipSlowReplicas
@@ -458,12 +459,12 @@ func TestClusterReplication_SkipSlowReplicas(t *testing.T) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
+				return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{}).Sort()
 			},
 			Want: wantSeries,
 		})
 
-		res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
+		res := app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
 		got := res.Trace.Contains("cancel request because -search.skipSlowReplicas is set and every group returned the needed number of responses according to replicationFactor")
 		if got != want {
 			t.Errorf("unexpected number of skipSlowReplicas messages in request trace: got %d, want %d (full trace:\n%v)", got, want, res.Trace)
@@ -654,7 +655,7 @@ func TestClusterGroupReplication(t *testing.T) {
 		numRecs    = numMetrics * numSamples
 	)
 	var recs []string
-	wantSeries := &at.PrometheusAPIV1SeriesResponse{
+	wantSeries := &at.APIV1SeriesResponse{
 		Status: "success",
 		Data:   make([]map[string]string, numMetrics),
 	}
@@ -668,7 +669,7 @@ func TestClusterGroupReplication(t *testing.T) {
 		}
 	}
 	wantSeries.Sort()
-	c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{})
+	c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{})
 	c.forceFlush(tc)
 
 	opts := &testGroupReplicationOpts{
@@ -694,7 +695,7 @@ type testGroupReplicationOpts struct {
 	numGroups  int
 	numNodes   int
 	numRecs    int
-	wantSeries *at.PrometheusAPIV1SeriesResponse
+	wantSeries *at.APIV1SeriesResponse
 }
 
 // testGroupDataIsWrittenSeveralTimes checks that multiple
@@ -747,7 +748,7 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
+				return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-31T00:00:00Z",
 				}).Sort()
@@ -768,12 +769,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query response",
 			Got: func() any {
-				return app.PrometheusAPIV1Query(t, "metric_1", at.QueryOpts{
+				return app.APIV1Query(t, "metric_1", at.QueryOpts{
 					Time: "2024-01-01T00:05:00Z",
 					Step: "5m",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "vector",
@@ -810,12 +811,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query response",
 			Got: func() any {
-				return app.PrometheusAPIV1Query(t, "metric_1[5m]", at.QueryOpts{
+				return app.APIV1Query(t, "metric_1[5m]", at.QueryOpts{
 					Time: "2024-01-01T00:05:00Z",
 					Step: "5m",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "matrix",
@@ -847,13 +848,13 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/query_range response",
 			Got: func() any {
-				return app.PrometheusAPIV1QueryRange(t, "metric_1", at.QueryOpts{
+				return app.APIV1QueryRange(t, "metric_1", at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-01T00:10:00Z",
 					Step:  "5m",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "matrix",
@@ -883,12 +884,12 @@ func testGroupDeduplication(tc *at.TestCase, opts *testGroupReplicationOpts) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/export response",
 			Got: func() any {
-				return app.PrometheusAPIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
+				return app.APIV1Export(t, `{__name__="metric_1"}`, at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-01T00:03:00Z",
 				})
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status: "success",
 				Data: &at.QueryData{
 					ResultType: "matrix",
@@ -929,7 +930,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
+				return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-31T00:00:00Z",
 				}).Sort()
@@ -937,7 +938,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
 			Want: opts.wantSeries,
 		})
 
-		res := app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
+		res := app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{Trace: "1"})
 		got := res.Trace.Contains("cancel request because -search.skipSlowReplicas is set and every group returned the needed number of responses according to replicationFactor")
 		if got < wantMin || got > wantMax {
 			t.Errorf("unexpected number of skipSlowReplicas messages in request trace: got %d, %d <= want <= %d (full trace:\n%v)", got, wantMin, wantMax, res.Trace)
@@ -973,7 +974,7 @@ func testGroupSkipSlowReplicas(tc *at.TestCase, opts *testGroupReplicationOpts)
 
 	// The data is replicated across N groups of M nodes. Replication factor is
 	// globalRF. There is no replication across the nodes within each group or
-	//it is unknown it there is one.
+	// it is unknown it there is one.
 	//
 	// Max number of nodes to skip is M*(globalRF-1). This corresponds to the
 	// case when N-globalRF+1 groups have received the response from all of
@@ -1020,17 +1021,17 @@ func testGroupPartialResponse(tc *at.TestCase, opts *testGroupReplicationOpts) {
 		tc.Assert(&at.AssertOptions{
 			Msg: "unexpected /api/v1/series response",
 			Got: func() any {
-				return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
+				return app.APIV1Series(t, `{__name__=~".*"}`, at.QueryOpts{
 					Start: "2024-01-01T00:00:00Z",
 					End:   "2024-01-31T00:00:00Z",
 				}).Sort()
 			},
-			Want: &at.PrometheusAPIV1SeriesResponse{
+			Want: &at.APIV1SeriesResponse{
 				Status:    "success",
 				IsPartial: wantPartial,
 			},
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
+				cmpopts.IgnoreFields(apptest.APIV1SeriesResponse{}, "Data"),
 			},
 		})
 	}
@@ -1128,10 +1129,10 @@ func TestClusterReplication_PartialResponseMultitenant(t *testing.T) {
 		recs[i] = fmt.Sprintf("metric_%d %d", i, rand.IntN(1000))
 	}
 
-	c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{
+	c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{
 		Tenant: "0",
 	})
-	c.vminsert.PrometheusAPIV1ImportPrometheus(t, recs, at.QueryOpts{
+	c.vminsert.APIV1ImportPrometheus(t, recs, at.QueryOpts{
 		Tenant: "1",
 	})
 	tc.ForceFlush(c.vmstorages...)
@@ -1144,14 +1145,14 @@ func TestClusterReplication_PartialResponseMultitenant(t *testing.T) {
 			Msg: "unexpected /api/v1/query response",
 			Got: func() any {
 				qo := at.QueryOpts{Tenant: "multitenant", Trace: "1"}
-				return app.PrometheusAPIV1Query(t, `{__name__=~"metric_.*"}`, qo)
+				return app.APIV1Query(t, `{__name__=~"metric_.*"}`, qo)
 			},
-			Want: &at.PrometheusAPIV1QueryResponse{
+			Want: &at.APIV1QueryResponse{
 				Status:    "success",
 				IsPartial: wantPartial,
 			},
 			CmpOpts: []cmp.Option{
-				cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Data"),
+				cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Data"),
 			},
 		})
 	}

apptest/tests/rollup_result_cache_test.go

@@ -0,0 +1,78 @@
+package tests
+
+import (
+	"os"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+func TestClusterRollupResultCache(t *testing.T) {
+	os.RemoveAll(t.Name())
+
+	cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+	vmstorage := tc.MustStartVmstorage("vmstorage", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage",
+		"-retentionPeriod=100y",
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
+		"-storageNode=" + vmstorage.VminsertAddr(),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmstorage.VmselectAddr(),
+		"-search.tenantCacheExpireDuration=0",
+	})
+
+	var tenantLabelsSamples = []string{
+		`foo_bar{vm_account_id="5"} 1.00 1652169720000`,                    // 2022-05-10T08:00:00Z'
+		`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
+	}
+
+	vminsert.APIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
+	vmstorage.ForceFlush(t)
+
+	want := apptest.NewAPIV1QueryResponse(t,
+		`{"data":
+	   {"result":[
+	        {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"values":[[1652169720,"1"],[1652169780,"1"]]},
+	        {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id":"15"},"values":[[1652169720,"3"],[1652169780,"3"]]}
+	               ]
+	   }
+	}`,
+	)
+
+	got := vmselect.APIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
+		Tenant:       "multitenant",
+		Start:        "2022-05-10T07:59:00.000Z",
+		End:          "2022-05-10T08:05:00.000Z",
+		Step:         "1m",
+		ExtraFilters: []string{`{vm_account_id="5",vm_project_id="15"}`, `{vm_account_id="5",vm_project_id="0"}`},
+	})
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	want = apptest.NewAPIV1QueryResponse(t,
+		`{"data":
+	   {"result":[]}
+	}`,
+	)
+
+	got = vmselect.APIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
+		Tenant:       "multitenant",
+		Start:        "2022-05-10T07:59:00.000Z",
+		End:          "2022-05-10T08:05:00.000Z",
+		Step:         "1m",
+		ExtraFilters: []string{`{vm_account_id="99",vm_project_id="99"}`},
+	})
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+}

apptest/tests/sharding_test.go

@@ -38,7 +38,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
 
 	const numMetrics = 1000
 	records := make([]string, numMetrics)
-	want := &apptest.PrometheusAPIV1SeriesResponse{
+	want := &apptest.APIV1SeriesResponse{
 		Status:    "success",
 		IsPartial: false,
 		Data:      make([]map[string]string, numMetrics),
@@ -49,7 +49,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
 		want.Data[i] = map[string]string{"__name__": name}
 	}
 	want.Sort()
-	vminsert.PrometheusAPIV1ImportPrometheus(t, records, apptest.QueryOpts{})
+	vminsert.APIV1ImportPrometheus(t, records, apptest.QueryOpts{})
 	vmstorage1.ForceFlush(t)
 	vmstorage2.ForceFlush(t)
 
@@ -74,7 +74,7 @@ func TestClusterVminsertShardsDataVmselectBuildsFullResultFromShards(t *testing.
 	tc.Assert(&apptest.AssertOptions{
 		Msg: "unexpected /api/v1/series response",
 		Got: func() any {
-			res := vmselect.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{})
+			res := vmselect.APIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{})
 			res.Sort()
 			return res
 		},

apptest/tests/snapshot_test.go

@@ -28,7 +28,7 @@ func TestSingleSnapshots_CreateListDelete(t *testing.T) {
 	for i := range numSamples {
 		samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
 	}
-	sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
 	sut.ForceFlush(t)
 
 	// Create several snapshots using VictoriaMetrics and Prometheus endpoints.
@@ -113,7 +113,7 @@ func TestClusterSnapshots_CreateListDelete(t *testing.T) {
 	for i := range numSamples {
 		samples[i] = fmt.Sprintf("metric_%03d %d", i, i)
 	}
-	sut.PrometheusAPIV1ImportPrometheus(t, samples, at.QueryOpts{})
+	sut.APIV1ImportPrometheus(t, samples, at.QueryOpts{})
 	sut.ForceFlush(t)
 
 	// Create several snapshots for both vmstorage replicas using

apptest/tests/special_query_regression_test.go

@@ -4,9 +4,10 @@ import (
 	"os"
 	"testing"
 
-	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
 // TestSingleSpecialQueryRegression is used to test queries that have experienced issues for specific data sets.
@@ -35,7 +36,7 @@ func TestClusterSpecialQueryRegression(t *testing.T) {
 	testSpecialQueryRegression(tc, sut)
 }
 
-func testSpecialQueryRegression(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testSpecialQueryRegression(tc *at.TestCase, sut at.WriteQuerier) {
 	// prometheus
 	testCaseSensitiveRegex(tc, sut)
 	testDuplicateLabel(tc, sut)
@@ -50,12 +51,12 @@ func testSpecialQueryRegression(tc *at.TestCase, sut at.PrometheusWriteQuerier)
 	testSubqueryAggregation(tc, sut)
 }
 
-func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testCaseSensitiveRegex(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// case-sensitive-regex
 	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/161
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`prometheus.sensitiveRegex{label="sensitiveRegex"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 		`prometheus.sensitiveRegex{label="SensitiveRegex"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 	}, at.QueryOpts{})
@@ -64,12 +65,12 @@ func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/export response",
 		Got: func() any {
-			return sut.PrometheusAPIV1Export(t, `{label=~'(?i)sensitiveregex'}`, at.QueryOpts{
+			return sut.APIV1Export(t, `{label=~'(?i)sensitiveregex'}`, at.QueryOpts{
 				Start: "2024-02-05T08:50:00.700Z",
 				End:   "2024-02-05T09:00:00.700Z",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -86,17 +87,17 @@ func testCaseSensitiveRegex(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 			},
 		},
 		CmpOpts: []cmp.Option{
-			cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+			cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 		},
 	})
 }
 
-func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testDuplicateLabel(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// duplicate_label
 	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/172
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`prometheus.duplicate_label{label="duplicate", label="duplicate"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 	}, at.QueryOpts{})
 	sut.ForceFlush(t)
@@ -104,12 +105,12 @@ func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/export response",
 		Got: func() any {
-			return sut.PrometheusAPIV1Export(t, `{__name__='prometheus.duplicate_label'}`, at.QueryOpts{
+			return sut.APIV1Export(t, `{__name__='prometheus.duplicate_label'}`, at.QueryOpts{
 				Start: "2024-02-05T08:50:00.700Z",
 				End:   "2024-02-05T09:00:00.700Z",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -122,17 +123,17 @@ func testDuplicateLabel(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 			},
 		},
 		CmpOpts: []cmp.Option{
-			cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+			cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 		},
 	})
 }
 
-func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testTooBigLookbehindWindow(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// too big look-behind window
 	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`prometheus.too_big_lookbehind{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 	}, at.QueryOpts{})
 	sut.ForceFlush(t)
@@ -140,12 +141,12 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query response",
 		Got: func() any {
-			return sut.PrometheusAPIV1Query(t, `prometheus.too_big_lookbehind{label="foo"}[100y]`, at.QueryOpts{
+			return sut.APIV1Query(t, `prometheus.too_big_lookbehind{label="foo"}[100y]`, at.QueryOpts{
 				Step: "5m",
 				Time: "2024-02-05T08:57:36.700Z",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -163,7 +164,7 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
 
 	// too big look-behind window - query range
 	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5553
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`prometheus.too_big_lookbehind_range{label="foo"} 13 1707123496700`, // 2024-02-05T08:58:16.700Z
 		`prometheus.too_big_lookbehind_range{label="foo"} 12 1707123466700`, // 2024-02-05T08:57:46.700Z
 		`prometheus.too_big_lookbehind_range{label="foo"} 11 1707123436700`, // 2024-02-05T08:57:16.700Z
@@ -174,13 +175,13 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query_range response",
 		Got: func() any {
-			return sut.PrometheusAPIV1QueryRange(t, `prometheus.too_big_lookbehind_range{label="foo"}`, at.QueryOpts{
+			return sut.APIV1QueryRange(t, `prometheus.too_big_lookbehind_range{label="foo"}`, at.QueryOpts{
 				Start: "2024-02-05T08:56:46.700Z",
 				End:   "2024-02-05T08:58:16.700Z",
 				Step:  "30s",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -200,12 +201,12 @@ func testTooBigLookbehindWindow(tc *at.TestCase, sut at.PrometheusWriteQuerier)
 	})
 }
 
-func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testMatchSeries(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// match_series
 	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/155
-	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+	sut.APIV1ImportPrometheus(t, []string{
 		`GenBearTemp{db="TenMinute",Park="1",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 		`GenBearTemp{db="TenMinute",Park="2",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
 		`GenBearTemp{db="TenMinute",Park="3",TurbineType="V112"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
@@ -216,12 +217,12 @@ func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/series response",
 		Got: func() any {
-			return sut.PrometheusAPIV1Series(t, `{__name__="GenBearTemp"}`, at.QueryOpts{
+			return sut.APIV1Series(t, `{__name__="GenBearTemp"}`, at.QueryOpts{
 				Start: "2024-02-04T08:57:36.700Z",
 				End:   "2024-02-05T08:57:36.700Z",
 			}).Sort()
 		},
-		Want: &at.PrometheusAPIV1SeriesResponse{
+		Want: &at.APIV1SeriesResponse{
 			Status:    "success",
 			IsPartial: false,
 			Data: []map[string]string{
@@ -234,7 +235,7 @@ func testMatchSeries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	})
 }
 
-func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testComparisonNotInfNotNan(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// comparison-not-inf-not-nan
@@ -258,13 +259,13 @@ func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier)
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query_range response",
 		Got: func() any {
-			return sut.PrometheusAPIV1QueryRange(t, `1/(not_nan_not_inf-1)!=inf!=nan`, at.QueryOpts{
+			return sut.APIV1QueryRange(t, `1/(not_nan_not_inf-1)!=inf!=nan`, at.QueryOpts{
 				Start: "2024-02-05T06:50:36.000Z",
 				End:   "2024-02-05T09:58:37.000Z",
 				Step:  "60",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -281,7 +282,7 @@ func testComparisonNotInfNotNan(tc *at.TestCase, sut at.PrometheusWriteQuerier)
 	})
 }
 
-func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testEmptyLabelMatch(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// empty-label-match
@@ -304,13 +305,13 @@ func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query_range response",
 		Got: func() any {
-			return sut.PrometheusAPIV1QueryRange(t, `empty_label_match{foo=~'bar|'}`, at.QueryOpts{
+			return sut.APIV1QueryRange(t, `empty_label_match{foo=~'bar|'}`, at.QueryOpts{
 				Start: "2024-02-05T08:55:36.000Z",
 				End:   "2024-02-05T08:57:36.000Z",
 				Step:  "60s",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -333,7 +334,7 @@ func testEmptyLabelMatch(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	})
 }
 
-func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testMaxLookbehind(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// max_lookback_set
@@ -357,14 +358,14 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query_range response",
 		Got: func() any {
-			return sut.PrometheusAPIV1QueryRange(t, `max_lookback_set{foo=~'bar|'}`, at.QueryOpts{
+			return sut.APIV1QueryRange(t, `max_lookback_set{foo=~'bar|'}`, at.QueryOpts{
 				Start:       "2024-02-05T08:55:06.000Z",
 				End:         "2024-02-05T08:57:37.000Z",
 				Step:        "10s",
 				MaxLookback: "1s",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -404,13 +405,13 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query_range response",
 		Got: func() any {
-			return sut.PrometheusAPIV1QueryRange(t, `max_lookback_unset{foo=~'bar|'}`, at.QueryOpts{
+			return sut.APIV1QueryRange(t, `max_lookback_unset{foo=~'bar|'}`, at.QueryOpts{
 				Start: "2024-02-05T08:55:06.000Z",
 				End:   "2024-02-05T08:57:37.000Z",
 				Step:  "10s",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -441,7 +442,7 @@ func testMaxLookbehind(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	})
 }
 
-func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testNonNanAsMissingData(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// not-nan-as-missing-data
@@ -465,13 +466,13 @@ func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query_range response",
 		Got: func() any {
-			return sut.PrometheusAPIV1QueryRange(t, `not_nan_as_missing_data>1`, at.QueryOpts{
+			return sut.APIV1QueryRange(t, `not_nan_as_missing_data>1`, at.QueryOpts{
 				Start: "2024-02-05T08:57:34.000Z",
 				End:   "2024-02-05T08:57:36.000Z",
 				Step:  "1s",
 			})
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "matrix",
@@ -496,7 +497,7 @@ func testNonNanAsMissingData(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	})
 }
 
-func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
+func testSubqueryAggregation(tc *at.TestCase, sut at.WriteQuerier) {
 	t := tc.T()
 
 	// subquery-aggregation
@@ -520,14 +521,14 @@ func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	tc.Assert(&at.AssertOptions{
 		Msg: "unexpected /api/v1/query response",
 		Got: func() any {
-			got := sut.PrometheusAPIV1Query(t, `min by (item) (min_over_time(forms_daily_count[10m:1m]))`, at.QueryOpts{
+			got := sut.APIV1Query(t, `min by (item) (min_over_time(forms_daily_count[10m:1m]))`, at.QueryOpts{
 				Time:          "2024-02-05T08:56:35.000Z",
 				LatencyOffset: "1ms",
 			})
 			got.Sort()
 			return got
 		},
-		Want: &at.PrometheusAPIV1QueryResponse{
+		Want: &at.APIV1QueryResponse{
 			Status: "success",
 			Data: &at.QueryData{
 				ResultType: "vector",
@@ -546,7 +547,7 @@ func testSubqueryAggregation(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
 	})
 }
 
-func getRowsInsertedTotal(t *testing.T, sut at.PrometheusWriteQuerier) int {
+func getRowsInsertedTotal(t *testing.T, sut at.WriteQuerier) int {
 	t.Helper()
 
 	selector := `vm_rows_inserted_total{type="graphite"}`

apptest/tests/vmagent_remotewrite_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"sync"
 	"testing"
 
@@ -11,6 +12,82 @@ import (
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
+// TestSingleVMAgentReloadConfigs verifies that vmagent reload new configurations on SIGHUP signal
+func TestSingleVMAgentReloadConfigs(t *testing.T) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingle := tc.MustStartDefaultVmsingle()
+
+	relabelingRules := `
+- replacement: value1
+  target_label: label1
+  `
+	relabelFilePath := fmt.Sprintf("%s/%s", t.TempDir(), "relabel_config.yaml")
+	if err := os.WriteFile(relabelFilePath, []byte(relabelingRules), os.ModePerm); err != nil {
+		t.Fatalf("cannot create file=%q: %s", relabelFilePath, err)
+	}
+
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
+		`-remoteWrite.forcePromProto=true`,
+		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
+		fmt.Sprintf(`-remoteWrite.url=http://%s/api/v1/write`, vmsingle.HTTPAddr()),
+		fmt.Sprintf(`-remoteWrite.urlRelabelConfig=%s`, relabelFilePath),
+	}, ``)
+
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	vmsingle.ForceFlush(t)
+
+	tc.Assert(&at.AssertOptions{
+		Msg: `unexpected metrics stored on vmagent remote write`,
+		Got: func() any {
+			return vmsingle.APIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
+				Start: "2022-05-10T00:00:00Z",
+				End:   "2022-05-10T23:59:59Z",
+			}).Sort()
+		},
+		Want: &at.APIV1SeriesResponse{
+			Status: "success",
+			Data:   []map[string]string{{"__name__": "foo_bar", "label1": "value1"}},
+		},
+	})
+
+	relabelingRules = `
+- replacement: value2
+  target_label: label1
+  `
+
+	if err := os.WriteFile(relabelFilePath, []byte(relabelingRules), os.ModePerm); err != nil {
+		t.Fatalf("cannot create file=%q: %s", relabelFilePath, err)
+	}
+
+	vmagent.ReloadRelabelConfigs(t)
+
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"bar_foo 1 1652169600001", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	vmsingle.ForceFlush(t)
+
+	tc.Assert(&at.AssertOptions{
+		Msg: `unexpected metrics stored on vmagent remote write`,
+		Got: func() any {
+			return vmsingle.APIV1Series(t, `{__name__="bar_foo"}`, at.QueryOpts{
+				Start: "2022-05-10T00:00:00Z",
+				End:   "2022-05-10T23:59:59Z",
+			}).Sort()
+		},
+		Want: &at.APIV1SeriesResponse{
+			Status: "success",
+			Data:   []map[string]string{{"__name__": "bar_foo", "label1": "value2"}},
+		},
+	})
+}
+
 // TestSingleVMAgentZstdRemoteWrite verifies that vmagent can successfully perform
 // a remote write to vmsingle using VM protocol (zstd).
 func TestSingleVMAgentZstdRemoteWrite(t *testing.T) {
@@ -45,12 +122,12 @@ func testSingleVMAgentRemoteWrite(t *testing.T, forcePromProto bool) {
 	tc.Assert(&at.AssertOptions{
 		Msg: `unexpected metrics stored on vmagent remote write`,
 		Got: func() any {
-			return vmsingle.PrometheusAPIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
+			return vmsingle.APIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
 				Start: "2022-05-10T00:00:00Z",
 				End:   "2022-05-10T23:59:59Z",
 			}).Sort()
 		},
-		Want: &at.PrometheusAPIV1SeriesResponse{
+		Want: &at.APIV1SeriesResponse{
 			Status: "success",
 			Data:   []map[string]string{{"__name__": "foo_bar"}},
 		},

apptest/tests/vmctl_native_migration_test.go

@@ -0,0 +1,149 @@
+package tests
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestSingleToSingleVmctlNativeProtocol(t *testing.T) {
+	os.RemoveAll(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleSrc := tc.MustStartVmsingle("vmsingle_src", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmsingle_src",
+		"-retentionPeriod=100y",
+	})
+	// we need a separate vmsingle for the destination to avoid conflicts
+	vmsingleDst := tc.MustStartVmsingle("vmsingle_dst", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmsingle_dst",
+		"-retentionPeriod=100y",
+	})
+
+	vmSrcAddr := fmt.Sprintf("http://%s/", vmsingleSrc.HTTPAddr())
+	vmDstAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+
+	flags := []string{
+		`vm-native`,
+		`--vm-native-src-addr=` + vmSrcAddr,
+		`--vm-native-dst-addr=` + vmDstAddr,
+		`--vm-native-filter-match={__name__=~".*"}`,
+		`--vm-native-filter-time-start=2025-05-30T16:39:00Z`,
+		`--disable-progress-bar=true`,
+	}
+
+	testVmctlNativeProtocol(tc, vmsingleSrc, vmsingleDst, flags)
+}
+
+func TestClusterTenantsToTenantsVmctlNativeProtocol(t *testing.T) {
+	os.RemoveAll(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	clusterSrc := tc.MustStartCluster(&apptest.ClusterOptions{
+		Vmstorage1Instance: "vmstorageSrc1",
+		Vmstorage2Instance: "vmstorageSrc2",
+		VminsertInstance:   "vminsertSrc",
+		VmselectInstance:   "vmselectSrc",
+	})
+	clusterDst := tc.MustStartCluster(&apptest.ClusterOptions{
+		Vmstorage1Instance: "vmstorageDst1",
+		Vmstorage2Instance: "vmstorageDst2",
+		VminsertInstance:   "vminsertDst",
+		VmselectInstance:   "vmselectDst",
+	})
+
+	vmSrcAddr := fmt.Sprintf("http://%s/", clusterSrc.Vmselect.HTTPAddr())
+	vmDstAddr := fmt.Sprintf("http://%s/", clusterDst.Vminsert.HTTPAddr())
+
+	flags := []string{
+		`vm-native`,
+		`--vm-native-src-addr=` + vmSrcAddr,
+		`--vm-native-dst-addr=` + vmDstAddr,
+		`--vm-native-filter-match={__name__=~".*"}`,
+		`--vm-native-filter-time-start=2025-05-30T16:39:00Z`,
+		`--disable-progress-bar=true`,
+		`--vm-intercluster`,
+	}
+
+	testVmctlNativeProtocol(tc, clusterSrc, clusterDst, flags)
+}
+
+func testVmctlNativeProtocol(tc *apptest.TestCase, srcSut apptest.WriteQuerier, dstSut apptest.WriteQuerier, vmctlFlags []string) {
+	t := tc.T()
+	t.Helper()
+
+	cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
+
+	// test for empty data request in the source
+	got := srcSut.APIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
+		Step: "5m",
+		Time: "2025-05-30T12:45:00Z",
+	})
+
+	want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// Prepare the source vmsingle with some data
+	// Insert some data.
+	const numSamples = 1000
+	const ingestTimestamp = " 1748623176000" // 2025-05-30T16:39:36Z
+
+	expectedQueryData := apptest.QueryData{
+		ResultType: "matrix",
+		Result:     make([]*apptest.QueryResult, 0, numSamples),
+	}
+	dataSet := make([]string, numSamples)
+	for i := range numSamples {
+		metricsName := fmt.Sprintf("metric_%03d", i)
+		metrics := map[string]string{"__name__": metricsName}
+		sample := &apptest.Sample{Value: float64(i), Timestamp: 1748623176000}
+		expectedQueryData.Result = append(expectedQueryData.Result, &apptest.QueryResult{
+			Metric:  metrics,
+			Samples: []*apptest.Sample{sample},
+		})
+
+		dataSet[i] = fmt.Sprintf("%s %d %s", metricsName, i, ingestTimestamp)
+	}
+
+	wantResponse := apptest.APIV1QueryResponse{
+		Status: "success",
+		Data:   &expectedQueryData,
+	}
+
+	wantResponse.Sort()
+
+	srcSut.APIV1ImportPrometheus(t, dataSet, apptest.QueryOpts{})
+	srcSut.ForceFlush(t)
+
+	tc.MustStartVmctl("vmctl", vmctlFlags)
+
+	dstSut.ForceFlush(t)
+
+	tc.Assert(&apptest.AssertOptions{
+		Retries: 300,
+		Msg:     `unexpected metrics stored on vmsingle via the native protocol`,
+		Got: func() any {
+			exported := dstSut.APIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
+				Start: "2025-05-30T16:39:36Z",
+				End:   "2025-05-30T16:39:37Z",
+			})
+			exported.Sort()
+			return exported.Data.Result
+		},
+		Want: wantResponse.Data.Result,
+		CmpOpts: []cmp.Option{
+			cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
+		},
+	})
+}

apptest/tests/vmctl_prometheus_migration_test.go

@@ -55,24 +55,24 @@ func TestClusterVmctlPrometheusProtocol(t *testing.T) {
 	testPrometheusProtocol(tc, cluster, vmctlFlags)
 }
 
-func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
+func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.WriteQuerier, vmctlFlags []string) {
 	t := tc.T()
 	t.Helper()
 
-	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+	cmpOpt := cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType")
 
 	// test for empty data request
-	got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
+	got := sut.APIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
 		Step: "5m",
 		Time: "2025-06-02T17:14:00Z",
 	})
 
-	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
+	want := apptest.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
 	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
-	_ = tc.MustStartVmctl("vmctl", vmctlFlags)
+	tc.MustStartVmctl("vmctl", vmctlFlags)
 
 	sut.ForceFlush(t)
 
@@ -88,7 +88,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
 		t.Fatalf("cannot read expected series response file: %s", err)
 	}
 
-	var wantResponse apptest.PrometheusAPIV1QueryResponse
+	var wantResponse apptest.APIV1QueryResponse
 	if err := json.Unmarshal(bytes, &wantResponse); err != nil {
 		t.Fatalf("cannot unmarshal expected series response file: %s", err)
 	}
@@ -99,7 +99,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
 		Retries: 300,
 		Msg:     `unexpected metrics stored on vmsingle via the prometheus protocol`,
 		Got: func() any {
-			expected := sut.PrometheusAPIV1Export(t, `{__name__="vm_log_messages_total", location=~"VictoriaMetrics/lib/ingestserver/opentsdb/server.go:(48|59)"}`, apptest.QueryOpts{
+			expected := sut.APIV1Export(t, `{__name__="vm_log_messages_total", location=~"VictoriaMetrics/lib/ingestserver/opentsdb/server.go:(48|59)"}`, apptest.QueryOpts{
 				Start: "2025-06-02T00:00:00Z",
 				End:   "2025-06-02T23:59:59Z",
 			})
@@ -108,7 +108,7 @@ func testPrometheusProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
 		},
 		Want: wantResponse.Data.Result,
 		CmpOpts: []cmp.Option{
-			cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
+			cmpopts.IgnoreFields(apptest.APIV1QueryResponse{}, "Status", "Data.ResultType"),
 		},
 	})
 }

apptest/tests/vmctl_remote_read_mogration_test.go

@@ -0,0 +1,157 @@
+package tests
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/prometheus/prometheus/prompb"
+
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+func TestSingleVmctlRemoteReadProtocol(t *testing.T) {
+	os.RemoveAll(t.Name())
+
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadProtocol(tc, vmsingleDst, newRemoteReadServer, vmctlFlags)
+}
+
+func TestSingleVmctlRemoteReadStreamProtocol(t *testing.T) {
+	os.RemoveAll(t.Name())
+
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingleDst := tc.MustStartDefaultVmsingle()
+	vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--remote-read-use-stream=true`,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadProtocol(tc, vmsingleDst, newRemoteReadStreamServer, vmctlFlags)
+}
+
+func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
+	os.RemoveAll(t.Name())
+
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	clusterDst := tc.MustStartDefaultCluster()
+
+	vmAddr := fmt.Sprintf("http://%s/", clusterDst.Vminsert.HTTPAddr())
+	vmctlFlags := []string{
+		`remote-read`,
+		`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
+		`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
+		`--remote-read-step-interval=minute`,
+		`--vm-addr=` + vmAddr,
+		`--vm-account-id=0`,
+		`--disable-progress-bar=true`,
+	}
+
+	testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
+}
+
+func testRemoteReadProtocol(tc *at.TestCase, sut at.WriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
+	t := tc.T()
+	t.Helper()
+
+	rrs := newRemoteReadServer(t)
+	defer rrs.Close()
+
+	expectedResult := transformSeriesToQueryResult(rrs.storage.store)
+
+	cmpOpt := cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType")
+	// test for empty data request
+	got := sut.APIV1Query(t, `{__name__=~".*"}`, at.QueryOpts{
+		Step: "5m",
+		Time: "2025-06-02T17:14:00Z",
+	})
+
+	want := at.NewAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	vmctlFlags = append(vmctlFlags, `--remote-read-src-addr=`+rrs.HTTPAddr())
+	tc.MustStartVmctl("vmctl", vmctlFlags)
+
+	sut.ForceFlush(t)
+
+	tc.Assert(&at.AssertOptions{
+		// For cluster version, we need to wait longer for the metrics to be stored
+		Retries: 300,
+		Msg:     `unexpected metrics stored on vmsingle via the prometheus protocol`,
+		Got: func() any {
+			expected := sut.APIV1Export(t, `{__name__=~".*"}`, at.QueryOpts{
+				Start: "2025-06-11T15:31:10Z",
+				End:   "2025-06-11T15:32:20Z",
+			})
+			expected.Sort()
+			return expected.Data.Result
+		},
+		Want: expectedResult,
+		CmpOpts: []cmp.Option{
+			cmpopts.IgnoreFields(at.APIV1QueryResponse{}, "Status", "Data.ResultType"),
+		},
+	})
+}
+
+func newRemoteReadServer(t *testing.T) *RemoteReadServer {
+	t.Helper()
+
+	series := GenerateRemoteReadSeries(1749655870, 1749655880, 10, 10)
+
+	rrServer := NewRemoteReadServer(t, series)
+
+	return rrServer
+}
+
+func newRemoteReadStreamServer(t *testing.T) *RemoteReadServer {
+	t.Helper()
+
+	series := GenerateRemoteReadSeries(1749655870, 1749655880, 10, 10)
+
+	rrServer := NewRemoteReadStreamServer(t, series)
+
+	return rrServer
+}
+
+func transformSeriesToQueryResult(series []*prompb.TimeSeries) []*at.QueryResult {
+	result := make([]*at.QueryResult, len(series))
+	for i, s := range series {
+		metric := make(map[string]string, len(s.Labels))
+		for _, label := range s.Labels {
+			metric[label.Name] = label.Value
+		}
+		samples := make([]*at.Sample, len(s.Samples))
+		for j, sample := range s.Samples {
+			samples[j] = &at.Sample{Timestamp: sample.Timestamp, Value: sample.Value}
+		}
+		result[i] = &at.QueryResult{Metric: metric, Samples: samples}
+	}
+	return result
+}

apptest/vlsingle.go

@@ -0,0 +1,136 @@
+package apptest
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
+)
+
+// Vlsingle holds the state of a vlsingle app and provides vlsingle-specific
+// functions.
+type Vlsingle struct {
+	*app
+	*ServesMetrics
+
+	storageDataPath string
+	httpListenAddr  string
+
+	forceFlushURL string
+}
+
+// StartVlsingle starts an instance of vlsingle with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr).
+func StartVlsingle(instance string, flags []string, cli *Client) (*Vlsingle, error) {
+	app, stderrExtracts, err := startApp(instance, "../../bin/victoria-logs", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-storageDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
+			"-httpListenAddr":  "127.0.0.1:0",
+		},
+		extractREs: []*regexp.Regexp{
+			logsStorageDataPathRE,
+			httpListenAddrRE,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vlsingle{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
+			cli:        cli,
+		},
+		storageDataPath: stderrExtracts[0],
+		httpListenAddr:  stderrExtracts[1],
+
+		forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
+	}, nil
+}
+
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
+func (app *Vlsingle) ForceFlush(t *testing.T) {
+	t.Helper()
+
+	_, statusCode := app.cli.Get(t, app.forceFlushURL)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// JSONLineWrite is a test helper function that inserts a
+// collection of records in json line format by sending a HTTP
+// POST request to /insert/jsonline vlsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api
+func (app *Vlsingle) JSONLineWrite(t *testing.T, records []string, opts QueryOptsLogs) {
+	t.Helper()
+
+	data := []byte(strings.Join(records, "\n"))
+
+	url := fmt.Sprintf("http://%s/insert/jsonline", app.httpListenAddr)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+
+	_, statusCode := app.cli.Post(t, url, "text/plain", data)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// NativeWrite is a test helper function that sends a collection of records
+// to /internal/insert API.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vlinsert/internalinsert/internalinsert.go
+func (app *Vlsingle) NativeWrite(t *testing.T, records []logstorage.InsertRow, opts QueryOpts) {
+	t.Helper()
+	var data []byte
+	for _, record := range records {
+		data = record.Marshal(data)
+	}
+	dstURL := fmt.Sprintf("http://%s/internal/insert", app.httpListenAddr)
+	uv := opts.asURLValues()
+	uv.Add("version", "v1")
+	dstURL += "?" + uv.Encode()
+
+	app.cli.Post(t, dstURL, "application/octet-stream", data)
+}
+
+// LogsQLQuery is a test helper function that performs
+// PromQL/MetricsQL range query by sending a HTTP POST request to
+// /select/logsql/query endpoint.
+//
+// See https://docs.victoriametrics.com/victorialogs/querying/#querying-logs
+func (app *Vlsingle) LogsQLQuery(t *testing.T, query string, opts QueryOptsLogs) *LogsQLQueryResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("query", query)
+
+	url := fmt.Sprintf("http://%s/select/logsql/query", app.httpListenAddr)
+	res, _ := app.cli.PostForm(t, url, values)
+	return NewLogsQLQueryResponse(t, res)
+}
+
+// HTTPAddr returns the address at which the vmstorage process is listening
+// for http connections.
+func (app *Vlsingle) HTTPAddr() string {
+	return app.httpListenAddr
+}
+
+// String returns the string representation of the vlsingle app state.
+func (app *Vlsingle) String() string {
+	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q}", []any{
+		app.app, app.storageDataPath, app.httpListenAddr}...)
+}

apptest/vmagent.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 )
@@ -101,6 +102,30 @@ func (app *Vmagent) RemoteWritePacketsDroppedTotal(t *testing.T) int {
 	return int(total)
 }
 
+// ReloadRelabelConfigs sends SIGHUP to trigger relabel config reload
+// and waits until vmagent_relabel_config_reloads_total increases.
+// Fails the test if no reload is detected within 3 seconds.
+func (app *Vmagent) ReloadRelabelConfigs(t *testing.T) {
+	prevTotal := app.GetMetric(t, "vmagent_relabel_config_reloads_total")
+
+	if err := app.process.Signal(syscall.SIGHUP); err != nil {
+		t.Fatalf("could not send SIGHUP signal to %s process: %v", app.instance, err)
+	}
+
+	var currTotal float64
+	for i := 0; i < 30; i++ {
+		currTotal = app.GetMetric(t, "vmagent_relabel_config_reloads_total")
+		if currTotal > prevTotal {
+			return
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	if currTotal <= prevTotal {
+		t.Fatalf("relabel configs were not reloaded after SIGHUP signal; previous total: %f, current total: %f", prevTotal, currTotal)
+	}
+}
+
 // sendBlocking sends the data to vmstorage by executing `send` function and
 // waits until the data is actually sent.
 //

apptest/vmbackup.go

@@ -0,0 +1,13 @@
+package apptest
+
+// StartVmbackup starts an instance of vmbackup with the given flags and waits
+// until it exits.
+func StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string) error {
+	flags := []string{
+		"-storageDataPath=" + storageDataPath,
+		"-snapshot.createURL=" + snapshotCreateURL,
+		"-dst=" + dst,
+	}
+	_, _, err := startApp(instance, "../../bin/vmbackup", flags, &appOptions{wait: true})
+	return err
+}

apptest/vmctl.go

@@ -1,18 +1,7 @@
 package apptest
 
-// Vmctl holds the state of a vmctl app and provides vmctl-specific functions
-type Vmctl struct {
-	*app
-}
-
 // StartVmctl starts an instance of vmctl cli with the given flags
-func StartVmctl(instance string, flags []string) (*Vmctl, error) {
-	app, _, err := startApp(instance, "../../bin/vmctl", flags, &appOptions{wait: true})
-	if err != nil {
-		return nil, err
-	}
-
-	return &Vmctl{
-		app: app,
-	}, nil
+func StartVmctl(instance string, flags []string) error {
+	_, _, err := startApp(instance, "../../bin/vmctl", flags, &appOptions{wait: true})
+	return err
 }

apptest/vminsert.go

@@ -127,12 +127,12 @@ func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts)
 	app.cli.Write(t, app.graphiteListenAddr, records)
 }
 
-// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
+// APIV1ImportCSV is a test helper function that inserts a collection
 // of records in CSV format for the given tenant by sending an HTTP POST
 // request to prometheus/api/v1/import/csv vminsert endpoint.
 //
 // See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
+func (app *Vminsert) APIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/csv", app.httpListenAddr, opts.getTenant())
@@ -150,12 +150,12 @@ func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, op
 	})
 }
 
-// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
+// APIV1ImportNative is a test helper function that inserts a collection
 // of records in Native format for the given tenant by sending an HTTP POST
 // request to prometheus/api/v1/import/native vminsert endpoint.
 //
 // See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
+func (app *Vminsert) APIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/native", app.httpListenAddr, opts.getTenant())
@@ -195,10 +195,10 @@ func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOp
 	})
 }
 
-// PrometheusAPIV1Write is a test helper function that inserts a
+// APIV1Write is a test helper function that inserts a
 // collection of records in Prometheus remote-write format by sending a HTTP
 // POST request to /prometheus/api/v1/write vminsert endpoint.
-func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts) {
+func (app *Vminsert) APIV1Write(t *testing.T, records []pb.TimeSeries, opts QueryOpts) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.getTenant())
@@ -212,13 +212,13 @@ func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries,
 	})
 }
 
-// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// APIV1ImportPrometheus is a test helper function that inserts a
 // collection of records in Prometheus text exposition format for the given
 // tenant by sending a HTTP POST request to
 // /prometheus/api/v1/import/prometheus vminsert endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
-func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
+func (app *Vminsert) APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.getTenant())

apptest/vmrestore.go

@@ -0,0 +1,12 @@
+package apptest
+
+// StartVmrestore starts an instance of vmrestore with the given flags and waits
+// until it exits.
+func StartVmrestore(instance, src, storageDataPath string) error {
+	flags := []string{
+		"-src=" + src,
+		"-storageDataPath=" + storageDataPath,
+	}
+	_, _, err := startApp(instance, "../../bin/vmrestore", flags, &appOptions{wait: true})
+	return err
+}

apptest/vmselect.go

@@ -55,12 +55,18 @@ func (app *Vmselect) ClusternativeListenAddr() string {
 	return app.clusternativeListenAddr
 }
 
-// PrometheusAPIV1Export is a test helper function that performs the export of
+// HTTPAddr returns the address at which the vmselect process is
+// listening for incoming HTTP requests.
+func (app *Vmselect) HTTPAddr() string {
+	return app.httpListenAddr
+}
+
+// APIV1Export is a test helper function that performs the export of
 // raw samples in JSON line format by sending a HTTP POST request to
 // /prometheus/api/v1/export vmselect endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
-func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+func (app *Vmselect) APIV1Export(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse {
 	t.Helper()
 
 	exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export", app.httpListenAddr, opts.getTenant())
@@ -68,15 +74,15 @@ func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts Quer
 	values.Add("match[]", query)
 	values.Add("format", "promapi")
 	res, _ := app.cli.PostForm(t, exportURL, values)
-	return NewPrometheusAPIV1QueryResponse(t, res)
+	return NewAPIV1QueryResponse(t, res)
 }
 
-// PrometheusAPIV1ExportNative is a test helper function that performs the export of
+// APIV1ExportNative is a test helper function that performs the export of
 // raw samples in native binary format by sending an HTTP POST request to
 // /prometheus/api/v1/export/native vmselect endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
-func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
+func (app *Vmselect) APIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
 	t.Helper()
 
 	exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export/native", app.httpListenAddr, opts.getTenant())
@@ -87,12 +93,12 @@ func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opt
 	return []byte(res)
 }
 
-// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
+// APIV1Query is a test helper function that performs PromQL/MetricsQL
 // instant query by sending a HTTP POST request to /prometheus/api/v1/query
 // vmselect endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
-func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+func (app *Vmselect) APIV1Query(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse {
 	t.Helper()
 
 	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query", app.httpListenAddr, opts.getTenant())
@@ -100,15 +106,15 @@ func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts Query
 	values.Add("query", query)
 
 	res, _ := app.cli.PostForm(t, queryURL, values)
-	return NewPrometheusAPIV1QueryResponse(t, res)
+	return NewAPIV1QueryResponse(t, res)
 }
 
-// PrometheusAPIV1QueryRange is a test helper function that performs
+// APIV1QueryRange is a test helper function that performs
 // PromQL/MetricsQL range query by sending a HTTP POST request to
 // /prometheus/api/v1/query_range vmselect endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
-func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+func (app *Vmselect) APIV1QueryRange(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse {
 	t.Helper()
 
 	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query_range", app.httpListenAddr, opts.getTenant())
@@ -116,14 +122,14 @@ func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts
 	values.Add("query", query)
 
 	res, _ := app.cli.PostForm(t, queryURL, values)
-	return NewPrometheusAPIV1QueryResponse(t, res)
+	return NewAPIV1QueryResponse(t, res)
 }
 
-// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
+// APIV1Series sends a query to a /prometheus/api/v1/series endpoint
 // and returns the list of time series that match the query.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
+func (app *Vmselect) APIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *APIV1SeriesResponse {
 	t.Helper()
 
 	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series", app.httpListenAddr, opts.getTenant())
@@ -131,7 +137,7 @@ func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts
 	values.Add("match[]", matchQuery)
 
 	res, _ := app.cli.PostForm(t, seriesURL, values)
-	return NewPrometheusAPIV1SeriesResponse(t, res)
+	return NewAPIV1SeriesResponse(t, res)
 }
 
 // DeleteSeries sends a query to a /prometheus/api/v1/admin/tsdb/delete_series
@@ -221,6 +227,24 @@ func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date strin
 	return status
 }
 
+// APIV1AdminTenants sends a query to a /admin/tenants endpoint
+func (app *Vmselect) APIV1AdminTenants(t *testing.T) *AdminTenantsResponse {
+	t.Helper()
+
+	tenantsURL := fmt.Sprintf("http://%s/admin/tenants", app.httpListenAddr)
+	res, statusCode := app.cli.Get(t, tenantsURL)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var tenants *AdminTenantsResponse
+	if err := json.Unmarshal([]byte(res), tenants); err != nil {
+		t.Fatalf("could not unmarshal tenants response data:\n%s\n err: %v", res, err)
+	}
+
+	return tenants
+}
+
 // String returns the string representation of the vmselect app state.
 func (app *Vmselect) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)

apptest/vmsingle.go

@@ -10,8 +10,9 @@ import (
 	"testing"
 	"time"
 
-	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/golang/snappy"
+
+	pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
@@ -142,12 +143,12 @@ func (app *Vmsingle) GraphiteWrite(t *testing.T, records []string, _ QueryOpts)
 	app.cli.Write(t, app.graphiteWriteAddr, records)
 }
 
-// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
+// APIV1ImportCSV is a test helper function that inserts a collection
 // of records in CSV format for the given tenant by sending an HTTP POST
 // request to /api/v1/import/csv vmsingle endpoint.
 //
 // See https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-import-csv-data
-func (app *Vmsingle) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
+func (app *Vmsingle) APIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/api/v1/import/csv", app.httpListenAddr)
@@ -163,12 +164,12 @@ func (app *Vmsingle) PrometheusAPIV1ImportCSV(t *testing.T, records []string, op
 	}
 }
 
-// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
+// APIV1ImportNative is a test helper function that inserts a collection
 // of records in native format for the given tenant by sending an HTTP POST
 // request to /api/v1/import/native vmsingle endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-native-format
-func (app *Vmsingle) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
+func (app *Vmsingle) APIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/api/v1/import/native", app.httpListenAddr)
@@ -205,10 +206,10 @@ func (app *Vmsingle) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOp
 	}
 }
 
-// PrometheusAPIV1Write is a test helper function that inserts a
+// APIV1Write is a test helper function that inserts a
 // collection of records in Prometheus remote-write format by sending a HTTP
 // POST request to /prometheus/api/v1/write vmsingle endpoint.
-func (app *Vmsingle) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries, _ QueryOpts) {
+func (app *Vmsingle) APIV1Write(t *testing.T, records []pb.TimeSeries, _ QueryOpts) {
 	t.Helper()
 
 	wr := pb.WriteRequest{Timeseries: records}
@@ -219,12 +220,12 @@ func (app *Vmsingle) PrometheusAPIV1Write(t *testing.T, records []pb.TimeSeries,
 	}
 }
 
-// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// APIV1ImportPrometheus is a test helper function that inserts a
 // collection of records in Prometheus text exposition format by sending a HTTP
 // POST request to /prometheus/api/v1/import/prometheus vmsingle endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
-func (app *Vmsingle) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
+func (app *Vmsingle) APIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
 	t.Helper()
 
 	// add extra label
@@ -242,27 +243,27 @@ func (app *Vmsingle) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str
 	}
 }
 
-// PrometheusAPIV1Export is a test helper function that performs the export of
+// APIV1Export is a test helper function that performs the export of
 // raw samples in JSON line format by sending a HTTP POST request to
 // /prometheus/api/v1/export vmsingle endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
-func (app *Vmsingle) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+func (app *Vmsingle) APIV1Export(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse {
 	t.Helper()
 	values := opts.asURLValues()
 	values.Add("match[]", query)
 	values.Add("format", "promapi")
 
 	res, _ := app.cli.PostForm(t, app.prometheusAPIV1ExportURL, values)
-	return NewPrometheusAPIV1QueryResponse(t, res)
+	return NewAPIV1QueryResponse(t, res)
 }
 
-// PrometheusAPIV1ExportNative is a test helper function that performs the export of
+// APIV1ExportNative is a test helper function that performs the export of
 // raw samples in native binary format by sending an HTTP POST request to
 // /prometheus/api/v1/export/native vmselect endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
-func (app *Vmsingle) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
+func (app *Vmsingle) APIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
 	t.Helper()
 
 	t.Helper()
@@ -274,47 +275,47 @@ func (app *Vmsingle) PrometheusAPIV1ExportNative(t *testing.T, query string, opt
 	return []byte(res)
 }
 
-// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
+// APIV1Query is a test helper function that performs PromQL/MetricsQL
 // instant query by sending a HTTP POST request to /prometheus/api/v1/query
 // vmsingle endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
-func (app *Vmsingle) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+func (app *Vmsingle) APIV1Query(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse {
 	t.Helper()
 
 	values := opts.asURLValues()
 	values.Add("query", query)
 	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryURL, values)
-	return NewPrometheusAPIV1QueryResponse(t, res)
+	return NewAPIV1QueryResponse(t, res)
 }
 
-// PrometheusAPIV1QueryRange is a test helper function that performs
+// APIV1QueryRange is a test helper function that performs
 // PromQL/MetricsQL range query by sending a HTTP POST request to
 // /prometheus/api/v1/query_range vmsingle endpoint.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
-func (app *Vmsingle) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+func (app *Vmsingle) APIV1QueryRange(t *testing.T, query string, opts QueryOpts) *APIV1QueryResponse {
 	t.Helper()
 
 	values := opts.asURLValues()
 	values.Add("query", query)
 
 	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryRangeURL, values)
-	return NewPrometheusAPIV1QueryResponse(t, res)
+	return NewAPIV1QueryResponse(t, res)
 }
 
-// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
+// APIV1Series sends a query to a /prometheus/api/v1/series endpoint
 // and returns the list of time series that match the query.
 //
 // See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmsingle) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
+func (app *Vmsingle) APIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *APIV1SeriesResponse {
 	t.Helper()
 
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
 
 	res, _ := app.cli.PostForm(t, app.prometheusAPIV1SeriesURL, values)
-	return NewPrometheusAPIV1SeriesResponse(t, res)
+	return NewAPIV1SeriesResponse(t, res)
 }
 
 // APIV1StatusMetricNamesStats sends a query to a /api/v1/status/metric_names_stats endpoint
@@ -363,8 +364,7 @@ func (app *Vmsingle) APIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts Qu
 func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	t.Helper()
 
-	queryURL := fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, "", nil)
+	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), "", nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}
@@ -377,6 +377,11 @@ func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	return &res
 }
 
+// SnapshotCreateURL returns the URL for creating snapshots.
+func (app *Vmsingle) SnapshotCreateURL() string {
+	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
+}
+
 // APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
 // /api/v1/admin/tsdb/snapshot endpoint.
 //

apptest/vmstorage.go

@@ -99,8 +99,7 @@ func (app *Vmstorage) ForceMerge(t *testing.T) {
 func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	t.Helper()
 
-	queryURL := fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, "", nil)
+	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), "", nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}
@@ -113,6 +112,11 @@ func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	return &res
 }
 
+// SnapshotCreateURL returns the URL for creating snapshots.
+func (app *Vmstorage) SnapshotCreateURL() string {
+	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
+}
+
 // SnapshotList lists existing database snapshots by sending a query to the
 // /snapshot/list endpoint.
 //