docs/vmanomaly: release v1.29.4 (#10963)

Update anomaly detection docs to release `v1.29.4`

.github/workflows/build.yml

@@ -57,6 +57,8 @@ jobs:
             arch: amd64
           - os: openbsd
             arch: amd64
+          - os: netbsd
+            arch: amd64
           - os: windows
             arch: amd64
     steps:

app/vmagent/remotewrite/client.go

@@ -59,6 +59,8 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	basicAuthUsernameFile = flagutil.NewArrayString("remoteWrite.basicAuth.usernameFile", "Optional path to basic auth username to use for the corresponding -remoteWrite.url. "+
+		"The file is re-read every second")
 	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
 		"The file is re-read every second")
@@ -223,12 +225,14 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 		hdrs = strings.Split(headersValue, "^^")
 	}
 	username := basicAuthUsername.GetOptionalArg(argIdx)
+	usernameFile := basicAuthUsernameFile.GetOptionalArg(argIdx)
 	password := basicAuthPassword.GetOptionalArg(argIdx)
 	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
 	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
+	if username != "" || usernameFile != "" || password != "" || passwordFile != "" {
 		basicAuthCfg = &promauth.BasicAuthConfig{
 			Username:     username,
+			UsernameFile: usernameFile,
 			Password:     promauth.NewSecret(password),
 			PasswordFile: passwordFile,
 		}
@@ -466,7 +470,7 @@ again:
 				goto again
 			}
 
-			logger.Warnf("failed to repack zstd block (%s bytes) to snappy: %s; The block will be rejected. "+
+			logger.Warnf("failed to repack zstd block (%d bytes) to snappy: %s; The block will be rejected. "+
 				"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
 				zstdBlockLen, err)
 		}

app/vmagent/remotewrite/remotewrite.go

@@ -699,7 +699,7 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
 			}
 			tmpLabels.Labels = hashLabels
 		}
-		h := getLabelsHash(hashLabels)
+		h := getLabelsHashForShard(hashLabels)
 
 		// Get the rwctxIdx through consistent hashing and then map it to the index in shards.
 		// The rwctxIdx is not always equal to the shardIdx, for example, when some rwctx are not available.
@@ -790,11 +790,28 @@ var (
 	dailySeriesLimitRowsDropped  = metrics.NewCounter(`vmagent_daily_series_limit_rows_dropped_total`)
 )
 
+// getLabelsHashForShard is a separate function from getLabelsHash because
+// it omits the '=' separator between label name and value for backward compatibility.
+// Changing it would re-shard all series across remoteWrite targets.
+func getLabelsHashForShard(labels []prompb.Label) uint64 {
+	bb := labelsHashBufPool.Get()
+	b := bb.B[:0]
+	for _, label := range labels {
+		b = append(b, label.Name...)
+		b = append(b, label.Value...)
+	}
+	h := xxhash.Sum64(b)
+	bb.B = b
+	labelsHashBufPool.Put(bb)
+	return h
+}
+
 func getLabelsHash(labels []prompb.Label) uint64 {
 	bb := labelsHashBufPool.Get()
 	b := bb.B[:0]
 	for _, label := range labels {
 		b = append(b, label.Name...)
+		b = append(b, '=')
 		b = append(b, label.Value...)
 	}
 	h := xxhash.Sum64(b)

app/vmagent/remotewrite/remotewrite_test.go

@@ -25,7 +25,7 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		t.Helper()
 
 		// Distribute itemsCount hashes returned by getLabelsHash() across bucketsCount buckets.
-		itemsCount := 1_000 * bucketsCount
+		itemsCount := 10_000 * bucketsCount
 		m := make([]int, bucketsCount)
 		var labels []prompb.Label
 		for i := range itemsCount {
@@ -44,10 +44,12 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		}
 
 		// Verify that the distribution is even
-		expectedItemsPerBucket := itemsCount / bucketsCount
+		expectedItemsPerBucket := float64(itemsCount / bucketsCount)
+		allowedDeviation := math.Round(float64(expectedItemsPerBucket) * 0.04)
 		for _, n := range m {
-			if math.Abs(1-float64(n)/float64(expectedItemsPerBucket)) > 0.04 {
-				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want around %d", bucketsCount, n, expectedItemsPerBucket)
+			if math.Abs(expectedItemsPerBucket-float64(n)) > allowedDeviation {
+				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want in range [%.0f, %.0f]",
+					bucketsCount, n, expectedItemsPerBucket-allowedDeviation, expectedItemsPerBucket+allowedDeviation)
 			}
 		}
 	}

app/vmalert/datasource/client_test.go

@@ -772,7 +772,7 @@ func TestHeaders(t *testing.T) {
 
 	// basic auth
 	f(func() *Client {
-		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
+		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "", "bar", ""))
 		if err != nil {
 			t.Fatalf("Error get auth config: %s", err)
 		}
@@ -817,7 +817,7 @@ func TestHeaders(t *testing.T) {
 
 	// custom header overrides basic auth
 	f(func() *Client {
-		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
+		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "", "bar", ""))
 		if err != nil {
 			t.Fatalf("Error get auth config: %s", err)
 		}

app/vmalert/datasource/init.go

@@ -27,6 +27,7 @@ var (
 		"Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthUsernameFile = flag.String("datasource.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
 	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
 
@@ -105,7 +106,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -datasource.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/notifier/alertmanager.go

@@ -191,7 +191,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 
 	aCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
+		vmalertutil.WithBasicAuth(ba.Username, ba.UsernameFile, ba.Password.String(), ba.PasswordFile),
 		vmalertutil.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
 		vmalertutil.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
 		vmalertutil.WithHeaders(strings.Join(authCfg.Headers, "^^")),

app/vmalert/notifier/init.go

@@ -36,6 +36,7 @@ var (
 		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
 		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
+	basicAuthUsernameFile = flagutil.NewArrayString("notifier.basicAuth.usernameFile", "Optional path to basic auth username file for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
 
@@ -213,6 +214,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			},
 			BasicAuth: &promauth.BasicAuthConfig{
 				Username:     basicAuthUsername.GetOptionalArg(i),
+				UsernameFile: basicAuthUsernameFile.GetOptionalArg(i),
 				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
 				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
 			},

app/vmalert/remoteread/init.go

@@ -28,6 +28,7 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
+	basicAuthUsernameFile = flag.String("remoteRead.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
 	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
 
@@ -80,7 +81,7 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/remotewrite/client.go

@@ -11,6 +11,7 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/cespare/xxhash/v2"
@@ -18,6 +19,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -57,6 +60,11 @@ type Client struct {
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
+
+	// Whether to encode the write request with VictoriaMetrics remote write protocol.
+	// It is set to true by default, and will be switched to false if the client
+	// receives specific errors indicating that the remote storage doesn't support VictoriaMetrics remote write protocol.
+	isVMRemoteWrite atomic.Bool
 }
 
 // Config is config for remote write client.
@@ -116,6 +124,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		doneCh:        make(chan struct{}),
 		input:         make(chan prompb.TimeSeries, cfg.MaxQueueSize),
 	}
+	c.isVMRemoteWrite.Store(true)
 
 	for i := 0; i < cc; i++ {
 		c.wg.Go(func() {
@@ -265,8 +274,16 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	defer wr.Reset()
 	defer bufferFlushDuration.UpdateDuration(time.Now())
 
-	data := wr.MarshalProtobuf(nil)
-	b := snappy.Encode(nil, data)
+	bb := writeRequestBufPool.Get()
+	bb.B = wr.MarshalProtobuf(bb.B[:0])
+	zb := compressBufPool.Get()
+	defer compressBufPool.Put(zb)
+	if c.isVMRemoteWrite.Load() {
+		zb.B = zstd.CompressLevel(zb.B[:0], bb.B, 0)
+	} else {
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+	}
+	writeRequestBufPool.Put(bb)
 
 	maxRetryInterval := *retryMaxTime
 	bt := timeutil.NewBackoffTimer(*retryMinInterval, maxRetryInterval)
@@ -278,17 +295,17 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	attempts := 0
 L:
 	for {
-		err := c.send(ctx, b)
+		err := c.send(ctx, zb.B)
 		if err != nil && (errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err)) {
 			// Something in the middle between client and destination might be closing
 			// the connection. So we do a one more attempt in hope request will succeed.
-			err = c.send(ctx, b)
+			err = c.send(ctx, zb.B)
 		}
 		if err == nil {
 			sentRows.Add(len(wr.Timeseries))
-			sentBytes.Add(len(b))
+			sentBytes.Add(len(zb.B))
 			flushedRows.Update(float64(len(wr.Timeseries)))
-			flushedBytes.Update(float64(len(b)))
+			flushedBytes.Update(float64(len(zb.B)))
 			return
 		}
 
@@ -340,12 +357,16 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
 
-	// RFC standard compliant headers
-	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("User-Agent", "vmalert")
 	req.Header.Set("Content-Type", "application/x-protobuf")
 
-	// Prometheus compliant headers
-	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if encoding.IsZstd(data) {
+		req.Header.Set("Content-Encoding", "zstd")
+		req.Header.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
+	} else {
+		req.Header.Set("Content-Encoding", "snappy")
+		req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	}
 
 	if c.authCfg != nil {
 		err = c.authCfg.SetHeaders(req, true)
@@ -374,6 +395,29 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 		// respond with HTTP 2xx status code when write is successful.
 		return nil
 	case 4:
+		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
+		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
+		// - Real-world implementations of v1 use both 400 and 415 status codes.
+		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
+		if resp.StatusCode == http.StatusUnsupportedMediaType || resp.StatusCode == http.StatusBadRequest {
+			if encoding.IsZstd(data) {
+				logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
+					"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", req.URL.Redacted())
+				zstdBlockLen := len(data)
+				data, err = repackBlockFromZstdToSnappy(data)
+				if err == nil {
+					logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
+						"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", req.URL.Redacted())
+					c.isVMRemoteWrite.Store(false)
+					return c.send(ctx, data)
+				}
+
+				logger.Warnf("failed to repack zstd block (%d bytes) to snappy: %s; The block will be rejected. "+
+					"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
+					zstdBlockLen, err)
+			}
+		}
+
 		if resp.StatusCode != http.StatusTooManyRequests {
 			// MUST NOT retry write requests on HTTP 4xx responses other than 429
 			return &nonRetriableError{
@@ -394,3 +438,19 @@ type nonRetriableError struct {
 func (e *nonRetriableError) Error() string {
 	return e.err.Error()
 }
+
+var (
+	writeRequestBufPool bytesutil.ByteBufferPool
+	compressBufPool     bytesutil.ByteBufferPool
+)
+
+// repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
+func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
+	plainBlock := make([]byte, 0, len(zstdBlock)*2)
+	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
+	if err != nil {
+		return nil, err
+	}
+
+	return snappy.Encode(nil, plainBlock), nil
+}

app/vmalert/remotewrite/client_test.go

@@ -12,8 +12,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang/snappy"
-
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
@@ -159,8 +158,8 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	h := r.Header.Get("Content-Encoding")
-	if h != "snappy" {
-		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not snappy (%q)", h))
+	if h != "zstd" {
+		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not zstd (%q)", h))
 	}
 
 	h = r.Header.Get("Content-Type")
@@ -168,9 +167,9 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("header read error: Content-Type is not x-protobuf (%q)", h))
 	}
 
-	h = r.Header.Get("X-Prometheus-Remote-Write-Version")
-	if h != "0.1.0" {
-		rw.err(w, fmt.Errorf("header read error: X-Prometheus-Remote-Write-Version is not 0.1.0 (%q)", h))
+	h = r.Header.Get("X-VictoriaMetrics-Remote-Write-Version")
+	if h != "1" {
+		rw.err(w, fmt.Errorf("header read error: X-VictoriaMetrics-Remote-Write-Version is not 1 (%q)", h))
 	}
 
 	data, err := io.ReadAll(r.Body)
@@ -180,7 +179,7 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 	}
 	defer func() { _ = r.Body.Close() }()
 
-	b, err := snappy.Decode(nil, data)
+	b, err := zstd.Decompress(nil, data)
 	if err != nil {
 		rw.err(w, fmt.Errorf("decode err: %w", err))
 		return

app/vmalert/remotewrite/debug_client.go

@@ -9,8 +9,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/golang/snappy"
-
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -64,19 +63,17 @@ func (c *DebugClient) Close() error {
 }
 
 func (c *DebugClient) send(data []byte) error {
-	b := snappy.Encode(nil, data)
+	b := zstd.CompressLevel(nil, data, 0)
 	r := bytes.NewReader(b)
 	req, err := http.NewRequest(http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
 
-	// RFC standard compliant headers
-	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("Content-Encoding", "zstd")
 	req.Header.Set("Content-Type", "application/x-protobuf")
 
-	// Prometheus compliant headers
-	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	req.Header.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
 
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")

app/vmalert/remotewrite/init.go

@@ -13,8 +13,8 @@ import (
 )
 
 var (
-	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
-		"and recording rules results in form of timeseries. "+
+	addr = flag.String("remoteWrite.url", "", "Optional URL to persist alerts state and recording rules results in form of timeseries. "+
+		"It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. "+
 		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
@@ -26,6 +26,7 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthUsernameFile = flag.String("remoteWrite.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
 	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
 
@@ -83,7 +84,7 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/vmalertutil/auth.go

@@ -20,11 +20,12 @@ func AuthConfig(filterOptions ...AuthConfigOptions) (*promauth.Config, error) {
 }
 
 // WithBasicAuth returns AuthConfigOptions and initialized promauth.BasicAuthConfig based on given params
-func WithBasicAuth(username, password, passwordFile string) AuthConfigOptions {
+func WithBasicAuth(username, usernameFile, password, passwordFile string) AuthConfigOptions {
 	return func(config *promauth.HTTPClientConfig) {
-		if username != "" || password != "" || passwordFile != "" {
+		if username != "" || usernameFile != "" || password != "" || passwordFile != "" {
 			config.BasicAuth = &promauth.BasicAuthConfig{
 				Username:     username,
+				UsernameFile: usernameFile,
 				Password:     promauth.NewSecret(password),
 				PasswordFile: passwordFile,
 			}

app/vmui/packages/vmui/src/pages/TopQueries/TopQueryPanel/TopQueryPanel.tsx

@@ -1,4 +1,4 @@
-import { FC, useState } from "react";
+import { FC, ReactNode, useState } from "react";
 import { TopQuery } from "../../../types";
 import JsonView from "../../../components/Views/JsonView/JsonView";
 import { CodeIcon, TableIcon } from "../../../components/Main/Icons";
@@ -8,10 +8,18 @@ import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 
+export interface TopQueryColumn {
+  title?: string;
+  tooltip?: string;
+  key: keyof TopQuery;
+  sortBy?: keyof TopQuery;
+  format?: (row: TopQuery) => ReactNode;
+}
+
 export interface TopQueryPanelProps {
   rows: TopQuery[],
   title?: string,
-  columns: {title?: string, key: (keyof TopQuery), sortBy?: (keyof TopQuery)}[],
+  columns: TopQueryColumn[],
   defaultOrderBy?: keyof TopQuery,
 }
 const tabs = ["table", "JSON"].map((t, i) => ({

app/vmui/packages/vmui/src/pages/TopQueries/TopQueryTable/TopQueryTable.tsx

@@ -3,7 +3,7 @@ import { TopQuery } from "../../../types";
 import { getComparator, stableSort } from "../../../components/Table/helpers";
 import { TopQueryPanelProps } from "../TopQueryPanel/TopQueryPanel";
 import classNames from "classnames";
-import { ArrowDropDownIcon, CopyIcon, PlayCircleOutlineIcon } from "../../../components/Main/Icons";
+import { ArrowDropDownIcon, CopyIcon, InfoOutlinedIcon, PlayCircleOutlineIcon } from "../../../components/Main/Icons";
 import Button from "../../../components/Main/Button/Button";
 import Tooltip from "../../../components/Main/Tooltip/Tooltip";
 import { Link } from "react-router-dom";
@@ -35,26 +35,40 @@ const TopQueryTable:FC<TopQueryPanelProps> = ({ rows, columns, defaultOrderBy })
     <table className="vm-table">
       <thead className="vm-table-header">
         <tr className="vm-table__row vm-table__row_header">
-          {columns.map((col) => (
-            <th
-              className="vm-table-cell vm-table-cell_header vm-table-cell_sort"
-              onClick={createSortHandler(col.sortBy || col.key)}
-              key={col.key}
-            >
-              <div className="vm-table-cell__content">
-                {col.title || col.key}
-                <div
-                  className={classNames({
-                    "vm-table__sort-icon": true,
-                    "vm-table__sort-icon_active": orderBy === col.key,
-                    "vm-table__sort-icon_desc": orderDir === "desc" && orderBy === col.key
-                  })}
-                >
-                  <ArrowDropDownIcon/>
+          {columns.map((col) => {
+            const sortKey = col.sortBy || col.key;
+
+            return (
+              <th
+                className="vm-table-cell vm-table-cell_header vm-table-cell_sort"
+                onClick={createSortHandler(sortKey)}
+                key={col.key}
+              >
+                <div className="vm-table-cell__content">
+                  {col.title || col.key}
+                  {col.tooltip && (
+                    <Tooltip
+                      placement="top-center"
+                      title={col.tooltip}
+                    >
+                      <span className="vm-top-queries-table__info-icon">
+                        <InfoOutlinedIcon/>
+                      </span>
+                    </Tooltip>
+                  )}
+                  <div
+                    className={classNames({
+                      "vm-table__sort-icon": true,
+                      "vm-table__sort-icon_active": orderBy === sortKey,
+                      "vm-table__sort-icon_desc": orderDir === "desc" && orderBy === sortKey
+                    })}
+                  >
+                    <ArrowDropDownIcon/>
+                  </div>
                 </div>
-              </div>
-            </th>
-          ))}
+              </th>
+            );
+          })}
           <th className="vm-table-cell vm-table-cell_header"/> {/* empty cell for actions */}
         </tr>
       </thead>
@@ -69,7 +83,7 @@ const TopQueryTable:FC<TopQueryPanelProps> = ({ rows, columns, defaultOrderBy })
                 className="vm-table-cell"
                 key={col.key}
               >
-                {row[col.key] || "-"}
+                {col.format?.(row) ?? row[col.key] ?? "-"}
               </td>
             ))}
             <td className="vm-table-cell vm-table-cell_no-padding">

app/vmui/packages/vmui/src/pages/TopQueries/hooks/useFetchTopQueries.ts

@@ -34,7 +34,7 @@ const processResponse = (data: TopQueriesData) => {
     target.forEach(t => {
       const timeRange = getDurationFromMilliseconds(t.timeRangeSeconds*1000);
       t.url = getQueryUrl(t, timeRange);
-      t.timeRange = timeRange;
+      t.timeRange = timeRange || "instant";
     });
   });
 

app/vmui/packages/vmui/src/pages/TopQueries/hooks/useTopQueriesColumns.ts

@@ -0,0 +1,77 @@
+import { useMemo } from "react";
+import { TopQueryColumn } from "../TopQueryPanel/TopQueryPanel";
+import { humanizeSeconds } from "../../../utils/time";
+import { formatBytes } from "../../../utils/bytes";
+
+type UseTopQueriesColumns = {
+  maxLifetime: string;
+};
+
+export const useTopQueriesColumns = ({ maxLifetime }: UseTopQueriesColumns) => {
+  return useMemo(() => {
+    const queryCol: TopQueryColumn = {
+      key: "query"
+    };
+
+    const timeRangeCol: TopQueryColumn = {
+      key: "timeRange",
+      sortBy: "timeRangeSeconds",
+      title: "range",
+      tooltip: "The time range between start and end of the query request. 'instant' means the query was executed at a single point in time without a time range"
+    };
+
+    const countCol: TopQueryColumn = {
+      key: "count",
+      tooltip: `The number of times the query was executed over the last ${maxLifetime}`,
+    };
+
+    const topBySumDuration: TopQueryColumn[] = [
+      queryCol,
+      {
+        key: "sumDurationSeconds",
+        title: "duration",
+        tooltip: `Cumulative time spent executing the query across all its invocations over the last ${maxLifetime}`,
+        format: (row) => humanizeSeconds(row.sumDurationSeconds)
+      },
+      timeRangeCol,
+      countCol,
+    ];
+
+    const topByAvgDuration: TopQueryColumn[] = [
+      queryCol,
+      {
+        key: "avgDurationSeconds",
+        title: "duration",
+        tooltip: `Average time spent executing the query over the last ${maxLifetime}`,
+        format: (row) => humanizeSeconds(row.avgDurationSeconds)
+      },
+      timeRangeCol,
+      countCol,
+    ];
+
+    const topByCount: TopQueryColumn[] = [
+      queryCol,
+      timeRangeCol,
+      countCol,
+    ];
+
+    const topByAvgMemoryUsage: TopQueryColumn[] = [
+      queryCol,
+      {
+        key: "avgMemoryBytes",
+        title: "memory",
+        tooltip: `Average memory used during query execution over the last ${maxLifetime}`,
+        format: (row) => formatBytes(row.avgMemoryBytes)
+      },
+      timeRangeCol,
+      countCol,
+    ];
+
+    return {
+      topBySumDuration,
+      topByAvgDuration,
+      topByCount,
+      topByAvgMemoryUsage,
+    };
+  }, [maxLifetime]);
+};

app/vmui/packages/vmui/src/pages/TopQueries/index.tsx

@@ -15,6 +15,7 @@ import "./style.scss";
 import useDeviceDetect from "../../hooks/useDeviceDetect";
 import classNames from "classnames";
 import useStateSearchParams from "../../hooks/useStateSearchParams";
+import { useTopQueriesColumns } from "./hooks/useTopQueriesColumns";
 
 const exampleDuration = "30ms, 15s, 3d4h, 1y2w";
 
@@ -23,6 +24,7 @@ const TopQueries: FC = () => {
 
   const [topN, setTopN] = useStateSearchParams(10, "topN");
   const [maxLifetime, setMaxLifetime] = useStateSearchParams("10m", "maxLifetime");
+  const columns = useTopQueriesColumns({ maxLifetime });
 
   const { data, error, loading, fetch } = useFetchTopQueries({ topN, maxLifetime });
 
@@ -145,52 +147,33 @@ const TopQueries: FC = () => {
 
       {error && <Alert variant="error">{error}</Alert>}
 
-      {data && (<>
+      {data && (
         <div className="vm-top-queries-panels">
           <TopQueryPanel
+            title="Queries with most summary time to execute"
             rows={data.topBySumDuration}
-            title={"Queries with most summary time to execute"}
-            columns={[
-              { key: "query" },
-              { key: "sumDurationSeconds", title: "sum duration, sec" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
-            defaultOrderBy={"sumDurationSeconds"}
+            columns={columns.topBySumDuration}
+            defaultOrderBy="sumDurationSeconds"
           />
           <TopQueryPanel
+            title="Most heavy queries"
             rows={data.topByAvgDuration}
-            title={"Most heavy queries"}
-            columns={[
-              { key: "query" },
-              { key: "avgDurationSeconds", title: "avg duration, sec" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
-            defaultOrderBy={"avgDurationSeconds"}
+            columns={columns.topByAvgDuration}
+            defaultOrderBy="avgDurationSeconds"
           />
           <TopQueryPanel
+            title="Most frequently executed queries"
             rows={data.topByCount}
-            title={"Most frequently executed queries"}
-            columns={[
-              { key: "query" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
+            columns={columns.topByCount}
           />
           <TopQueryPanel
+            title="Queries with most memory to execute"
             rows={data.topByAvgMemoryUsage}
-            title={"Queries with most memory to execute"}
-            columns={[
-              { key: "query" },
-              { key: "avgMemoryBytes", title: "avg memory usage, bytes" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
-            defaultOrderBy={"avgMemoryBytes"}
+            columns={columns.topByAvgMemoryUsage}
+            defaultOrderBy="avgMemoryBytes"
           />
         </div>
-      </>)}
+      )}
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/TopQueries/style.scss

@@ -1,5 +1,19 @@
 @use "src/styles/variables" as *;
 
+.vm-top-queries-table {
+  &__info-icon {
+    display: inline-flex;
+    align-items: center;
+    color: $color-text-secondary;
+    margin-left: 4px;
+
+    svg {
+      width: 14px;
+      height: 14px;
+    }
+  }
+}
+
 .vm-top-queries {
   display: grid;
   align-items: flex-start;

app/vmui/packages/vmui/src/utils/bytes.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { formatBytes } from "./bytes";
+
+describe("formatBytes", () => {
+  it("returns null for invalid values", () => {
+    expect(formatBytes(-1)).toBeNull();
+    expect(formatBytes(Number.NaN)).toBeNull();
+    expect(formatBytes(Number.POSITIVE_INFINITY)).toBeNull();
+    expect(formatBytes(Number.NEGATIVE_INFINITY)).toBeNull();
+  });
+
+  it("formats zero bytes", () => {
+    expect(formatBytes(0)).toBe("0 B");
+  });
+
+  it("formats bytes", () => {
+    expect(formatBytes(0.5)).toBe("0.5 B");
+    expect(formatBytes(1)).toBe("1 B");
+    expect(formatBytes(512)).toBe("512 B");
+    expect(formatBytes(1023)).toBe("1023 B");
+  });
+
+  it("formats kilobytes", () => {
+    expect(formatBytes(1024)).toBe("1 KB");
+    expect(formatBytes(1536)).toBe("1.5 KB");
+  });
+
+  it("formats megabytes", () => {
+    expect(formatBytes(1024 ** 2)).toBe("1 MB");
+    expect(formatBytes(2.5 * 1024 ** 2)).toBe("2.5 MB");
+  });
+
+  it("formats gigabytes, terabytes and petabytes", () => {
+    expect(formatBytes(1024 ** 3)).toBe("1 GB");
+    expect(formatBytes(1024 ** 4)).toBe("1 TB");
+    expect(formatBytes(1024 ** 5)).toBe("1 PB");
+  });
+
+  it("caps values above PB to PB unit", () => {
+    expect(formatBytes(1024 ** 6)).toBe("1024 PB");
+  });
+
+  it("rounds to two decimals", () => {
+    expect(formatBytes(1234)).toBe("1.21 KB");
+    expect(formatBytes(1234567)).toBe("1.18 MB");
+  });
+});

app/vmui/packages/vmui/src/utils/bytes.ts

@@ -0,0 +1,14 @@
+const LOG_1024 = Math.log(1024);
+const UNITS = ["B", "KB", "MB", "GB", "TB", "PB"] as const;
+
+export const formatBytes = (bytes: number): string | null => {
+  if (!Number.isFinite(bytes) || bytes < 0) return null;
+  if (bytes === 0) return "0 B";
+
+  const unitIndex = Math.min(
+    Math.max(Math.floor(Math.log(bytes) / LOG_1024), 0),
+    UNITS.length - 1
+  );
+
+  return `${parseFloat((bytes / 1024 ** unitIndex).toFixed(2))} ${UNITS[unitIndex]}`;
+};

apptest/client.go

@@ -2,6 +2,7 @@ package apptest
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -13,6 +14,10 @@ import (
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
+	"github.com/golang/snappy"
 )
 
 // Client is used for interacting with the apps over the network.
@@ -147,27 +152,34 @@ func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
 	return string(b)
 }
 
-// ServesMetrics is used to retrieve the app's metrics.
+// metricsClient is used to retrieve the app's metrics.
 //
 // This type is expected to be embedded by the apps that serve metrics.
-type ServesMetrics struct {
-	metricsURL string
-	cli        *Client
+type metricsClient struct {
+	metricsCli *Client
+	url        string
+}
+
+func newMetricsClient(cli *Client, addr string) *metricsClient {
+	return &metricsClient{
+		metricsCli: cli,
+		url:        fmt.Sprintf("http://%s/metrics", addr),
+	}
 }
 
 // GetIntMetric retrieves the value of a metric served by an app at /metrics URL.
 // The value is then converted to int.
-func (app *ServesMetrics) GetIntMetric(t *testing.T, metricName string) int {
+func (c *metricsClient) GetIntMetric(t *testing.T, metricName string) int {
 	t.Helper()
 
-	return int(app.GetMetric(t, metricName))
+	return int(c.GetMetric(t, metricName))
 }
 
 // GetMetric retrieves the value of a metric served by an app at /metrics URL.
-func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
+func (c *metricsClient) GetMetric(t *testing.T, metricName string) float64 {
 	t.Helper()
 
-	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
+	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -188,12 +200,12 @@ func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
 
 // GetMetricsByPrefix retrieves the values of all metrics that start with given
 // prefix.
-func (app *ServesMetrics) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
+func (c *metricsClient) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
 	t.Helper()
 
 	values := []float64{}
 
-	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
+	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -217,12 +229,12 @@ func (app *ServesMetrics) GetMetricsByPrefix(t *testing.T, prefix string) []floa
 	return values
 }
 
-func (app *ServesMetrics) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []float64 {
+func (c *metricsClient) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []float64 {
 	t.Helper()
 
 	values := []float64{}
 
-	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
+	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -245,3 +257,756 @@ func (app *ServesMetrics) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []
 	}
 	return values
 }
+
+// rpcRowsSentTotal retrieves the values of all vminsert
+// `vm_rpc_rows_sent_total` metrics (there will be one for each vmstorage) and
+// returns their integer sum.
+func (c *metricsClient) rpcRowsSentTotal(t *testing.T) int {
+	total := 0.0
+	for _, v := range c.GetMetricsByPrefix(t, "vm_rpc_rows_sent_total") {
+		total += v
+	}
+	return int(total)
+}
+
+type vmselectClient struct {
+	vmselectCli              *Client
+	url                      func(op, path string, opts QueryOpts) string
+	metricNamesStatsResetURL string
+	tenantsURL               string
+}
+
+// PrometheusAPIV1Export is a test helper function that performs the export of
+// raw samples in JSON line format by sending a request to
+// /prometheus/api/v1/export endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
+func (c *vmselectClient) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/export", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1ExportNative is a test helper function that performs the export of
+// raw samples in native binary format by sending a request to
+// /prometheus/api/v1/export/native endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
+func (c *vmselectClient) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/export/native", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return []byte(res)
+}
+
+// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
+// instant query by sending a request to /prometheus/api/v1/query endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
+func (c *vmselectClient) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/query", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1QueryRange is a test helper function that performs
+// PromQL/MetricsQL range query by sending a request to
+// /prometheus/api/v1/query_range endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
+func (c *vmselectClient) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/query_range", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1Series retrieves list of time series that match the query by
+// sending a request to /prometheus/api/v1/series endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (c *vmselectClient) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/series", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesResponse(t, res)
+}
+
+// PrometheusAPIV1SeriesCount retrieves the total number of time series by
+// sending a request to /prometheus/api/v1/series/count endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (c *vmselectClient) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/series/count", opts)
+	values := opts.asURLValues()
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesCountResponse(t, res)
+}
+
+// PrometheusAPIV1Labels retrieves the label names for time series that match a
+// query by sending a request to /prometheus/api/v1/labels endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
+func (c *vmselectClient) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/labels", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1LabelsResponse(t, res)
+}
+
+// PrometheusAPIV1LabelValues retrieves the labels values for the metrics that
+// match the query by sending a request to /prometheus/api/v1/label/.../values
+// endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
+func (c *vmselectClient) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
+	t.Helper()
+	path := fmt.Sprintf("prometheus/api/v1/label/%s/values", labelName)
+	url := c.url("select", path, opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1LabelValuesResponse(t, res)
+}
+
+// PrometheusAPIV1Metadata retrieves metadata for the given metric by sending a
+// request to /prometheus/api/v1/metadata endpoint.
+func (c *vmselectClient) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/metadata", opts)
+	values := opts.asURLValues()
+	values.Add("metric", metric)
+	values.Add("limit", strconv.Itoa(limit))
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1Metadata(t, res)
+}
+
+// PrometheusAPIV1AdminTSDBDeleteSeries deletes the series that match the query
+// by sending a request to /prometheus/api/v1/admin/tsdb/delete_series.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
+func (c *vmselectClient) PrometheusAPIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("delete", "prometheus/api/v1/admin/tsdb/delete_series", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// PrometheusAPIV1StatusMetricNamesStats sends a query to
+// /prometheus/api/v1/status/metric_names_stats endpoint and returns the metric
+// usage stats response for given params.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (c *vmselectClient) PrometheusAPIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/status/metric_names_stats", opts)
+	values := opts.asURLValues()
+	values.Add("limit", limit)
+	values.Add("le", le)
+	values.Add("match_pattern", matchPattern)
+	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+	var resp MetricNamesStatsResponse
+	if err := json.Unmarshal([]byte(res), &resp); err != nil {
+		t.Fatalf("could not unmarshal metric names stats response data:\n%s\n err: %v", res, err)
+	}
+	return resp
+}
+
+// PrometheusAPIV1StatusTSDB retrieves the TSDB status for the time series that
+// match the query on the given date by sending a request to
+// /prometheus/api/v1/status/tsdb endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
+func (c *vmselectClient) PrometheusAPIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/status/tsdb", opts)
+	values := opts.asURLValues()
+	addNonEmpty := func(name, value string) {
+		if len(value) == 0 {
+			return
+		}
+		values.Add(name, value)
+	}
+	addNonEmpty("match[]", matchQuery)
+	addNonEmpty("topN", topN)
+	addNonEmpty("date", date)
+	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var status TSDBStatusResponse
+	if err := json.Unmarshal([]byte(res), &status); err != nil {
+		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
+	}
+	status.Sort()
+	return status
+}
+
+// GraphiteMetricsIndex retrieves the list of all metrics by sending a request
+// to /graphite/metrics/index.json endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+func (c *vmselectClient) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/metrics/index.json", opts)
+	res, statusCode := c.vmselectCli.Get(t, url, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var index GraphiteMetricsIndexResponse
+	if err := json.Unmarshal([]byte(res), &index); err != nil {
+		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
+	}
+	return index
+}
+
+// GraphiteMetricsFind finds metrics under a given path by sending a request
+// to /metrics/find endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+// and https://graphite.readthedocs.io/en/latest/metrics_api.html#metrics-find
+func (c *vmselectClient) GraphiteMetricsFind(t *testing.T, query string, opts QueryOpts) GraphiteMetricsFindResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/metrics/find", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
+	}
+
+	var res GraphiteMetricsFindResponse
+	if err := json.Unmarshal([]byte(resText), &res); err != nil {
+		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
+	}
+	return res
+}
+
+// GraphiteMetricsExpand expands the given query with matching paths by sending
+// a request to /graphite/metrics/expand endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+// and https://graphite.readthedocs.io/en/latest/metrics_api.html#metrics-expand
+func (c *vmselectClient) GraphiteMetricsExpand(t *testing.T, query string, opts QueryOpts) GraphiteMetricsExpandResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/metrics/expand", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
+	}
+
+	var res GraphiteMetricsExpandResponse
+	if err := json.Unmarshal([]byte(resText), &res); err != nil {
+		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
+	}
+	return res
+}
+
+// GraphiteRender retrieves the raw metric data by sending a request to
+// /graphite/render endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#render-api
+// and https://graphite-api.readthedocs.io/en/latest/api.html#the-render-api-render
+func (c *vmselectClient) GraphiteRender(t *testing.T, target string, opts QueryOpts) GraphiteRenderResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/render", opts)
+	values := opts.asURLValues()
+	values.Add("format", "json")
+	values.Add("target", target)
+	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
+	}
+
+	var res GraphiteRenderResponse
+	if err := json.Unmarshal([]byte(resText), &res); err != nil {
+		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
+	}
+	return res
+}
+
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a request to /graphite/tags/tagSeries
+// endpoint.
+func (c *vmselectClient) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("select", "graphite/tags/tagSeries", opts)
+	values := opts.asURLValues()
+	values.Add("path", record)
+	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+// GraphiteTagsTagMultiSeries is a test helper function that registers Graphite
+// tags for a multiple time series by sending a request to
+// /graphite/tags/tagSeries endpoint.
+func (c *vmselectClient) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("select", "graphite/tags/tagMultiSeries", opts)
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+// PrometheusAPIV1AdminStatusMetricNamesStatsReset resets the metric name usage
+// stats by sending a request to
+// /prometheus/api/v1/admin/status/metric_names_stats/reset endpoint
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (c *vmselectClient) PrometheusAPIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts QueryOpts) {
+	t.Helper()
+	values := opts.asURLValues()
+	res, statusCode := c.vmselectCli.PostForm(t, c.metricNamesStatsResetURL, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// APIV1AdminTenants retrieves the list of tenants by sending a request to
+// /admin/tenants endpoint.
+func (c *vmselectClient) APIV1AdminTenants(t *testing.T, opts QueryOpts) *AdminTenantsResponse {
+	t.Helper()
+	res, statusCode := c.vmselectCli.Get(t, c.tenantsURL, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	tenants := &AdminTenantsResponse{}
+	if err := json.Unmarshal([]byte(res), tenants); err != nil {
+		t.Fatalf("could not unmarshal tenants response data:\n%s\n err: %v", res, err)
+	}
+
+	return tenants
+}
+
+type vminsertClient struct {
+	vminsertCli        *Client
+	url                func(op, path string, opts QueryOpts) string
+	openTSDBURL        func(op, path string, opts QueryOpts) string
+	graphiteListenAddr string
+	sendBlocking       func(t *testing.T, numRecordsToSend int, send func())
+}
+
+// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
+// of records in CSV format for the given tenant by sending an HTTP POST
+// request to prometheus/api/v1/import/csv vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (c *vminsertClient) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/import/csv", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, len(records), func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
+// of records in Native format for the given tenant by sending an HTTP POST
+// request to prometheus/api/v1/import/native vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (c *vminsertClient) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/import/native", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, 1, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1Write is a test helper function that inserts a
+// collection of records in Prometheus remote-write format by sending a HTTP
+// POST request to /prometheus/api/v1/write vminsert endpoint.
+func (c *vminsertClient) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/write", opts)
+	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
+	recordsCount := len(wr.Timeseries)
+	if prommetadata.IsEnabled() {
+		recordsCount += len(wr.Metadata)
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/x-protobuf")
+	c.sendBlocking(t, recordsCount, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// collection of records in Prometheus text exposition format for the given
+// tenant by sending a HTTP POST request to
+// /prometheus/api/v1/import/prometheus vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
+func (c *vminsertClient) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/import/prometheus", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	var recordsCount int
+	var metadataRecords int
+	uniqueMetadataMetricNames := make(map[string]struct{})
+	for _, record := range records {
+		// metric metadata has the following format:
+		//# HELP importprometheus_series
+		//# TYPE importprometheus_series
+		// it results into single metadata record
+		if strings.HasPrefix(record, "# ") {
+			metadataItems := strings.Split(record, " ")
+			if len(metadataItems) < 3 {
+				t.Fatalf("BUG: unexpected metadata format=%q", record)
+			}
+			metricName := metadataItems[2]
+			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
+				continue
+			}
+			uniqueMetadataMetricNames[metricName] = struct{}{}
+			metadataRecords++
+			continue
+		}
+		recordsCount++
+	}
+	if prommetadata.IsEnabled() {
+		recordsCount += metadataRecords
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, recordsCount, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// InfluxWrite is a test helper function that inserts a collection of records in
+// Influx line format by sending a HTTP POST request to /influx/write endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
+func (c *vminsertClient) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "influx/write", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, len(records), func() {
+		t.Helper()
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// OpentelemetryV1Metrics is a test helper function that inserts a
+// collection of records in Opentelemetry protocol format by sending a HTTP
+// POST request to /opentelemetry/v1/metrics vminsert endpoint.
+func (c *vminsertClient) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsData, opts QueryOpts) {
+	t.Helper()
+
+	var recordsCount int
+	for _, rss := range md.ResourceMetrics {
+		for _, sm := range rss.ScopeMetrics {
+			recordsCount += len(sm.Metrics)
+			for _, m := range sm.Metrics {
+				if prommetadata.IsEnabled() {
+					recordsCount += len(m.Metadata)
+				}
+			}
+		}
+	}
+	url := c.url("insert", "opentelemetry/v1/metrics", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := md.MarshalProtobuf(nil)
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/x-protobuf")
+	c.sendBlocking(t, recordsCount, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusOK {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+		}
+	})
+}
+
+// OpenTSDBAPIPut is a test helper function that inserts a collection of
+// records in OpenTSDB format for the given tenant by sending an HTTP POST
+// request to /opentsdb/api/put vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (c *vminsertClient) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.openTSDBURL("insert", "opentsdb/api/put", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte("[" + strings.Join(records, ",") + "]")
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/json")
+	c.sendBlocking(t, len(records), func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// ZabbixConnectorHistory is a test helper function that inserts a
+// collection of records in zabbixconnector  format by sending a HTTP
+// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
+func (c *vminsertClient) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "zabbixconnector/api/v1/history", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/json")
+	c.sendBlocking(t, len(records), func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusOK {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+		}
+	})
+
+}
+
+// GraphiteWrite is a test helper function that sends a
+// collection of records to graphiteListenAddr port.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
+func (c *vminsertClient) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
+	t.Helper()
+	c.vminsertCli.Write(t, c.graphiteListenAddr, records)
+}
+
+type vmstorageClient struct {
+	vmstorageCli   *Client
+	httpListenAddr string
+}
+
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
+func (c *vmstorageClient) ForceFlush(t *testing.T) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/internal/force_flush", c.httpListenAddr)
+	_, statusCode := c.vmstorageCli.Get(t, url, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// ForceMerge is a test helper function that forces the merging of parts.
+func (c *vmstorageClient) ForceMerge(t *testing.T) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/internal/force_merge", c.httpListenAddr)
+	_, statusCode := c.vmstorageCli.Get(t, url, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// SnapshotCreate creates a database snapshot by sending a query to the
+// /snapshot/create endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
+	t.Helper()
+
+	data, statusCode := c.vmstorageCli.Post(t, c.SnapshotCreateURL(), nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotCreateResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotCreateURL returns the URL for creating snapshots.
+func (c *vmstorageClient) SnapshotCreateURL() string {
+	return fmt.Sprintf("http://%s/snapshot/create", c.httpListenAddr)
+}
+
+// APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
+// /api/v1/admin/tsdb/snapshot endpoint.
+//
+// See https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot.
+func (c *vmstorageClient) APIV1AdminTSDBSnapshot(t *testing.T) *APIV1AdminTSDBSnapshotResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/api/v1/admin/tsdb/snapshot", c.httpListenAddr)
+	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res APIV1AdminTSDBSnapshotResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal prometheus snapshot create response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotList lists existing database snapshots by sending a query to the
+// /snapshot/list endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotList(t *testing.T) *SnapshotListResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/snapshot/list", c.httpListenAddr)
+	data, statusCode := c.vmstorageCli.Get(t, url, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotListResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotDelete deletes a snapshot by sending a query to the
+// /snapshot/delete endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", c.httpListenAddr, snapshotName)
+	data, statusCode := c.vmstorageCli.Delete(t, url)
+	wantStatusCodes := map[int]bool{
+		http.StatusOK:                  true,
+		http.StatusInternalServerError: true,
+	}
+	if !wantStatusCodes[statusCode] {
+		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
+	}
+
+	var res SnapshotDeleteResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotDeleteAll deletes all snapshots by sending a query to the
+// /snapshot/delete_all endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/snapshot/delete_all", c.httpListenAddr)
+	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotDeleteAllResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}

apptest/model.go

@@ -27,13 +27,16 @@ type PrometheusQuerier interface {
 	PrometheusAPIV1LabelValues(t *testing.T, labelName, query string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse
 	PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
 	PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata
-
-	APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts)
+	PrometheusAPIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse
+	PrometheusAPIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts)
 
 	// TODO(@rtm0): Prometheus does not provide this API. Either move it to a
 	// separate interface or rename this interface to allow for multiple querier
 	// types.
 	GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse
+	GraphiteMetricsFind(t *testing.T, query string, opts QueryOpts) GraphiteMetricsFindResponse
+	GraphiteMetricsExpand(t *testing.T, query string, opts QueryOpts) GraphiteMetricsExpandResponse
+	GraphiteRender(t *testing.T, target string, opts QueryOpts) GraphiteRenderResponse
 	GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts)
 	GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts)
 }
@@ -91,6 +94,9 @@ type QueryOpts struct {
 	Format         string
 	NoCache        string
 	Headers        http.Header
+	From           string
+	Until          string
+	StorageStep    string
 }
 
 func (qos *QueryOpts) getHeaders() http.Header {
@@ -123,6 +129,9 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("latency_offset", qos.LatencyOffset)
 	addNonEmpty("format", qos.Format)
 	addNonEmpty("nocache", qos.NoCache)
+	addNonEmpty("from", qos.From)
+	addNonEmpty("until", qos.Until)
+	addNonEmpty("storage_step", qos.StorageStep)
 
 	return uv
 }
@@ -480,10 +489,6 @@ type TSDBStatusResponse struct {
 	Data      TSDBStatusResponseData
 }
 
-// GraphiteMetricsIndexResponse is an in-memory representation of the json response
-// returned by the /graphite/metrics/index.json endpoint.
-type GraphiteMetricsIndexResponse = []string
-
 // AdminTenantsResponse is an in-memory representation of the json response
 // returned by the /api/v1/admin/tenants endpoint.
 type AdminTenantsResponse struct {
@@ -533,3 +538,32 @@ func sortTSDBStatusResponseEntries(entries []TSDBStatusResponseEntry) {
 		return left.Count < right.Count
 	})
 }
+
+// GraphiteMetricsIndexResponse is an in-memory representation of the json response
+// returned by the /graphite/metrics/index.json endpoint.
+type GraphiteMetricsIndexResponse = []string
+
+type GraphiteMetric struct {
+	Id            string
+	Text          string
+	AllowChildren int
+	Expandable    int
+	Leaf          int
+}
+
+// GraphiteMetricsIndexResponse is an in-memory representation of the json response
+// returned by the /graphite/metrics/find endpoint.
+type GraphiteMetricsFindResponse = []GraphiteMetric
+
+// GraphiteMetricsExpandResponse is an in-memory representation of the json response
+// returned by the /graphite/metrics/expand endpoint.
+type GraphiteMetricsExpandResponse = []string
+
+type GraphiteRenderedTarget struct {
+	Target     string
+	Datapoints [][2]float64
+}
+
+// GraphiteRenderResponse is an in-memory representation of the json response
+// returned by the /graphite/render endpoint.
+type GraphiteRenderResponse = []GraphiteRenderedTarget

apptest/tests/legacy_indexdb_test.go

@@ -214,7 +214,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 	newSUT := opts.startNewSUT()
 	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
 
-	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	newSUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	wantNoResults := &want{
 		series:       []map[string]string{},
 		queryResults: []*at.QueryResult{},
@@ -877,7 +877,7 @@ func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
 	// Ingest legacy2 records, ensure the queries return only legacy2.
 	legacySUT = opts.startLegacySUT()
 	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1, numMetrics)
-	legacySUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	legacySUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	assertQueries(legacySUT, `{__name__=~".*"}`, wantEmpty, numMetrics)
 	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy2Data, at.QueryOpts{})
 	legacySUT.ForceFlush(t)
@@ -891,7 +891,7 @@ func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
 	newSUT = opts.startNewSUT()
 	// series count includes deleted metrics
 	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy2New1, 3*numMetrics)
-	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	newSUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	// series count includes deleted metrics
 	assertQueries(newSUT, `{__name__=~".*"}`, wantEmpty, 3*numMetrics)
 	opts.stopNewSUT()

apptest/tests/metric_names_stats_test.go

@@ -48,7 +48,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3"},
 		},
 	}
-	got := sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got := sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -63,7 +63,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -90,7 +90,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 		},
 	}
 	expectedStatsResponse.Sort()
-	gotStatus := sut.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{})
+	gotStatus := sut.PrometheusAPIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
 		t.Errorf("unexpected APIV1StatusTSDB response (-want, +got):\n%s", diff)
 	}
@@ -105,7 +105,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -118,17 +118,17 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "2", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "2", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
 	// reset state and check empty request response
-	sut.APIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
+	sut.PrometheusAPIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
 	expected = apptest.MetricNamesStatsResponse{
 		Records: []apptest.MetricNamesStatsRecord{},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -158,7 +158,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 		fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VmselectAddr(), vmstorage2.VmselectAddr()),
 	})
 	// verify empty stats
-	resp := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "0:0"})
+	resp := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "0:0"})
 	if len(resp.Records) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Records), 0)
 	}
@@ -198,7 +198,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 				{MetricName: "metric_name_3"},
 			},
 		}
-		gotStats := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
+		gotStats := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expected, gotStats); diff != "" {
 			t.Errorf("unexpected response (-want, +got):\n%s", diff)
 		}
@@ -216,7 +216,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 				{MetricName: "metric_name_1", QueryRequestsCount: 3},
 			},
 		}
-		gotStats = vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
+		gotStats = vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expected, gotStats); diff != "" {
 			t.Errorf("unexpected response tenant: %s (-want, +got):\n%s", tenantID, diff)
 		}
@@ -243,9 +243,9 @@ func TestClusterMetricNamesStats(t *testing.T) {
 			},
 		}
 		expectedStatsResponse.Sort()
-		gotStatus := vmselect.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{Tenant: tenantID})
+		gotStatus := vmselect.PrometheusAPIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
-			t.Errorf("unexpected APIV1StatusTSDB response tenant: %s (-want, +got):\n%s", tenantID, diff)
+			t.Errorf("unexpected TSDB status for tenant %s (-want, +got):\n%s", tenantID, diff)
 		}
 	}
 
@@ -258,14 +258,14 @@ func TestClusterMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_1", QueryRequestsCount: 9},
 		},
 	}
-	gotStats := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
+	gotStats := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
 	if diff := cmp.Diff(expected, gotStats); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
 	// reset cache and check empty state
-	vmselect.MetricNamesStatsReset(t, apptest.QueryOpts{})
-	resp = vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
+	vmselect.PrometheusAPIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
+	resp = vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
 	if len(resp.Records) != 0 {
 		t.Fatalf("want 0 records, got: %d", len(resp.Records))
 	}

apptest/tests/multitenancy_via_headers_test.go

@@ -219,7 +219,7 @@ func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
 	tenantID := make(http.Header)
 	tenantID.Set("AccountID", "5")
 	tenantID.Set("ProjectID", "15")
-	vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
 		Headers: tenantID,
 	})
 	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
@@ -244,7 +244,7 @@ func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
 	}
 
 	// Delete series for multitenant with tenant filter
-	vmselect.APIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
 		Headers: multitenant,
 	})
 

apptest/tests/multitenant_test.go

@@ -192,7 +192,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	}
 
 	// Delete series from specific tenant
-	vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "5:15",
 	})
 	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
@@ -215,7 +215,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	}
 
 	// Delete series for multitenant with tenant filter
-	vmselect.APIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
 		Tenant: "multitenant",
 	})
 

apptest/vmagent.go

@@ -19,9 +19,11 @@ import (
 // Vmagent holds the state of a vmagent app and provides vmagent-specific functions
 type Vmagent struct {
 	*app
-	*ServesMetrics
+	*metricsClient
 
 	httpListenAddr string
+
+	cli *Client
 }
 
 // StartVmagent starts an instance of vmagent with the given flags. It also
@@ -46,12 +48,10 @@ func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfig
 	}
 
 	return &Vmagent{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
-		},
+		app:            app,
+		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
 		httpListenAddr: stderrExtracts[0],
+		cli:            cli,
 	}, nil
 }
 

apptest/vmauth.go

@@ -1,7 +1,6 @@
 package apptest
 
 import (
-	"fmt"
 	"io"
 	"regexp"
 	"syscall"
@@ -17,7 +16,7 @@ var httpBuilitinListenAddrRE = regexp.MustCompile(`pprof handlers are exposed at
 // functions.
 type Vmauth struct {
 	*app
-	*ServesMetrics
+	*metricsClient
 
 	httpListenAddr string
 	configFilePath string
@@ -45,11 +44,8 @@ func StartVmauth(instance string, flags []string, cli *Client, configFilePath st
 	}
 
 	return &Vmauth{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
-		},
+		app:            app,
+		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
 		httpListenAddr: stderrExtracts[0],
 		configFilePath: configFilePath,
 		cli:            cli,

apptest/vminsert.go

@@ -3,31 +3,21 @@ package apptest
 import (
 	"fmt"
 	"io"
-	"net/http"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
-
-	"github.com/golang/snappy"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 )
 
 // Vminsert holds the state of a vminsert app and provides vminsert-specific
 // functions.
 type Vminsert struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vminsertClient
 
 	httpListenAddr          string
 	clusternativeListenAddr string
-	graphiteListenAddr      string
-	openTSDBListenAddr      string
-
-	cli *Client
 }
 
 // storageNodes returns the storage node addresses passed to vminsert via
@@ -73,17 +63,26 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
+	metricsClient := newMetricsClient(cli, stderrExtracts[0])
 	return &Vminsert{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
+		app:           app,
+		metricsClient: metricsClient,
+		vminsertClient: &vminsertClient{
+			vminsertCli: cli,
+			url: func(op, path string, opts QueryOpts) string {
+				return getClusterPath(stderrExtracts[0], op, path, opts)
+			},
+			openTSDBURL: func(op, path string, opts QueryOpts) string {
+				return getClusterPath(stderrExtracts[3], op, path, opts)
+			},
+			graphiteListenAddr: stderrExtracts[2],
+			sendBlocking: func(t *testing.T, numRecordsToSend int, send func()) {
+				t.Helper()
+				sendBlocking(t, metricsClient, numRecordsToSend, send)
+			},
 		},
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
-		graphiteListenAddr:      stderrExtracts[2],
-		openTSDBListenAddr:      stderrExtracts[3],
-		cli:                     cli,
 	}, nil
 }
 
@@ -99,247 +98,6 @@ func (app *Vminsert) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
-// InfluxWrite is a test helper function that inserts a
-// collection of records in Influx line format by sending a HTTP
-// POST request to /influx/write vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
-func (app *Vminsert) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "influx/write", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// GraphiteWrite is a test helper function that sends a
-// collection of records to graphiteListenAddr port.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
-func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
-	t.Helper()
-	app.cli.Write(t, app.graphiteListenAddr, records)
-}
-
-// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
-// of records in CSV format for the given tenant by sending an HTTP POST
-// request to prometheus/api/v1/import/csv vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/csv", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
-// of records in Native format for the given tenant by sending an HTTP POST
-// request to prometheus/api/v1/import/native vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/native", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, 1, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// OpenTSDBAPIPut is a test helper function that inserts a collection of
-// records in OpenTSDB format for the given tenant by sending an HTTP POST
-// request to /opentsdb/api/put vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.openTSDBListenAddr, "insert", "opentsdb/api/put", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte("[" + strings.Join(records, ",") + "]")
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1Write is a test helper function that inserts a
-// collection of records in Prometheus remote-write format by sending a HTTP
-// POST request to /prometheus/api/v1/write vminsert endpoint.
-func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/write", opts)
-	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
-	recordsCount := len(wr.Timeseries)
-	if prommetadata.IsEnabled() {
-		recordsCount += len(wr.Metadata)
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	app.sendBlocking(t, recordsCount, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
-// collection of records in Prometheus text exposition format for the given
-// tenant by sending a HTTP POST request to
-// /prometheus/api/v1/import/prometheus vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
-func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/prometheus", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	var recordsCount int
-	var metadataRecords int
-	uniqueMetadataMetricNames := make(map[string]struct{})
-	for _, record := range records {
-		// metric metadata has the following format:
-		//# HELP importprometheus_series
-		//# TYPE importprometheus_series
-		// it results into single metadata record
-		if strings.HasPrefix(record, "# ") {
-			metadataItems := strings.Split(record, " ")
-			if len(metadataItems) < 2 {
-				t.Fatalf("BUG: unexpected metadata format=%q", record)
-			}
-			metricName := metadataItems[2]
-			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
-				continue
-			}
-			uniqueMetadataMetricNames[metricName] = struct{}{}
-			metadataRecords++
-			continue
-		}
-		recordsCount++
-	}
-	if prommetadata.IsEnabled() {
-		recordsCount += metadataRecords
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, recordsCount, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// ZabbixConnectorHistory is a test helper function that inserts a
-// collection of records in zabbixconnector  format by sending a HTTP
-// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
-func (app *Vminsert) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "zabbixconnector/api/v1/history", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusOK {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-		}
-	})
-
-}
-
-// OpentelemetryV1Metrics is a test helper function that inserts a
-// collection of records in Opentelemetry protocol format by sending a HTTP
-// POST request to /opentelemetry/v1/metrics vminsert endpoint.
-func (app *Vminsert) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsData, opts QueryOpts) {
-	t.Helper()
-
-	var recordsCount int
-	for _, rss := range md.ResourceMetrics {
-		for _, sm := range rss.ScopeMetrics {
-			recordsCount += len(sm.Metrics)
-			for _, m := range sm.Metrics {
-				if prommetadata.IsEnabled() {
-					recordsCount += len(m.Metadata)
-				}
-			}
-		}
-	}
-	url := getClusterPath(app.httpListenAddr, "insert", "opentelemetry/v1/metrics", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := md.MarshalProtobuf(nil)
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	app.sendBlocking(t, recordsCount, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusOK {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-		}
-	})
-}
-
 // String returns the string representation of the vminsert app state.
 func (app *Vminsert) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)
@@ -355,13 +113,10 @@ func (app *Vminsert) String() string {
 // Waiting is implemented a retrieving the value of `vm_rpc_rows_sent_total`
 // metric and checking whether it is equal or greater than the wanted value.
 // If it is, then the data has been sent to vmstorage.
-//
-// Unreliable if the records are inserted concurrently.
-// TODO(rtm0): Put sending and waiting into a critical section to make reliable?
-func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
+func sendBlocking(t *testing.T, c *metricsClient, numRecordsToSend int, send func()) {
 	t.Helper()
 
-	wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
+	wantRowsSentCount := c.rpcRowsSentTotal(t) + numRecordsToSend
 
 	send()
 
@@ -370,7 +125,7 @@ func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func(
 		period  = 100 * time.Millisecond
 	)
 	for range retries {
-		d := app.rpcRowsSentTotal(t)
+		d := c.rpcRowsSentTotal(t)
 		if d >= wantRowsSentCount {
 			return
 		}
@@ -378,14 +133,3 @@ func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func(
 	}
 	t.Fatalf("timed out while waiting for inserted rows to be sent to vmstorage")
 }
-
-// rpcRowsSentTotal retrieves the values of all vminsert
-// `vm_rpc_rows_sent_total` metrics (there will be one for each vmstorage) and
-// returns their integer sum.
-func (app *Vminsert) rpcRowsSentTotal(t *testing.T) int {
-	total := 0.0
-	for _, v := range app.GetMetricsByPrefix(t, "vm_rpc_rows_sent_total") {
-		total += v
-	}
-	return int(total)
-}

apptest/vmselect.go

@@ -1,20 +1,17 @@
 package apptest
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"regexp"
-	"strconv"
-	"testing"
 )
 
 // Vmselect holds the state of a vmselect app and provides vmselect-specific
 // functions.
 type Vmselect struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vmselectClient
 
 	httpListenAddr          string
 	clusternativeListenAddr string
@@ -41,10 +38,15 @@ func StartVmselect(instance string, flags []string, cli *Client, output io.Write
 	}
 
 	return &Vmselect{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
+		app:           app,
+		metricsClient: newMetricsClient(cli, stderrExtracts[0]),
+		vmselectClient: &vmselectClient{
+			vmselectCli: cli,
+			url: func(op, path string, opts QueryOpts) string {
+				return getClusterPath(stderrExtracts[0], op, path, opts)
+			},
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[0]),
+			tenantsURL:               fmt.Sprintf("http://%s/admin/tenants", stderrExtracts[0]),
 		},
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
@@ -64,299 +66,6 @@ func (app *Vmselect) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
-// PrometheusAPIV1Export is a test helper function that performs the export of
-// raw samples in JSON line format by sending a HTTP POST request to
-// /prometheus/api/v1/export vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
-func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-	res, _ := app.cli.PostForm(t, exportURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1ExportNative is a test helper function that performs the export of
-// raw samples in native binary format by sending an HTTP POST request to
-// /prometheus/api/v1/export/native vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
-func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
-	t.Helper()
-
-	exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export/native", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-	res, _ := app.cli.PostForm(t, exportURL, values, opts.Headers)
-	return []byte(res)
-}
-
-// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
-// instant query by sending a HTTP POST request to /prometheus/api/v1/query
-// vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
-func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1QueryRange is a test helper function that performs
-// PromQL/MetricsQL range query by sending a HTTP POST request to
-// /prometheus/api/v1/query_range vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
-func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query_range", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
-// and returns the list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
-	t.Helper()
-
-	seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesResponse(t, res)
-}
-
-// PrometheusAPIV1SeriesCount sends a query to a /prometheus/api/v1/series/count endpoint
-// and returns the total number of time series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmselect) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
-	t.Helper()
-
-	seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series/count", opts)
-	values := opts.asURLValues()
-
-	res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesCountResponse(t, res)
-}
-
-// PrometheusAPIV1Labels sends a query to a /prometheus/api/v1/labels endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
-func (app *Vmselect) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/labels", opts)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelsResponse(t, res)
-}
-
-// PrometheusAPIV1LabelValues sends a query to a /prometheus/api/v1/label/.../values endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
-func (app *Vmselect) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	suffix := fmt.Sprintf("prometheus/api/v1/label/%s/values", labelName)
-	queryURL := getClusterPath(app.httpListenAddr, "select", suffix, opts)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelValuesResponse(t, res)
-}
-
-// PrometheusAPIV1Metadata sends a query to a /prometheus/api/v1/metadata endpoint
-// and returns the results.
-func (app *Vmselect) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("metric", metric)
-	values.Add("limit", strconv.Itoa(limit))
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/metadata", opts)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1Metadata(t, res)
-}
-
-// APIV1AdminTSDBDeleteSeries deletes the series that match the query by sending
-// a request to /api/v1/admin/tsdb/delete_series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
-func (app *Vmselect) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
-	t.Helper()
-
-	queryURL := getClusterPath(app.httpListenAddr, "delete", "prometheus/api/v1/admin/tsdb/delete_series", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// MetricNamesStats sends a query to a /select/tenant/prometheus/api/v1/status/metric_names_stats endpoint
-// and returns the statistics response for given params.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmselect) MetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("limit", limit)
-	values.Add("le", le)
-	values.Add("match_pattern", matchPattern)
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/metric_names_stats", opts)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-	var resp MetricNamesStatsResponse
-	if err := json.Unmarshal([]byte(res), &resp); err != nil {
-		t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", res, err)
-	}
-	return resp
-}
-
-// MetricNamesStatsReset sends a query to a /admin/api/v1/status/metric_names_stats/reset endpoint
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmselect) MetricNamesStatsReset(t *testing.T, opts QueryOpts) {
-	t.Helper()
-
-	values := opts.asURLValues()
-	queryURL := fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", app.httpListenAddr)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
-// //
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
-func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/tsdb", opts)
-	values := opts.asURLValues()
-	addNonEmpty := func(name, value string) {
-		if len(value) == 0 {
-			return
-		}
-		values.Add(name, value)
-	}
-	addNonEmpty("match[]", matchQuery)
-	addNonEmpty("topN", topN)
-	addNonEmpty("date", date)
-
-	res, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var status TSDBStatusResponse
-	if err := json.Unmarshal([]byte(res), &status); err != nil {
-		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
-	}
-	status.Sort()
-	return status
-}
-
-// GraphiteMetricsIndex sends a query to a /graphite/metrics/index.json
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
-func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "graphite/metrics/index.json", opts)
-	res, statusCode := app.cli.Get(t, url, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var index GraphiteMetricsIndexResponse
-	if err := json.Unmarshal([]byte(res), &index); err != nil {
-		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
-	}
-	return index
-}
-
-// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
-// for a single time series by sending a HTTP POST request to
-// /graphite/tags/tagSeries vmsingle endpoint.
-func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagSeries", opts)
-	values := opts.asURLValues()
-	values.Add("path", record)
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-func (app *Vmselect) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagMultiSeries", opts)
-	values := opts.asURLValues()
-	for _, rec := range records {
-		values.Add("path", rec)
-	}
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-// APIV1AdminTenants sends a query to a /admin/tenants endpoint
-func (app *Vmselect) APIV1AdminTenants(t *testing.T) *AdminTenantsResponse {
-	t.Helper()
-
-	tenantsURL := fmt.Sprintf("http://%s/admin/tenants", app.httpListenAddr)
-	res, statusCode := app.cli.Get(t, tenantsURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var tenants *AdminTenantsResponse
-	if err := json.Unmarshal([]byte(res), tenants); err != nil {
-		t.Fatalf("could not unmarshal tenants response data:\n%s\n err: %v", res, err)
-	}
-
-	return tenants
-}
-
 // String returns the string representation of the vmselect app state.
 func (app *Vmselect) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)

apptest/vmsingle.go

@@ -1,49 +1,25 @@
 package apptest
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"regexp"
-	"strconv"
-	"strings"
 	"testing"
 	"time"
-
-	"github.com/golang/snappy"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 )
 
 // Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
 // functions.
 type Vmsingle struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vmstorageClient
+	*vmselectClient
+	*vminsertClient
 
 	storageDataPath string
 	httpListenAddr  string
-
-	// vmstorage URLs.
-	forceFlushURL string
-	forceMergeURL string
-
-	// vminsert URLs.
-	influxLineWriteURL                 string
-	graphiteWriteAddr                  string
-	openTSDBHTTPURL                    string
-	prometheusAPIV1ImportPrometheusURL string
-	prometheusAPIV1WriteURL            string
-
-	// vmselect URLs.
-	prometheusAPIV1ExportURL       string
-	prometheusAPIV1ExportNativeURL string
-	prometheusAPIV1QueryURL        string
-	prometheusAPIV1QueryRangeURL   string
-	prometheusAPIV1SeriesURL       string
 }
 
 // StartVmsingleAt starts an instance of vmsingle with the given flags. It also
@@ -70,617 +46,39 @@ func StartVmsingleAt(instance, binary string, flags []string, cli *Client, outpu
 	}
 
 	return &Vmsingle{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
-			cli:        cli,
+		app:           app,
+		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
+		vmstorageClient: &vmstorageClient{
+			vmstorageCli:   cli,
+			httpListenAddr: stderrExtracts[1],
+		},
+		vmselectClient: &vmselectClient{
+			vmselectCli: cli,
+			url: func(op, path string, opts QueryOpts) string {
+				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+			},
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[1]),
+			tenantsURL:               "vmsingle-does-not-serve-tenants",
+		},
+		vminsertClient: &vminsertClient{
+			vminsertCli: cli,
+			url: func(_, path string, _ QueryOpts) string {
+				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+			},
+			openTSDBURL: func(_, path string, _ QueryOpts) string {
+				return fmt.Sprintf("http://%s/%s", stderrExtracts[3], path)
+			},
+			graphiteListenAddr: stderrExtracts[2],
+			sendBlocking: func(t *testing.T, _ int, send func()) {
+				t.Helper()
+				send()
+			},
 		},
 		storageDataPath: stderrExtracts[0],
 		httpListenAddr:  stderrExtracts[1],
-
-		forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
-		forceMergeURL: fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]),
-
-		influxLineWriteURL:                 fmt.Sprintf("http://%s/influx/write", stderrExtracts[1]),
-		graphiteWriteAddr:                  stderrExtracts[2],
-		openTSDBHTTPURL:                    fmt.Sprintf("http://%s", stderrExtracts[3]),
-		prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
-		prometheusAPIV1WriteURL:            fmt.Sprintf("http://%s/prometheus/api/v1/write", stderrExtracts[1]),
-		prometheusAPIV1ExportURL:           fmt.Sprintf("http://%s/prometheus/api/v1/export", stderrExtracts[1]),
-		prometheusAPIV1ExportNativeURL:     fmt.Sprintf("http://%s/prometheus/api/v1/export/native", stderrExtracts[1]),
-		prometheusAPIV1QueryURL:            fmt.Sprintf("http://%s/prometheus/api/v1/query", stderrExtracts[1]),
-		prometheusAPIV1QueryRangeURL:       fmt.Sprintf("http://%s/prometheus/api/v1/query_range", stderrExtracts[1]),
-		prometheusAPIV1SeriesURL:           fmt.Sprintf("http://%s/prometheus/api/v1/series", stderrExtracts[1]),
 	}, nil
 }
 
-// ForceFlush is a test helper function that forces the flushing of inserted
-// data, so it becomes available for searching immediately.
-func (app *Vmsingle) ForceFlush(t *testing.T) {
-	t.Helper()
-
-	_, statusCode := app.cli.Get(t, app.forceFlushURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// ForceMerge is a test helper function that forces the merging of parts.
-func (app *Vmsingle) ForceMerge(t *testing.T) {
-	t.Helper()
-
-	_, statusCode := app.cli.Get(t, app.forceMergeURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// InfluxWrite is a test helper function that inserts a
-// collection of records in Influx line format by sending a HTTP
-// POST request to /influx/write vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
-func (app *Vmsingle) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	data := []byte(strings.Join(records, "\n"))
-
-	url := app.influxLineWriteURL
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// GraphiteWrite is a test helper function that sends a collection of records
-// to graphiteListenAddr port.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
-func (app *Vmsingle) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
-	t.Helper()
-	app.cli.Write(t, app.graphiteWriteAddr, records)
-}
-
-// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
-// of records in CSV format for the given tenant by sending an HTTP POST
-// request to /api/v1/import/csv vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-import-csv-data
-func (app *Vmsingle) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/api/v1/import/csv", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
-// of records in native format for the given tenant by sending an HTTP POST
-// request to /api/v1/import/native vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-native-format
-func (app *Vmsingle) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/api/v1/import/native", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// OpenTSDBAPIPut is a test helper function that inserts a collection of
-// records in OpenTSDB format for the given tenant by sending an HTTP POST
-// request to /api/put vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http
-func (app *Vmsingle) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	// add extra label
-	url := app.openTSDBHTTPURL + "/api/put"
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte("[" + strings.Join(records, ",") + "]")
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1Write is a test helper function that inserts a
-// collection of records in Prometheus remote-write format by sending a HTTP
-// POST request to /prometheus/api/v1/write vmsingle endpoint.
-func (app *Vmsingle) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
-	t.Helper()
-
-	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	_, statusCode := app.cli.Post(t, app.prometheusAPIV1WriteURL, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
-// collection of records in Prometheus text exposition format by sending a HTTP
-// POST request to /prometheus/api/v1/import/prometheus vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
-func (app *Vmsingle) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	// add extra label
-	url := app.prometheusAPIV1ImportPrometheusURL
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	data := []byte(strings.Join(records, "\n"))
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1Export is a test helper function that performs the export of
-// raw samples in JSON line format by sending a HTTP POST request to
-// /prometheus/api/v1/export vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
-func (app *Vmsingle) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1ExportURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1ExportNative is a test helper function that performs the export of
-// raw samples in native binary format by sending an HTTP POST request to
-// /prometheus/api/v1/export/native vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
-func (app *Vmsingle) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
-	t.Helper()
-
-	t.Helper()
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1ExportNativeURL, values, opts.Headers)
-	return []byte(res)
-}
-
-// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
-// instant query by sending a HTTP POST request to /prometheus/api/v1/query
-// vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
-func (app *Vmsingle) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("query", query)
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1QueryRange is a test helper function that performs
-// PromQL/MetricsQL range query by sending a HTTP POST request to
-// /prometheus/api/v1/query_range vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
-func (app *Vmsingle) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("query", query)
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryRangeURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
-// and returns the list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmsingle) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1SeriesURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesResponse(t, res)
-}
-
-// PrometheusAPIV1SeriesCount sends a query to a /prometheus/api/v1/series/count endpoint
-// and returns the total number of time series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmsingle) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/series/count", app.httpListenAddr)
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesCountResponse(t, res)
-}
-
-// PrometheusAPIV1Labels sends a query to a /prometheus/api/v1/labels endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
-func (app *Vmsingle) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/labels", app.httpListenAddr)
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelsResponse(t, res)
-}
-
-// PrometheusAPIV1LabelValues sends a query to a /prometheus/api/v1/label/.../values endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
-func (app *Vmsingle) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/label/%s/values", app.httpListenAddr, labelName)
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelValuesResponse(t, res)
-}
-
-// PrometheusAPIV1Metadata sends a query to a /prometheus/api/v1/metadata endpoint
-// and returns the results.
-func (app *Vmsingle) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("metric", metric)
-	values.Add("limit", strconv.Itoa(limit))
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/metadata", app.httpListenAddr)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1Metadata(t, res)
-}
-
-// APIV1AdminTSDBDeleteSeries deletes the series that match the query by sending
-// a request to /api/v1/admin/tsdb/delete_series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
-func (app *Vmsingle) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/api/v1/admin/tsdb/delete_series", app.httpListenAddr)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// GraphiteMetricsIndex sends a query to a /metrics/index.json
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
-func (app *Vmsingle) GraphiteMetricsIndex(t *testing.T, _ QueryOpts) GraphiteMetricsIndexResponse {
-	t.Helper()
-
-	seriesURL := fmt.Sprintf("http://%s/metrics/index.json", app.httpListenAddr)
-	res, statusCode := app.cli.Get(t, seriesURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var index GraphiteMetricsIndexResponse
-	if err := json.Unmarshal([]byte(res), &index); err != nil {
-		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
-	}
-	return index
-}
-
-// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
-// for a single time series by sending a HTTP POST request to
-// /graphite/tags/tagSeries vmsingle endpoint.
-func (app *Vmsingle) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/graphite/tags/tagSeries", app.httpListenAddr)
-	values := opts.asURLValues()
-	values.Add("path", record)
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-func (app *Vmsingle) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/graphite/tags/tagMultiSeries", app.httpListenAddr)
-	values := opts.asURLValues()
-	for _, rec := range records {
-		values.Add("path", rec)
-	}
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-// APIV1StatusMetricNamesStats sends a query to a /api/v1/status/metric_names_stats endpoint
-// and returns the statistics response for given params.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmsingle) APIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("limit", limit)
-	values.Add("le", le)
-	values.Add("match_pattern", matchPattern)
-	queryURL := fmt.Sprintf("http://%s/api/v1/status/metric_names_stats", app.httpListenAddr)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-	var resp MetricNamesStatsResponse
-	if err := json.Unmarshal([]byte(res), &resp); err != nil {
-		t.Fatalf("could not unmarshal metric names stats response data:\n%s\n err: %v", res, err)
-	}
-	return resp
-}
-
-// APIV1AdminStatusMetricNamesStatsReset sends a query to a /api/v1/admin/status/metric_names_stats/reset endpoint
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmsingle) APIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts QueryOpts) {
-	t.Helper()
-
-	values := opts.asURLValues()
-	queryURL := fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", app.httpListenAddr)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// SnapshotCreate creates a database snapshot by sending a query to the
-// /snapshot/create endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
-	t.Helper()
-
-	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotCreateResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotCreateURL returns the URL for creating snapshots.
-func (app *Vmsingle) SnapshotCreateURL() string {
-	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-}
-
-// APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
-// /api/v1/admin/tsdb/snapshot endpoint.
-//
-// See https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot.
-func (app *Vmsingle) APIV1AdminTSDBSnapshot(t *testing.T) *APIV1AdminTSDBSnapshotResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/api/v1/admin/tsdb/snapshot", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res APIV1AdminTSDBSnapshotResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal prometheus snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotList lists existing database snapshots by sending a query to the
-// /snapshot/list endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotList(t *testing.T) *SnapshotListResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/list", app.httpListenAddr)
-	data, statusCode := app.cli.Get(t, queryURL, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotListResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDelete deletes a snapshot by sending a query to the
-// /snapshot/delete endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", app.httpListenAddr, snapshotName)
-	data, statusCode := app.cli.Delete(t, queryURL)
-	wantStatusCodes := map[int]bool{
-		http.StatusOK:                  true,
-		http.StatusInternalServerError: true,
-	}
-	if !wantStatusCodes[statusCode] {
-		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
-	}
-
-	var res SnapshotDeleteResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDeleteAll deletes all snapshots by sending a query to the
-// /snapshot/delete_all endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete_all", app.httpListenAddr)
-	data, statusCode := app.cli.Get(t, queryURL, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotDeleteAllResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
-// //
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
-func (app *Vmsingle) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
-	t.Helper()
-
-	seriesURL := fmt.Sprintf("http://%s/prometheus/api/v1/status/tsdb", app.httpListenAddr)
-	values := opts.asURLValues()
-	addNonEmpty := func(name, value string) {
-		if len(value) == 0 {
-			return
-		}
-		values.Add(name, value)
-	}
-	addNonEmpty("match[]", matchQuery)
-	addNonEmpty("topN", topN)
-	addNonEmpty("date", date)
-
-	res, statusCode := app.cli.PostForm(t, seriesURL, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var status TSDBStatusResponse
-	if err := json.Unmarshal([]byte(res), &status); err != nil {
-		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
-	}
-	status.Sort()
-	return status
-}
-
-// ZabbixConnectorHistory is a test helper function that inserts a
-// collection of records in zabbixconnector  format by sending a HTTP
-// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
-func (app *Vmsingle) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/zabbixconnector/api/v1/history", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// OpentelemetryV1Metrics is a test helper function that inserts a
-// collection of records in Opentelemetry protocol format by sending a HTTP
-// POST request to /opentelemetry/v1/metrics vmsingle endpoint.
-func (app *Vmsingle) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsData, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/opentelemetry/v1/metrics", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := md.MarshalProtobuf(nil)
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
 // HTTPAddr returns the address at which the vminsert process is
 // listening for incoming HTTP requests.
 func (app *Vmsingle) HTTPAddr() string {

apptest/vmstorage.go

@@ -1,13 +1,10 @@
 package apptest
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"regexp"
-	"testing"
 	"time"
 )
 
@@ -15,7 +12,8 @@ import (
 // functions.
 type Vmstorage struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vmstorageClient
 
 	storageDataPath string
 	httpListenAddr  string
@@ -47,10 +45,11 @@ func StartVmstorageAt(instance, binary string, flags []string, cli *Client, outp
 	}
 
 	return &Vmstorage{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
-			cli:        cli,
+		app:           app,
+		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
+		vmstorageClient: &vmstorageClient{
+			vmstorageCli:   cli,
+			httpListenAddr: stderrExtracts[1],
 		},
 		storageDataPath: stderrExtracts[0],
 		httpListenAddr:  stderrExtracts[1],
@@ -71,121 +70,6 @@ func (app *Vmstorage) VmselectAddr() string {
 	return app.vmselectAddr
 }
 
-// ForceFlush is a test helper function that forces the flushing of inserted
-// data, so it becomes available for searching immediately.
-func (app *Vmstorage) ForceFlush(t *testing.T) {
-	t.Helper()
-
-	forceFlushURL := fmt.Sprintf("http://%s/internal/force_flush", app.httpListenAddr)
-	_, statusCode := app.cli.Get(t, forceFlushURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// ForceMerge is a test helper function that forces the merging of parts.
-func (app *Vmstorage) ForceMerge(t *testing.T) {
-	t.Helper()
-
-	forceMergeURL := fmt.Sprintf("http://%s/internal/force_merge", app.httpListenAddr)
-	_, statusCode := app.cli.Get(t, forceMergeURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// SnapshotCreate creates a database snapshot by sending a query to the
-// /snapshot/create endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
-	t.Helper()
-
-	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotCreateResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotCreateURL returns the URL for creating snapshots.
-func (app *Vmstorage) SnapshotCreateURL() string {
-	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-}
-
-// SnapshotList lists existing database snapshots by sending a query to the
-// /snapshot/list endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotList(t *testing.T) *SnapshotListResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/list", app.httpListenAddr)
-	data, statusCode := app.cli.Get(t, queryURL, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotListResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDelete deletes a snapshot by sending a query to the
-// /snapshot/delete endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", app.httpListenAddr, snapshotName)
-	data, statusCode := app.cli.Delete(t, queryURL)
-	wantStatusCodes := map[int]bool{
-		http.StatusOK:                  true,
-		http.StatusInternalServerError: true,
-	}
-	if !wantStatusCodes[statusCode] {
-		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
-	}
-
-	var res SnapshotDeleteResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDeleteAll deletes all snapshots by sending a query to the
-// /snapshot/delete_all endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete_all", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotDeleteAllResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
 // String returns the string representation of the vmstorage app state.
 func (app *Vmstorage) String() string {
 	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q vminsertAddr: %q vmselectAddr: %q}", []any{

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.142.0
+    image: victoriametrics/vmauth:v1.143.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     depends_on:
       - "victoriametrics"
     ports:

docs/Makefile

@@ -61,11 +61,11 @@ docs-update-vmsingle-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) victoria-metrics && \
-	./bin/victoria-metrics -help > /tmp/victoria_metrics_common_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) victoria-metrics)
+	(cd /tmp/vm-opensource && ./bin/victoria-metrics -help > /tmp/victoria_metrics_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise && $(MAKE) victoria-metrics && \
-	./bin/victoria-metrics -help > /tmp/victoria_metrics_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise && $(MAKE) victoria-metrics)
+	(cd /tmp/vm-enterprise && ./bin/victoria-metrics -help > /tmp/victoria_metrics_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/victoria_metrics_common_flags.md && \
 	cat /tmp/victoria_metrics_common_flags_tmp.md >> docs/victoriametrics/victoria_metrics_common_flags.md && \
@@ -90,10 +90,11 @@ docs-update-vmauth-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmauth && \
-	./bin/vmauth -help > /tmp/vmauth_common_flags_tmp.md
-	git checkout $(TAG)-enterprise && $(MAKE) vmauth && \
-	./bin/vmauth -help > /tmp/vmauth_enterprise_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmauth)
+	(cd /tmp/vm-opensource && ./bin/vmauth -help > /tmp/vmauth_common_flags_tmp.md)
+
+	(cd /tmp/vm-enterprise && $(MAKE) vmauth)
+	(cd /tmp/vm-enterprise && ./bin/vmauth -help > /tmp/vmauth_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmauth_common_flags.md
 	cat /tmp/vmauth_common_flags_tmp.md >> docs/victoriametrics/vmauth_common_flags.md
@@ -115,11 +116,11 @@ docs-update-vmagent-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmagent && \
-	./bin/vmagent -help > /tmp/vmagent_common_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmagent)
+	(cd /tmp/vm-opensource && ./bin/vmagent -help > /tmp/vmagent_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise && $(MAKE) vmagent && \
-	./bin/vmagent -help > /tmp/vmagent_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise && $(MAKE) vmagent)
+	(cd /tmp/vm-enterprise && ./bin/vmagent -help > /tmp/vmagent_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmagent_common_flags.md && \
 	cat /tmp/vmagent_common_flags_tmp.md >> docs/victoriametrics/vmagent_common_flags.md && \
@@ -143,11 +144,11 @@ docs-update-vmalert-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmalert && \
-	./bin/vmalert -help > /tmp/vmalert_common_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmalert)
+	(cd /tmp/vm-opensource && ./bin/vmalert -help > /tmp/vmalert_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise && $(MAKE) vmalert && \
-	./bin/vmalert -help > /tmp/vmalert_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise && $(MAKE) vmalert)
+	(cd /tmp/vm-enterprise && ./bin/vmalert -help > /tmp/vmalert_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmalert_common_flags.md && \
 	cat /tmp/vmalert_common_flags_tmp.md >> docs/victoriametrics/vmalert_common_flags.md && \
@@ -170,11 +171,11 @@ docs-update-vmselect-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG)-cluster && $(MAKE) vmselect && \
-	./bin/vmselect -help > /tmp/vmselect_common_flags_tmp.md
+	(cd /tmp/vm-opensource-cluster && $(MAKE) vmselect)
+	(cd /tmp/vm-opensource-cluster && ./bin/vmselect -help > /tmp/vmselect_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise-cluster && $(MAKE) vmselect && \
-	./bin/vmselect -help > /tmp/vmselect_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise-cluster && $(MAKE) vmselect)
+	(cd /tmp/vm-enterprise-cluster && ./bin/vmselect -help > /tmp/vmselect_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmselect_common_flags.md && \
 	cat /tmp/vmselect_common_flags_tmp.md >> docs/victoriametrics/vmselect_common_flags.md && \
@@ -199,11 +200,11 @@ docs-update-vminsert-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG)-cluster && $(MAKE) vminsert && \
-	./bin/vminsert -help > /tmp/vminsert_common_flags_tmp.md
+	(cd /tmp/vm-opensource-cluster && $(MAKE) vminsert)
+	(cd /tmp/vm-opensource-cluster && ./bin/vminsert -help > /tmp/vminsert_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise-cluster && $(MAKE) vminsert && \
-	./bin/vminsert -help > /tmp/vminsert_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise-cluster && $(MAKE) vminsert)
+	(cd /tmp/vm-enterprise-cluster && ./bin/vminsert -help > /tmp/vminsert_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vminsert_common_flags.md && \
 	cat /tmp/vminsert_common_flags_tmp.md >> docs/victoriametrics/vminsert_common_flags.md && \
@@ -235,11 +236,11 @@ docs-update-vmstorage-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG)-cluster && $(MAKE) vmstorage && \
-	./bin/vmstorage -help > /tmp/vmstorage_common_flags_tmp.md
+	(cd /tmp/vm-opensource-cluster && $(MAKE) vmstorage)
+	(cd /tmp/vm-opensource-cluster && ./bin/vmstorage -help > /tmp/vmstorage_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise-cluster && $(MAKE) vmstorage && \
-	./bin/vmstorage -help > /tmp/vmstorage_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise-cluster && $(MAKE) vmstorage)
+	(cd /tmp/vm-enterprise-cluster && ./bin/vmstorage -help > /tmp/vmstorage_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmstorage_common_flags.md && \
 	cat /tmp/vmstorage_common_flags_tmp.md >> docs/victoriametrics/vmstorage_common_flags.md && \
@@ -263,13 +264,13 @@ docs-update-vmctl-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmctl && \
-	./bin/vmctl -help > /tmp/vmctl_flags_tmp.md && \
-	./bin/vmctl opentsdb -help > /tmp/vmctl_opentsdb_flags_tmp.md && \
-	./bin/vmctl influx -help > /tmp/vmctl_influx_flags_tmp.md && \
-	./bin/vmctl remote-read -help > /tmp/vmctl_remote-read_flags_tmp.md && \
-	./bin/vmctl prometheus -help > /tmp/vmctl_prometheus_flags_tmp.md && \
-	./bin/vmctl vm-native -help > /tmp/vmctl_vm-native_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmctl)
+	(cd /tmp/vm-opensource && ./bin/vmctl -help > /tmp/vmctl_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl opentsdb -help > /tmp/vmctl_opentsdb_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl influx -help > /tmp/vmctl_influx_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl remote-read -help > /tmp/vmctl_remote-read_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl prometheus -help > /tmp/vmctl_prometheus_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl vm-native -help > /tmp/vmctl_vm-native_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_flags.md && \
 	cat /tmp/vmctl_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_flags.md && \
@@ -306,6 +307,10 @@ endif
 	# remove Version line and the actual version line from vmctl_flags.md to reduce diffs noise
 	sed -i '/^VERSION:/,+1d' docs/victoriametrics/vmctl/vmctl_flags.md
 
+	# adjust flags with dynamic default values
+	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
+	sed -i '/prom-tmp-dir-path/ s|(default: "[^"]*")|(default: os.TempDir())|' docs/victoriametrics/vmctl/vmctl_prometheus_flags.md
+
 # docs-update-flags updates flags in the documentation
 # using the actual binaries compiled from the provided $TAG.
 # The command also normalizes the output a bit.
@@ -324,12 +329,27 @@ endif
 	# Install using: brew install gnu-sed gawk
 	# Add tools to PATH see how in `brew info gnu-sed` and `brew info gawk
 
-	orig_branch=$$(git rev-parse --abbrev-ref HEAD); \
-	$(MAKE) docs-update-vmctl-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmsingle-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmalert-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmauth-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmagent-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmselect-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vminsert-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmstorage-flags && git checkout "$$orig_branch"
+	rm -rf /tmp/vm-enterprise
+	git worktree remove /tmp/vm-enterprise || true
+	git worktree add /tmp/vm-enterprise $(TAG)-enterprise
+
+	rm -rf /tmp/vm-enterprise-cluster
+	git worktree remove /tmp/vm-enterprise-cluster || true
+	git worktree add /tmp/vm-enterprise-cluster $(TAG)-enterprise-cluster
+
+	rm -rf /tmp/vm-opensource
+	git worktree remove /tmp/vm-opensource || true
+	git worktree add /tmp/vm-opensource $(TAG)
+
+	rm -rf /tmp/vm-opensource-cluster
+	git worktree remove /tmp/vm-opensource-cluster || true
+	git worktree add /tmp/vm-opensource-cluster $(TAG)-cluster
+
+	$(MAKE) docs-update-vmctl-flags
+	$(MAKE) docs-update-vmsingle-flags
+	$(MAKE) docs-update-vmalert-flags
+	$(MAKE) docs-update-vmauth-flags
+	$(MAKE) docs-update-vmagent-flags
+	$(MAKE) docs-update-vmselect-flags
+	$(MAKE) docs-update-vminsert-flags
+	$(MAKE) docs-update-vmstorage-flags

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,15 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+## v1.29.4
+Released: 2026-05-15
+
+- IMPROVEMENT: Optimized [BacktestingScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) exact mode for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) (which are also used in [UI](https://docs.victoriametrics.com/anomaly-detection/ui/)-triggered tasks), leading to 2-200x faster results and more responsive UI interactions.
+
+- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.6.1](https://docs.victoriametrics.com/anomaly-detection/ui/#v161) to [v1.7.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v170), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v170) for details. Notable changes include iterative anomaly detection with progress bar, model inference speedups, improved AI Copilot UX and various bug fixes.
+
+- BUGFIX: Made model-level `queries` and `schedulers` attachment semantics explicit and safer. Omitting these args still keeps the legacy default of attaching a model to all queries or schedulers with a warning, while `queries: null`, `queries: []`, `schedulers: null`, and `schedulers: []` now mean the model is intentionally unattached and will be dropped from active execution. Use `queries: all` or `schedulers: all` to request all attachments explicitly. Orphan models, queries, and schedulers are logged and removed during config validation, including hot reload.
+
 ## v1.29.3
 Released: 2026-04-16
 

docs/anomaly-detection/FAQ.md

@@ -423,7 +423,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     # ...
     restart: always
     volumes:
@@ -641,7 +641,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.3 && docker image tag victoriametrics/vmanomaly:v1.29.3 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.29.4 && docker image tag victoriametrics/vmanomaly:v1.29.4 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/QuickStart.md

@@ -122,7 +122,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.3
+docker pull victoriametrics/vmanomaly:v1.29.4
 ```
 
 2. Create the license file with your license key.
@@ -142,7 +142,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.3 \
+    victoriametrics/vmanomaly:v1.29.4 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -159,7 +159,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.3 \
+    victoriametrics/vmanomaly:v1.29.4 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -172,7 +172,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     # ...
     restart: always
     volumes:

docs/anomaly-detection/UI.md

@@ -315,7 +315,7 @@ docker run -it --rm \
   -e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
   -p 8080:8080 \
   -p 8490:8490 \
-  victoriametrics/vmanomaly:v1.29.3 \
+  victoriametrics/vmanomaly:v1.29.4 \
   vmanomaly_config.yaml
 ```
 
@@ -640,6 +640,26 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
+### v1.7.0
+Released: 2026-05-15
+
+vmanomaly version: [v1.29.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1294)
+
+- FEATURE: Improved [AI Copilot](#ai-assistance) to 
+  - support textual attachments, such as configuration files, query examples, model outputs, etc.
+  - operate with current time when suggesting time range changes, e.g. "current month" or "from the beginning of the last month until now"
+- FEATURE: improve infer jobs scheduled from UI to
+  - run 2-100x faster for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) depending on a configuration vs [1.6.1](#v161) timings
+  - show stage-aware progress bar, e.g. "getting data", "fitting model", "inferring on chunk".
+- IMPROVEMENT: "Advanced Options" design is improved, section is collapsed by default to save vertical space, yet can be encoded as always open in UI state URL.
+- IMPROVEMENT: dropdowns in UI are now searcheable and constrained in size for better UX, especially when many options are available (e.g. models list, tenants list, etc.).
+- BUGFIX: in generated config ("Show Config" menu)
+  - special YAML values (like `-.inf`, `.inf`, `.nan`) are now properly quoted and formatted to avoid JSON converting issues when copied to Kubernetes CRs, which previously lead to rejection of the manifest before vmanomaly can consume it.
+  - with Data Source selected as "Logs/Traces" `reader.class` is now correctly set to `vlogs` (previously `vm`) in generated config, when "model only" toggle is turned off.
+- BUGFIX: now "Hide Common Labels" toggle in "Table View" works correctly and does not show common labels in the legend columns when turned on.
+- BUGFIX: improved dropdowns for better accessibility to be searcheable and constrained in size.
+- BUGFIX: "Prettify query" now works correctly for "Metrics" datasources.
+
 ### v1.6.1
 Released: 2026-04-16
 

docs/anomaly-detection/components/models.md

@@ -1265,7 +1265,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.2
+docker pull victoriametrics/vmanomaly:v1.29.4
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -1279,7 +1279,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.29.2 /config.yaml \
+victoriametrics/vmanomaly:v1.29.4 /config.yaml \
 --licenseFile=/license
 --watch
 ```

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -395,7 +395,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -240,23 +240,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -308,7 +308,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.142.0
+    image: docker.io/victoriametrics/vmauth:v1.143.0
     ports:
       - 8427:8427
     volumes:

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.142.0-enterprise
+    image: victoriametrics/vmauth:v1.143.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - victoriametrics
       - alertmanager

docs/victoriametrics/Articles.md

@@ -33,6 +33,7 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [Percona: How do We Keep Metrics for a Long Time in VictoriaMetrics](https://www.youtube.com/watch?v=SGZjY7xgDwE)
 * [Miro: Prometheus High Availability and Fault Tolerance strategy, long term storage with VictoriaMetrics](https://medium.com/miro-engineering/prometheus-high-availability-and-fault-tolerance-strategy-long-term-storage-with-victoriametrics-82f6f3f0409e)
 * [ZERODHA: Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
+* [ZERODHA: Monitoring K8s with Victoriametrics](https://www.youtube.com/watch?v=ZJQYW-cFOms)
 * [Criteo: VictoriaMetrics, a stress-free Prometheus Remote Storage for 1 Billion metrics](https://medium.com/criteo-engineering/victoriametrics-a-prometheus-remote-storage-solution-57081a3d8e61)
 * [Abios Gaming: Choosing a Time Series Database for High Cardinality Aggregations](https://abiosgaming.com/press/high-cardinality-aggregations/)
 * [Cybozu: Monitoring Kubernetes clusters with VictoriaMetrics and Grafana](https://blog.cybozu.io/entry/2021/03/18/115743)

docs/victoriametrics/Quick-Start.md

@@ -58,9 +58,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.142.0
+docker pull victoriametrics/victoria-metrics:v1.143.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.142.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.143.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,16 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
+
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. Thanks to @alexei38 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918).
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
+* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
+
 ## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)
 
 Released at 2026-05-08
@@ -39,6 +49,7 @@ Released at 2026-05-08
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
 * FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Kafka (Enterprise)` row with panels for monitoring traffic (bytes), messages in/out, producer and consumer errors. See [#10728](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10728).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support sending data to the configured `-remoteWrite.url` via [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol). See [#10929](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10929).
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly obtain `__meta_hetzner_hcloud_location` and `__meta_hetzner_hcloud_location_network_zone` labels for [hetzner_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs). Hetzner changed discovery [API response](https://docs.hetzner.cloud/changelog#2025-12-16-phasing-out-datacenters) and returns `location` information from different field. See [#10909](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10909). Thanks to @juliusrickert for contribution.
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly discover addresses of storage nodes with [enterprise automatic-vmstorage-discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery). Bug was introduced at [v1.141.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11410).
@@ -209,6 +220,23 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.9](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.9)
+
+Released at 2026-05-08
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* SECURITY: upgrade Go builder from Go1.26.2 to Go1.26.3. See [the list of issues addressed in Go1.26.3](https://github.com/golang/go/issues?q=milestone%3AGo1.26.3%20label%3ACherryPickApproved).
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly obtain `__meta_hetzner_hcloud_location` and `__meta_hetzner_hcloud_location_network_zone` labels for [hetzner_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs). Hetzner changed discovery [API response](https://docs.hetzner.cloud/changelog#2025-12-16-phasing-out-datacenters) and returns `location` information from different field. See [#10909](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10909). Thanks to @juliusrickert for contribution.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): now correctly respects disabling retries via `-maxRequestBodySizeToRetry=0`. Previously, disabling retries required setting both `-maxRequestBodySizeToRetry` and `-requestBufferSize` to `0`. See [#10857](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10857). Thanks to @andriibeee for the contribution.
+* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): explicitly set the MD5 checksum for `DeleteObjects` requests. Starting with [aws-sdk-go-v2/service/s3 v1.73.0](https://github.com/aws/aws-sdk-go-v2/blob/release-2025-01-15/service/s3/CHANGELOG.md#v1730-2025-01-15), the SDK switched the checksum algorithm from MD5 to CRC32, which [can break compatibility](https://github.com/aws/aws-sdk-go-v2/discussions/2960) with some third-party S3 implementations, including [Dell ECS](https://www.dell.com/en-us/lp/dt/elastic-cloud-storage). See [#10907](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10907).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): attempt to route requests to the first backend when all backends are marked as broken. Previously, `vmauth` returned an error immediately without attempting any backend. This change may improve success rate in rare edge cases. See [#10837](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10837).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): give priority to `extra_label` and `extra_filters[]` params defined in URL query string over those defined in request body form, to properly respect constraints imposed by `vmauth`. See [#10908](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10908).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): set CORS headers on `/api/v1/export`, `/api/v1/export/csv` and `/api/v1/export/native`. See [#10899](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10899). Thanks to @andriibeee for the contribution.
+
 ## [v1.136.8](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.8)
 
 Released at 2026-04-30
@@ -513,6 +541,20 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.22](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.22)
+
+Released at 2026-05-08
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* SECURITY: upgrade Go builder from Go1.26.2 to Go1.26.3. See [the list of issues addressed in Go1.26.3](https://github.com/golang/go/issues?q=milestone%3AGo1.26.3%20label%3ACherryPickApproved).
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly obtain `__meta_hetzner_hcloud_location` and `__meta_hetzner_hcloud_location_network_zone` labels for [hetzner_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs). Hetzner changed discovery [API response](https://docs.hetzner.cloud/changelog#2025-12-16-phasing-out-datacenters) and returns `location` information from different field. See [#10909](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10909). Thanks to @juliusrickert for contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): give priority to `extra_label` and `extra_filters[]` params defined in URL query string over those defined in request body form, to properly respect constraints imposed by `vmauth`. See [#10908](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10908).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): set CORS headers on `/api/v1/export`, `/api/v1/export/csv` and `/api/v1/export/native`. See [#10899](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10899). Thanks to @andriibeee for the contribution.
+
 ## [v1.122.21](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.21)
 
 Released at 2026-04-24

docs/victoriametrics/enterprise.md

@@ -15,25 +15,28 @@ aliases:
 - /enterprise/
 ---
 
-VictoriaMetrics and [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) components are provided
+VictoriaMetrics, [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) components are provided
 in two kinds - [Community edition](https://victoriametrics.com/products/open-source/) and [Enterprise edition](https://victoriametrics.com/products/enterprise/).
 
-VictoriaMetrics and VictoriaLogs community components are open source and are free to use:
+The community components are open source and are free to use:
 
 - See [VictoriaMetrics source code](https://github.com/VictoriaMetrics/VictoriaMetrics/) and [VictoriaMetrics license](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE).
 - See [VictoriaLogs source code](https://github.com/VictoriaMetrics/VictoriaLogs/) and [VictoriaLogs license](https://github.com/VictoriaMetrics/VictoriaLogs/blob/master/LICENSE).
+- See [VictoriaTraces source code](https://github.com/VictoriaMetrics/VictoriaTraces/) and [VictoriaLogs license](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/LICENSE).
 
-Enterprise components of VictoriaMetrics and VictoriaLogs are available at the following places:
+Both open-source and enterprise components are available at the following places:
 
-- Binary executables are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-  and [the release page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
+- Binary executables are available at the corresponding releases pages:
+  - [releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+  - [releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
+  - [releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest)
 - Docker images are available at [Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay](https://quay.io/organization/victoriametrics).
 
 Enterprise executables and Docker images have `enterprise` suffix in their names and tags.
 
 ## Valid cases for VictoriaMetrics Enterprise
 
-The use of Enterprise components of VictoriaMetrics and VictoriaLogs is permitted in the following cases:
+The use of Enterprise components is permitted in the following cases:
 
 - Evaluation use in non-production setups. Please, request a [trial license](https://victoriametrics.com/products/enterprise/trial/)
   and then pass it via `-license` or `-licenseFile` command-line flags as described [in these docs](#running-victoriametrics-enterprise).
@@ -104,25 +107,26 @@ Contact us via [this page](https://victoriametrics.com/products/enterprise/) if
 
 ## Running VictoriaMetrics Enterprise
 
-Enterprise components of VictoriaMetrics and VictoriaLogs are available in the following forms:
+Enterprise components are available in the following forms:
 
-- [Binary releases](#binary-releases)
-- [Docker images](#docker-images)
-- [Helm charts](#helm-charts)
-- [Kubernetes operator](#kubernetes-operator)
+- [Binary releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases)
+- [Docker images](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)
+- [Helm charts](https://docs.victoriametrics.com/victoriametrics/enterprise/#helm-charts)
+- [Kubernetes operator](https://docs.victoriametrics.com/victoriametrics/enterprise/#kubernetes-operator)
 
 ### Binary releases
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
-Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz`.
+Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
+[the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
+and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
 
-All the Enterprise components of VictoriaMetrics and VictoriaLogs require specifying the following command-line flags:
+All the Enterprise components require specifying the following command-line flags:
 
 - `-license` - this flag accepts VictoriaMetrics Enterprise license key, which can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/)
 - `-licenseFile` - this flag accepts a path to file with VictoriaMetrics Enterprise license key,
@@ -135,8 +139,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -148,26 +152,26 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 
 ### Docker images
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.142.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.143.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
-flag as described in the [binary-releases](#binary-releases) section.
+flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
 
 Enterprise license key can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/).
 
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.142.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.143.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.142.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.143.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -177,7 +181,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     ports:
       - 8428:8428
     volumes:
@@ -194,12 +198,12 @@ The example assumes that the license file is stored at `/vm-license` on the host
 
 ### Helm charts
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in cases [listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Helm charts for Enterprise components are available in our VictoriaMetrics [Helm repository](https://github.com/VictoriaMetrics/helm-charts).
 
 In order to run Enterprise helm chart it is required to provide the license key via `license` value in `values.yaml` file
-and adjust the image tag to the Enterprise one as described in the [docker-images](#docker-images) section.
+and adjust the image tag to the Enterprise one as described in the [docker-images](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images) section.
 
 Enterprise license key can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/).
 
@@ -209,7 +213,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -220,7 +224,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 
 license:
   secret:
@@ -250,7 +254,7 @@ Note that the license key provided by using secret is mounted in a file. This al
 
 ### Kubernetes operator
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Enterprise components can be deployed via [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/).
 In order to use Enterprise components it is required to provide the license key via `license` field and adjust the image tag to the enterprise one.
@@ -270,7 +274,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -287,7 +291,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 ```
 
 Example secret with license key:
@@ -313,7 +317,7 @@ See the full list of the CRD specifications in the [Operator API](https://docs.v
 
 ### Updating the license key
 
-Updating the license key for VictoriaMetrics and VictoriaLogs Enterprise components depends on the way
+Updating the license key for Enterprise components depends on the way
 the license key is provided to the component:
 - If the license key is provided via `-license` command-line flag, then the component should be restarted
   with the new license key.
@@ -338,7 +342,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz`
 
 Includes:
 
@@ -347,11 +351,11 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.142.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.143.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## Monitoring license expiration
 
-All the VictoriaMetrics and VictoriaLogs Enterprise components expose the following metrics at the `/metrics` page:
+All the Enterprise components expose the following metrics at the `/metrics` page:
 
 - `vm_license_expires_at` - license expiration date in unix timestamp format
 - `vm_license_expires_in_seconds` - the number of seconds left until the license expires

docs/victoriametrics/goals.md

@@ -12,21 +12,21 @@ aliases:
 ---
 ## Goals
 
-1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics and VictoriaLogs component in the most efficient way**.
+1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics products in the most efficient way**.
 1. Fixing bugs in the essential functionality of VictoriaMetrics components. Small usability bugs are usually the most annoying,
    so they **must be fixed first**. Bugs, which affect a small number of users at some rare edge cases, can be fixed later.
-1. Improving [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) for VictoriaMetrics components,
-   so users could find answers to their questions via Google or [Perplexity](https://www.perplexity.ai/) without the need
+1. Improving [public docs for VictoriaMetrics products](https://docs.victoriametrics.com).
+   so users could find answers to their questions via Google or any other AI-powered web search without the need
    to ask these questions at our [support channels](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#community-and-contributions).
-1. Simplifying usage of VictoriaMetrics components without breaking backwards compatibility, so users could regularly
-   upgrade to [the latest available release](https://docs.victoriametrics.com/victoriametrics/changelog/) and remain happy.
-1. Improving usability for the existing functionality of VictoriaMetrics components.
+1. Simplifying usage of VictoriaMetrics products without breaking backwards compatibility, so users could regularly
+   upgrade to [the latest available release](https://docs.victoriametrics.com/victoriametrics/changelog/) and become happier.
+1. Improving the usability for the existing functionality of VictoriaMetrics components.
 1. Improving the readability and maintainability of the code base by removing unnecessary abstractions and simplifying the code whenever possible.
 1. Improving development velocity by optimizing and simplifying CI/CD tasks, so they take less time to execute and debug.
 
 ## Non-goals
 
-1. Convincing people to use VictoriaMetrics components when there are better suited solutions exist for their tasks,
+1. Convincing people to use VictoriaMetrics products when there are better suited solutions exist for their tasks,
    since users will become angry at VictoriaMetrics after they discover better solutions.
 1. Breaking links to [VictoriaMetrics docs](https://docs.victoriametrics.com/victoriametrics/), since users will be unhappy seeing 404 page
    or unexpected results after they click some old link somewhere on the Internet or in the internal knowledge base.
@@ -35,25 +35,29 @@ aliases:
    since these features may break the essential functionality of VictoriaMetrics components, so a big share
    of the existing users may become unhappy after the upgrade.
 1. Adding unnecessary abstractions, since they may worsen project maintainability in the future.
-1. Implementing all the features users ask. These features should fit [the goals](#goals) of VictoriaMetrics.
+1. Implementing all the features users ask. These features should fit [the goals](https://docs.victoriametrics.com/victoriametrics/goals/#goals) of VictoriaMetrics.
    Other feature requests must be closed as `won't implement`, with the link to this page.
-1. Merging all the pull requests users submit. These pull requests should fit [the goals](#goals) of VictoriaMetrics.
+1. Merging all the pull requests users submit. These pull requests should fit [the goals](https://docs.victoriametrics.com/victoriametrics/goals/#goals) of VictoriaMetrics.
    Other pull requests must be closed as `won't merge`, with the link to this page.
 1. Slowing down and complicating CI/CD pipelines with non-essential tasks, since this results in development velocity slowdown.
 1. Introducing non-essential requirements, since this slows down development velocity.
 
 ## VictoriaMetrics proverbs
 
-- **Small usability fix is better than non-trivial feature.** Usability fix makes happy existing users.
-  Non-trivial feature may make happy some new users, while it may make upset a big share of existing users
-  if the feature breaks some essential functionality of VictoriaMetrics components or makes it less efficient.
+- **A small usability improvement is more valuable than a major new feature.**
+  The usability fix makes happy existing users. Non-trivial feature may make happy some new users,
+  while it may make upset a big share of existing users if the feature breaks the essential functionality
+  of VictoriaMetrics products or makes it less efficient.
 
-- **Good docs are better than new shiny feature.** Good docs help users discovering new functionality and dealing
-  with VictoriaMetrics components in the most efficient way. Nobody uses new shiny feature if it isn't documented properly.
+- **Having clean and concise documentation is more valuable than having a lot of great but undocumented features.**
+  Good docs help users discovering new functionality and dealing with VictoriaMetrics products in the most efficient way.
+  Nobody uses new shiny features if they aren't documented properly.
 
-- **Happy users are more important than the momentary revenue.** Happy users spread the word about VictoriaMetrics,
-  so more people convert to VictoriaMetrics users. Happy users are eager to become happy [customers](https://docs.victoriametrics.com/victoriametrics/enterprise/).
-  This increases long-term revenue.
+- **A simple solution is better than a smart one.**
+  The simple solution is easier to setup, operate, debug and troubleshoot than the smart solution. This saves users' time, costs and nerve cells.
 
-- **Simple solution is better than smart solution.** Simple solution is easier to setup, operate, debug and troubleshoot than the smart solution.
-  This saves users' time, costs and nerve cells.
+- **Happy users are more important than the short-term profit.**
+  Happy users spread the word about VictoriaMetrics, so more people convert to VictoriaMetrics users.
+  Happy users are eager to become happy [customers](https://docs.victoriametrics.com/victoriametrics/enterprise/)
+  over time. This increases long-term profit.
+  Upset users may be forced to become customers, but they will constantily search for the opportunity to switch to competing solutions.

docs/victoriametrics/integrations/opentelemetry.md

@@ -23,6 +23,8 @@ The following label sanitization options can be enabled:
   For example, `process.cpu.time{service.name="foo"}` is converted to `process_cpu_time_seconds_total{service_name="foo"}`.
 * `-opentelemetry.convertMetricNamesToPrometheus` - converts **only metric names** according to [OTLP Metric points to Prometheus specification](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.33.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus) for metrics ingested via OTLP.
   For example, `process.cpu.time{service.name="foo"}` is converted to `process_cpu_time_seconds_total{service.name="foo"}`. See more about this use case [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9830).
+* `-opentelemetry.labelNameUnderscoreSanitization` - controls whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. Reserved labels starting with `__` are not modified.
+  For example, `_mylabel` is converted to `key_mylabel`.
 
 > These flags can be applied on vmagent, vminsert or VictoriaMetrics single-node.
 

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/victoria-metrics-linux-amd64-v1.142.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.142.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/victoria-metrics-linux-amd64-v1.143.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.143.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/victoria-metrics-linux-amd64-v1.142.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.142.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/victoria-metrics-linux-amd64-v1.143.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.143.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/vmagent.md

@@ -483,15 +483,29 @@ by specifying `-remoteWrite.forcePromProto` command-line flag for the correspond
 
 By default `vmagent` collects the data without [tenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) identifiers
 and routes it to the remote storage specified via `-remoteWrite.url` command-line flag. The `-remoteWrite.url` can point to `/insert/<tenant_id>/prometheus/api/v1/write` path
-at `vminsert` according to [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format). In this case all the metrics are written to the given `<tenant_id>` tenant.
+at `vminsert` according to [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format). 
+```mermaid
+flowchart LR
+    A["requests_total{instance=foo}"] --> |/api/v1/write| V[vmagent]
+    B["requests_total{instance=bar}"] <--> |scrape| V
+    V --> |"/insert/#60;tenant_id#62;/#60;suffix#62;"| C[vminsert]
+```
+In this case, all the metrics written to `/insert/tenant_id/prometheus/api/v1/write` will belong to specified `<tenant_id>` tenant.
+
+### Multitenancy via labels
+
+vmagent can write data to multiple distinct tenants if `-remoteWrite.url` points to [multitenant url at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
+and tenant is specified via [multitenancy labels](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels):
+```mermaid
+flowchart LR
+    A["requests_total{instance=foo, vm_account_id=0}"] --> |/api/v1/write| V[vmagent]
+    B["requests_total{instance=bar, vm_account_id=1}"] <--> |scrape| V
+    V --> |"/insert/multitenant/#60;suffix#62;"| C[vminsert]
+```
+`<tenant_id>` is extracted from the `vm_account_id` and `vm_project_id` labels.
 
-The easiest way to write data to multiple distinct tenants is to specify the needed tenants via `vm_account_id` and `vm_project_id` labels
-and then to push metrics with these labels to [multitenant url at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels).
 The `vm_account_id` and `vm_project_id` labels can be specified via [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) before sending the metrics to `-remoteWrite.url`.
-
-For example, the following relabeling rule instructs sending metrics to `<account_id>:0` [tenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
-defined in the `prometheus.io/account_id` annotation of Kubernetes pod deployment:
-
+For example, the following relabeling rule instructs sending metrics to `<account_id>:0` tenant defined in the `prometheus.io/account_id` annotation of Kubernetes pod deployment:
 ```yaml
 scrape_configs:
 - kubernetes_sd_configs:
@@ -501,12 +515,10 @@ scrape_configs:
     target_label: vm_account_id
 ```
 
-In addition, vmagent could obtain tenant identifier from `__tenant_id__` label at target discovery phase.
+vmagent can get tenant identifier from `__tenant_id__` label at target discovery phase.
 It implicitly converts `__tenant_id__` label into `vm_account_id` and `vm_project_id` labels and attaches
 it to the scraped metrics and metrics metadata.
-For example, the following relabeling rule instructs sending metrics to `10:5` [tenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
-defined in the `prometheus.io/tenant_id: 10:5` annotation of Kubernetes pod deployment:
-
+For example, the following relabeling rule instructs sending metrics to `10:5` tenant defined in the `prometheus.io/tenant_id: 10:5` annotation of Kubernetes pod deployment:
 ```yaml
 scrape_configs:
 - kubernetes_sd_configs:
@@ -516,15 +528,45 @@ scrape_configs:
     target_label: __tenant_id__
 ```
 
-`vmagent` can accept data via the same multitenant endpoints (`/insert/<accountID>/<suffix>`) as `vminsert` at [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/)
-does according to [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format) if `-enableMultitenantHandlers` command-line flag is set.
-In this case, vmagent automatically converts tenant identifiers from the URL or headers to `vm_account_id` and `vm_project_id` labels and sets tenant info in metadata.
-These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via `-remoteWrite.relabelConfig`
-and `-remoteWrite.urlRelabelConfig` command-line flags. Metrics with `vm_account_id` and `vm_project_id` labels can be routed to the corresponding tenants
-when specifying `-remoteWrite.url` to [multitenant url at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels).
+vmagent can [enforce adding labels](https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics) to all scraped
+or forwarded metrics. 
 
-`vmagent` can accept tenant IDs specified via HTTP headers if both `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are set. 
-See more about [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers).
+### Multitenancy via path
+
+vmagent can write data to multiple distinct tenants if `-remoteWrite.url` points to [multitenant url at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels),
+tenant is specified in the [write path](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format) and `-enableMultitenantHandlers` command-line flag is set:
+```mermaid
+flowchart LR
+    A["requests_total{instance=foo}"] --> |/insert/0/#60;suffix#62;| V[vmagent]
+    B["requests_total{instance=bar}"] --> |/insert/1/#60;suffix#62;| V
+    V --> |"/insert/multitenant/#60;suffix#62;"| C[vminsert]
+```
+
+In this configuration, vmagent accepts writes via the same multitenant endpoints (`/insert/<accountID>/<suffix>`) [as vminsert](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format).
+For all received data, vmagent will automatically convert tenant identifiers from the URL to `vm_account_id` and `vm_project_id` labels and sets tenant info in metadata.
+These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via `-remoteWrite.relabelConfig`
+and `-remoteWrite.urlRelabelConfig` command-line flags. 
+
+### Multitenancy via headers
+
+vmagent can write data to multiple distinct tenants if `-remoteWrite.url` points to [multitenant url at VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels),
+tenant is specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}, both `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are set:
+```mermaid
+flowchart LR
+    A["requests_total{instance=foo}"] --> |/insert/#60;suffix#62; <br>--header AccountID: 0| V[vmagent]
+    B["requests_total{instance=bar}"] --> |/insert/#60;suffix#62; <br>--header AccountID: 1| V
+    V --> |"/insert/multitenant/#60;suffix#62;"| C[vminsert]
+```
+
+In this configuration, vmagent accepts writes via the same simplified multitenant endpoints (`/insert/<suffix>`) [as vminsert](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format).
+The tenant information is extracted from the `AccountID` and `ProjectID` HTTP headers, that are expected to be attached to all the incoming requests. If headers are missing, then tenant is set to `0:0` as default.
+
+For all received data, vmagent will automatically convert tenant identifiers from the headers to `vm_account_id` and `vm_project_id` labels and sets tenant info in metadata.
+These tenant labels are added before applying [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/) specified via `-remoteWrite.relabelConfig`
+and `-remoteWrite.urlRelabelConfig` command-line flags.
+
+vmauth can [enforce adding headers](https://docs.victoriametrics.com/victoriametrics/vmauth/#modifying-http-headers) to all
+forwarded requests via `headers` param in the config file.
 
 ## Adding labels to metrics
 

docs/victoriametrics/vmagent_common_flags.md

@@ -383,6 +383,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Optional basic auth username to use for the corresponding -remoteWrite.url
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
+  -remoteWrite.basicAuth.usernameFile array
+     Optional path to basic auth username to use for the corresponding -remoteWrite.url. The file is re-read every second
+     Supports an array of values separated by comma or specified via multiple flags.
+     Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -remoteWrite.bearerToken array
      Optional bearer auth token to use for the corresponding -remoteWrite.url
      Supports an array of values separated by comma or specified via multiple flags.

docs/victoriametrics/vmalert.md

@@ -17,8 +17,7 @@ aliases:
 or [recording](https://docs.victoriametrics.com/victoriametrics/vmalert/#recording-rules)
 rules against configured `-datasource.url`. For sending alerting notifications
 `vmalert` relies on [Alertmanager](https://github.com/prometheus/alertmanager) configured via `-notifier.url` flag.
-Recording rules results are persisted via [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
-protocol and require `-remoteWrite.url` to be configured.
+Recording rules results are persisted via remote write protocols and require `-remoteWrite.url` to be configured.
 `vmalert` is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
 implementation and aims to be compatible with its syntax.
 
@@ -68,7 +67,7 @@ To start using `vmalert` you will need the following things:
 * notifier address [optional] - reachable [Alert Manager](https://github.com/prometheus/alertmanager) instance for processing,
   aggregating alerts, and sending notifications. Please note, notifier address also supports Consul and DNS Service Discovery via
   [config file](https://docs.victoriametrics.com/victoriametrics/vmalert/#notifier-configuration-file).
-* remote write address [optional] - [remote write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+* remote write address [optional] - [remote write](https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol)
   compatible storage to persist rules and alerts state info. To persist results to multiple destinations use vmagent
   configured with multiple remote writes as a proxy;
 * remote read address [optional] - MetricsQL compatible datasource to restore alerts state from.
@@ -848,7 +847,7 @@ If you want rules to run concurrently based on the `concurrency` setting, set `-
 
 vmalert sends rule's expression to [/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) endpoint
 of the configured `-datasource.url`. Returned data is then processed according to the rule type and
-backfilled to `-remoteWrite.url` via [remote Write protocol](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations).
+backfilled to `-remoteWrite.url` via [remote Write protocol](https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol).
 vmalert respects `evaluationInterval` value set by flag or per-group during the replay.
 vmalert automatically disables caching on VictoriaMetrics side by sending `nocache=1` param. It allows
 to prevent cache pollution and unwanted time range boundaries adjustment during backfilling.

docs/victoriametrics/vmalert_common_flags.md

@@ -25,6 +25,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
      Optional path to basic auth password to use for -datasource.url
   -datasource.basicAuth.username string
      Optional basic auth username for -datasource.url
+  -datasource.basicAuth.usernameFile string
+     Optional path to basic auth username to use for -datasource.url
   -datasource.bearerToken string
      Optional bearer auth token to use for -datasource.url.
   -datasource.bearerTokenFile string
@@ -186,6 +188,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
      Optional basic auth username for -notifier.url
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
+  -notifier.basicAuth.usernameFile array
+     Optional path to basic auth username file for -notifier.url
+     Supports an array of values separated by comma or specified via multiple flags.
+     Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -notifier.bearerToken array
      Optional bearer token for -notifier.url
      Supports an array of values separated by comma or specified via multiple flags.
@@ -298,6 +304,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
      Optional path to basic auth password to use for -remoteRead.url
   -remoteRead.basicAuth.username string
      Optional basic auth username for -remoteRead.url
+  -remoteRead.basicAuth.usernameFile string
+     Optional path to basic auth username to use for -remoteRead.url
   -remoteRead.bearerToken string
      Optional bearer auth token to use for -remoteRead.url.
   -remoteRead.bearerTokenFile string
@@ -342,6 +350,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
      Optional path to basic auth password to use for -remoteWrite.url
   -remoteWrite.basicAuth.username string
      Optional basic auth username for -remoteWrite.url
+  -remoteWrite.basicAuth.usernameFile string
+     Optional path to basic auth username to use for -remoteWrite.url
   -remoteWrite.bearerToken string
      Optional bearer auth token to use for -remoteWrite.url.
   -remoteWrite.bearerTokenFile string
@@ -391,7 +401,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -remoteWrite.tlsServerName string
      Optional TLS server name to use for connections to -remoteWrite.url. By default, the server name from -remoteWrite.url is used
   -remoteWrite.url string
-     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
+     Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int

docs/victoriametrics/vmauth.md

@@ -1311,7 +1311,7 @@ Access logs contain limited information to prevent exposing sensitive data. See
 
 The printed log starts with `access_log` prefix and is followed with `request_host`, `request_uri`, `status_code`, `remote_addr`,
 `user_agent`, `referer`, `duration_ms` and `username` fields in [logfmt](https://brandur.org/logfmt) format. Such logs can be later
-analyzed in [VictoriaLogs](https://docs.victoriametrics.com/victorialogs):
+analyzed in [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
 ```logsql
 access_log | extract 'access_log <access_log>' | unpack_logfmt from access_log
 | stats by(username, request_host, status_code) count()

docs/victoriametrics/vmauth_common_flags.md

@@ -132,7 +132,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
   -maxQueueDuration duration
      The maximum duration to wait before rejecting incoming requests if concurrency limit specified via -maxConcurrentRequests or -maxConcurrentPerUserRequests command-line flags is reached. Requests are rejected with '429 Too Many Requests' http status code if the limit is still reached after the -maxQueueDuration duration. This allows graceful handling of short spikes in concurrent requests. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting (default 10s)
   -maxRequestBodySizeToRetry size
-     The maximum request body size to buffer in memory for potential retries at other backends. Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. See also -requestBufferSize
+     The maximum request body size to buffer in memory for potential retries at other backends. Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables retries. See also -requestBufferSize
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16384)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage

docs/victoriametrics/vmbackup.md

@@ -75,7 +75,7 @@ Full backups can be accelerated with `-origin` pointing to an already existing b
 data between the existing backup and new backup. It saves time and costs on data transfer.
 
 All commands below are provided for a single node version.
-For a cluster version `vmbackup` should be executed on each vmstorage node and `-snapshot.createUrl` should point to vmstorage:
+For a cluster version `vmbackup` should be executed on each vmstorage node and `-snapshot.createURL` should point to vmstorage:
 ```sh
 ./vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage1:8482/snapshot/create -dst=gs://<bucket>/vmstorage-1
 ```

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/vmutils-darwin-arm64-v1.142.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/vmutils-darwin-arm64-v1.143.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.142.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.143.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

docs/victoriametrics/vmctl/vmctl_prometheus_flags.md

@@ -29,7 +29,7 @@ OPTIONS:
    --prom-filter-time-end value                                       The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
    --prom-filter-label value                                          Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
    --prom-filter-label-value value                                    Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
-   --prom-tmp-dir-path value                                          Path to directory to be used for temporary files. (default: "/var/folders/ds/3kj5p3v17ll0hsyvq380ryvm0000gn/T/")
+   --prom-tmp-dir-path value                                          Path to directory to be used for temporary files. (default: os.TempDir())
    --vm-addr value                                                    VictoriaMetrics address to perform import requests. 
       Should be the same as --httpListenAddr value for single-node version or vminsert component. 
       When importing into the clustered version do not forget to set additionally --vm-account-id flag. 

docs/victoriametrics/vmgateway.md

@@ -419,6 +419,8 @@ Below is the list of configuration flags (it can be viewed by running `./vmgatew
      Optional path to basic auth password to use for -datasource.url
   -datasource.basicAuth.username string
      Optional basic auth username for -datasource.url
+  -datasource.basicAuth.usernameFile string
+     Optional path to basic auth username to use for -datasource.url
   -datasource.bearerToken string
      Optional bearer auth token to use for -datasource.url.
   -datasource.bearerTokenFile string

lib/backup/fslocal/fslocal.go

@@ -33,6 +33,7 @@ type FS struct {
 //
 // The returned fs must be stopped when no long needed with MustStop call.
 func (fs *FS) Init() error {
+	fs.Dir = filepath.Clean(fs.Dir)
 	if fs.MaxBytesPerSecond > 0 {
 		fs.bl = newBandwidthLimiter(fs.MaxBytesPerSecond)
 	}

lib/backup/fslocal/fslocal_test.go

@@ -0,0 +1,30 @@
+package fslocal
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestFSListPartsWithTrailingSlashInDir(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "testfile"), []byte("x"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	// trailing slash must not cause ListParts to panic
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823
+	fs := &FS{Dir: dir + string(filepath.Separator)}
+	if err := fs.Init(); err != nil {
+		t.Fatalf("Init error: %s", err)
+	}
+	defer fs.MustStop()
+
+	parts, err := fs.ListParts()
+	if err != nil {
+		t.Fatalf("ListParts error: %s", err)
+	}
+	if len(parts) != 1 {
+		t.Fatalf("expected 1 part, got %d", len(parts))
+	}
+}

lib/memory/memory.go

@@ -12,7 +12,7 @@ import (
 
 var (
 	allowedPercent = flag.Float64("memory.allowedPercent", 60, `Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage`)
-	allowedBytes   = flagutil.NewBytes("memory.allowedBytes", 0, `Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage`)
+	allowedBytes   = flagutil.NewBytes("memory.allowedBytes", 0, `Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).`)
 )
 
 var _ = metrics.NewGauge("process_memory_limit_bytes", func() float64 {
@@ -45,6 +45,10 @@ func initOnce() {
 		logger.Infof("limiting caches to %d bytes, leaving %d bytes to the OS according to -memory.allowedPercent=%g, system memory limit %d bytes", allowedMemory, remainingMemory, *allowedPercent, memoryLimit)
 	} else {
 		allowedMemory = allowedBytes.IntN()
+		if allowedMemory < 1*1024*1024 {
+			// It's fair to print a hint if the allowedBytes is set to too small, typically by misconfiguration.
+			logger.Warnf("allowed memory %d bytes set by -memory.allowedBytes is low. The process may behave unexpectedly.", allowedMemory)
+		}
 		remainingMemory = memoryLimit - allowedMemory
 		if remainingMemory <= 0 {
 			logger.Fatalf("FATAL: remaining memory %d bytes cannot be less than or equal to zero, detected system memory limit %d bytes, -memory.allowedBytes=%s", remainingMemory, memoryLimit, allowedBytes.String())

lib/promscrape/scrapework.go

@@ -961,6 +961,7 @@ func getLabelsHash(labels []prompb.Label) uint64 {
 
 	for _, label := range labels {
 		b = append(b, label.Name...)
+		b = append(b, '=')
 		b = append(b, label.Value...)
 	}
 	h := xxhash.Sum64(b)

lib/promscrape/scrapework_test.go

@@ -700,7 +700,7 @@ func TestScrapeWorkScrapeInternalStreamConcurrency(t *testing.T) {
 		StreamParse:   true,
 		ScrapeTimeout: time.Second * 42,
 		SeriesLimit:   4000,
-	}, 3, 4015, 2, 50)
+	}, 3, 4012, 2, 50)
 }
 
 func TestScrapeWorkScrapeInternalWithMaxScrapeSize(t *testing.T) {

lib/protoparser/opentelemetry/stream/sanitize.go

@@ -16,6 +16,9 @@ var (
 		"via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/")
 	convertMetricNamesToPrometheus = flag.Bool("opentelemetry.convertMetricNamesToPrometheus", false, "Whether to convert only metric names into Prometheus-compatible format for the metrics ingested "+
 		"via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/")
+	labelNameUnderscoreSanitization = flag.Bool("opentelemetry.labelNameUnderscoreSanitization", true, "Whether to enable prepending of 'key' to labels starting with '_' "+
+		"when -opentelemetry.usePrometheusNaming is enabled. Reserved labels starting with '__' are not modified. "+
+		"See https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/")
 )
 
 // unitMap is obtained from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/b8655058501bed61a06bb660869051491f46840b/pkg/translator/prometheus/normalize_name.go#L19
@@ -80,8 +83,6 @@ func (sctx *sanitizerContext) reset() {
 	sctx.labelBuf = sctx.labelBuf[:0]
 }
 
-// See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/b8655058501bed61a06bb660869051491f46840b/pkg/translator/prometheus/normalize_label.go#L26
-//
 // The returned string is valid until the next call to sanitizeLabelName.
 func (sctx *sanitizerContext) sanitizeLabelName(labelName string) string {
 	if !*usePrometheusNaming {
@@ -90,6 +91,8 @@ func (sctx *sanitizerContext) sanitizeLabelName(labelName string) string {
 	return sctx.sanitizePrometheusLabelName(labelName)
 }
 
+// sanitizePrometheusLabelName performs conversion and normalization of OpenTelemetry attributes to Prometheus labels.
+// It follows the Prometheus guidelines: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/prometheus#labels
 func (sctx *sanitizerContext) sanitizePrometheusLabelName(labelName string) string {
 	if len(labelName) == 0 {
 		return ""
@@ -97,7 +100,7 @@ func (sctx *sanitizerContext) sanitizePrometheusLabelName(labelName string) stri
 	labelName = promrelabel.SanitizeLabelName(labelName)
 	if labelName[0] >= '0' && labelName[0] <= '9' {
 		return sctx.concatLabel("key_", labelName)
-	} else if strings.HasPrefix(labelName, "_") && !strings.HasPrefix(labelName, "__") {
+	} else if *labelNameUnderscoreSanitization && strings.HasPrefix(labelName, "_") && !strings.HasPrefix(labelName, "__") {
 		return sctx.concatLabel("key", labelName)
 	}
 	return labelName

lib/protoparser/opentelemetry/stream/sanitize_test.go

@@ -24,6 +24,19 @@ func TestSanitizePrometheusLabelName(t *testing.T) {
 	f("1foo", "key_1foo")
 	f("_foo", "key_foo")
 	f("__bar", "__bar")
+
+	prev := *labelNameUnderscoreSanitization
+	*labelNameUnderscoreSanitization = false
+	defer func() {
+		*labelNameUnderscoreSanitization = prev
+	}()
+
+	f("", "")
+	f("foo", "foo")
+	f("foo_bar", "foo_bar")
+	f("1foo", "key_1foo")
+	f("_foo", "_foo")
+	f("__bar", "__bar")
 }
 
 func TestSanitizePrometheusMetricName(t *testing.T) {

lib/streamaggr/deduplicator.go

@@ -12,7 +12,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/histogram"
 )
 
 // Deduplicator deduplicates samples per each time series.
@@ -29,9 +28,8 @@ type Deduplicator struct {
 	stopCh chan struct{}
 
 	ms *metrics.Set
-	// time to wait after interval end before flush
-	flushAfter   *histogram.Fast
-	muFlushAfter sync.Mutex
+	// flushAfterMsec is the max sample lag (in milliseconds) observed in the current flush interval.
+	flushAfterMsec atomic.Int64
 }
 
 // NewDeduplicator returns new deduplicator, which deduplicates samples per each time series.
@@ -59,7 +57,6 @@ func NewDeduplicator(pushFunc PushFunc, enableWindows bool, interval time.Durati
 	}
 	d.cs.Store(cs)
 	if enableWindows {
-		d.flushAfter = histogram.GetFast()
 		d.minDeadline.Store(startTime.UnixMilli())
 	}
 	d.cs.Store(cs)
@@ -145,9 +142,15 @@ func (d *Deduplicator) Push(tss []prompb.TimeSeries) {
 	}
 
 	if d.enableWindows && maxLagMsec > 0 {
-		d.muFlushAfter.Lock()
-		d.flushAfter.Update(float64(maxLagMsec))
-		d.muFlushAfter.Unlock()
+		for {
+			old := d.flushAfterMsec.Load()
+			if maxLagMsec <= old {
+				break
+			}
+			if d.flushAfterMsec.CompareAndSwap(old, maxLagMsec) {
+				break
+			}
+		}
 	}
 
 	if len(ctx.blue) > 0 {
@@ -172,7 +175,6 @@ func dropSeriesLabels(dst, src []prompb.Label, labelNames []string) []prompb.Lab
 
 func (d *Deduplicator) runFlusher(pushFunc PushFunc) {
 	t := time.NewTicker(d.interval)
-	var fa *histogram.Fast
 	defer t.Stop()
 	for {
 		select {
@@ -180,12 +182,7 @@ func (d *Deduplicator) runFlusher(pushFunc PushFunc) {
 			return
 		case <-t.C:
 			if d.enableWindows {
-				// Calculate delay and wait
-				d.muFlushAfter.Lock()
-				fa, d.flushAfter = d.flushAfter, histogram.GetFast()
-				d.muFlushAfter.Unlock()
-				delay := time.Duration(fa.Quantile(flushQuantile)) * time.Millisecond
-				histogram.PutFast(fa)
+				delay := time.Duration(d.flushAfterMsec.Swap(0)) * time.Millisecond
 				time.Sleep(delay)
 			}
 			d.flush(pushFunc)

lib/streamaggr/quantiles.go

@@ -1,9 +1,10 @@
 package streamaggr
 
 import (
+	"strconv"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/valyala/histogram"
-	"strconv"
 )
 
 // quantilesAggrValue calculates output=quantiles, e.g. the given quantiles over the input samples.
@@ -19,18 +20,19 @@ func (av *quantilesAggrValue) pushSample(_ aggrConfig, sample *pushSample, _ str
 }
 
 func (av *quantilesAggrValue) flush(c aggrConfig, ctx *flushCtx, key string, _ bool) {
-	ac := c.(*quantilesAggrConfig)
-	if av.h != nil {
-		ac.quantiles = av.h.Quantiles(ac.quantiles[:0], ac.phis)
-		histogram.PutFast(av.h)
-		av.h = nil
+	if av.h == nil {
+		return
 	}
-	if len(ac.quantiles) > 0 {
-		for i, quantile := range ac.quantiles {
-			ac.b = strconv.AppendFloat(ac.b[:0], ac.phis[i], 'g', -1, 64)
-			phiStr := bytesutil.InternBytes(ac.b)
-			ctx.appendSeriesWithExtraLabel(key, "quantiles", quantile, "quantile", phiStr)
-		}
+	ac := c.(*quantilesAggrConfig)
+	ac.quantiles = av.h.Quantiles(ac.quantiles[:0], ac.phis)
+	histogram.PutFast(av.h)
+	// reset h to avoid producing stale results on the next flush if av didn't get new pushSample() calls
+	av.h = nil
+
+	for i, quantile := range ac.quantiles {
+		ac.b = strconv.AppendFloat(ac.b[:0], ac.phis[i], 'g', -1, 64)
+		phiStr := bytesutil.InternBytes(ac.b)
+		ctx.appendSeriesWithExtraLabel(key, "quantiles", quantile, "quantile", phiStr)
 	}
 }
 

lib/streamaggr/streamaggr.go

@@ -24,14 +24,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/valyala/histogram"
 	"gopkg.in/yaml.v2"
 )
 
-// defines ingested samples lag quantile to determine a time to wait before flush.
-// It's not configurable at the moment.
-const flushQuantile = 0.95
-
 var supportedOutputs = []string{
 	"avg",
 	"count_samples",
@@ -440,9 +435,9 @@ type aggregator struct {
 	// aggrOutputs contains aggregate states for the given outputs
 	aggrOutputs *aggrOutputs
 
-	// time to wait after interval end before flush
-	flushAfter   *histogram.Fast
-	muFlushAfter sync.Mutex
+	// flushAfterMsec is the max sample lag (in milliseconds) observed in the current flush interval.
+	// It is used to properly delay the flush time while using aggregation windows.
+	flushAfterMsec atomic.Int64
 
 	// suffix contains a suffix, which should be added to aggregate metric names
 	//
@@ -705,9 +700,6 @@ func newAggregator(cfg *Config, path string, pushFunc PushFunc, ms *metrics.Set,
 			minTime = minTime.Add(interval)
 		}
 	}
-	if enableWindows {
-		a.flushAfter = histogram.GetFast()
-	}
 	a.minDeadline.Store(minTime.UnixMilli())
 	cs := &currentState{}
 	if a.dedupInterval > 0 {
@@ -834,16 +826,10 @@ func (a *aggregator) runFlusher(pushFunc PushFunc, alignFlushToInterval, skipFlu
 	t := time.NewTicker(interval)
 	defer t.Stop()
 
-	var fa *histogram.Fast
 	for tickerWait(t) {
 		pf := pushFunc
 		if a.enableWindows {
-			// Calculate delay and wait
-			a.muFlushAfter.Lock()
-			fa, a.flushAfter = a.flushAfter, histogram.GetFast()
-			a.muFlushAfter.Unlock()
-			delay := time.Duration(fa.Quantile(flushQuantile)) * time.Millisecond
-			histogram.PutFast(fa)
+			delay := time.Duration(a.flushAfterMsec.Swap(0)) * time.Millisecond
 			time.Sleep(delay)
 		}
 
@@ -1044,9 +1030,15 @@ func (a *aggregator) Push(tss []prompb.TimeSeries, matchIdxs []uint32) {
 		}
 	}
 	if enableWindows && maxLagMsec > 0 {
-		a.muFlushAfter.Lock()
-		a.flushAfter.Update(float64(maxLagMsec))
-		a.muFlushAfter.Unlock()
+		for {
+			old := a.flushAfterMsec.Load()
+			if maxLagMsec <= old {
+				break
+			}
+			if a.flushAfterMsec.CompareAndSwap(old, maxLagMsec) {
+				break
+			}
+		}
 	}
 	a.samplesLag.Update(float64(maxLagMsec) / 1_000)
 

lib/streamaggr/streamaggr_synctest_test.go

@@ -17,11 +17,10 @@ func TestAggregatorsSuccess(t *testing.T) {
 	f := func(inputMetrics []string, interval time.Duration, outputMetricsExpected, config, matchIdxsStrExpected string) {
 		t.Helper()
 
+		var matchIdxs []uint32
+		var tssOutput []prompb.TimeSeries
 		synctest.Test(t, func(t *testing.T) {
-			var matchIdxs []uint32
-			var tssOutput []prompb.TimeSeries
 			var tssOutputLock sync.Mutex
-
 			// Initialize Aggregators
 			pushFunc := func(tss []prompb.TimeSeries) {
 				tssOutputLock.Lock()
@@ -32,31 +31,31 @@ func TestAggregatorsSuccess(t *testing.T) {
 			if err != nil {
 				t.Fatalf("cannot initialize aggregators: %s", err)
 			}
+			offsetMsecs := time.Now().UnixMilli()
 			for _, metrics := range inputMetrics {
 				// Push the inputMetrics to Aggregators
-				offsetMsecs := time.Now().UnixMilli()
 				tssInput := prometheus.MustParsePromMetrics(metrics, offsetMsecs)
 				matchIdxs = append(matchIdxs, a.Push(tssInput, nil)...)
-				time.Sleep(interval)
+				time.Sleep(interval + time.Millisecond) // shift by 1ms from flush border to avoid flaky tests
+				offsetMsecs += interval.Milliseconds()
 			}
-
 			a.MustStop()
-
-			// Verify matchIdxs equals to matchIdxsExpected
-			matchIdxsStr := ""
-			for _, v := range matchIdxs {
-				matchIdxsStr += strconv.Itoa(int(v))
-			}
-			if matchIdxsStr != matchIdxsStrExpected {
-				t.Fatalf("unexpected matchIdxs;\ngot\n%s\nwant\n%s", matchIdxsStr, matchIdxsStrExpected)
-			}
-
-			// Verify the tssOutput contains the expected metrics
-			outputMetrics := timeSeriessToString(tssOutput)
-			if outputMetrics != outputMetricsExpected {
-				t.Fatalf("unexpected output metrics;\ngot\n%s\nwant\n%s", outputMetrics, outputMetricsExpected)
-			}
 		})
+
+		// Verify matchIdxs equals to matchIdxsExpected
+		matchIdxsStr := ""
+		for _, v := range matchIdxs {
+			matchIdxsStr += strconv.Itoa(int(v))
+		}
+		if matchIdxsStr != matchIdxsStrExpected {
+			t.Fatalf("unexpected matchIdxs;\ngot\n%s\nwant\n%s", matchIdxsStr, matchIdxsStrExpected)
+		}
+
+		// Verify the tssOutput contains the expected metrics
+		outputMetrics := timeSeriessToString(tssOutput)
+		if outputMetrics != outputMetricsExpected {
+			t.Fatalf("unexpected output metrics;\ngot\n%s\nwant\n%s", outputMetrics, outputMetricsExpected)
+		}
 	}
 
 	// Empty config
@@ -635,6 +634,19 @@ cpu_usage:1m_without_cpu_quantiles{quantile="1"} 90
   outputs: ["quantiles(0, 0.5, 1)"]
 `, "1111111")
 
+	// no stale quantiles should be produced
+	f([]string{`
+cpu_usage{cpu="1"} 3
+cpu_usage{cpu="2"} 3`,
+		`cpu_usage{cpu="2"} 4`,
+	}, time.Minute, `cpu_usage:1m_quantiles{cpu="1",quantile="1"} 3
+cpu_usage:1m_quantiles{cpu="2",quantile="1"} 3
+cpu_usage:1m_quantiles{cpu="2",quantile="1"} 4
+`, `
+- interval: 1m
+  outputs: ["quantiles(1)"]
+`, "111")
+
 	// append additional label
 	f([]string{`
 foo{abc="123"} 4