docs/changelog: cut release v1.144.0

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/build.yml

@@ -57,15 +57,17 @@ jobs:
             arch: amd64
           - os: openbsd
             arch: amd64
+          - os: netbsd
+            arch: amd64
           - os: windows
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/changelog-linter.yml

@@ -9,7 +9,7 @@ jobs:
   tip-lint:
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: 'actions/checkout@v6'
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           fetch-depth: 0  # we need full history for commit verification
 

.github/workflows/check-licenses.yml

@@ -15,11 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           go-version-file: 'go.mod'
           cache: false
@@ -27,7 +27,7 @@ jobs:
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-go.yml

@@ -29,18 +29,18 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Set up Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache: false
           go-version-file: 'go.mod'
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/go-build
@@ -50,14 +50,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4.35.2
+        uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4.35.2
+        uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4.35.2
+        uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7  # v4.35.3
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -16,19 +16,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: __vm-docs
 
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v7
+        uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd  # v7.0.0
         id: import-gpg
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}

.github/workflows/test.yml

@@ -32,11 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum
@@ -47,7 +47,7 @@ jobs:
       - run: go version
 
       - name: Cache golangci-lint
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: |
             ~/.cache/golangci-lint
@@ -72,11 +72,11 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum
@@ -94,11 +94,11 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/vmui.yml

@@ -32,11 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
       - name: Cache node_modules
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
         with:
           path: app/vmui/packages/vmui/node_modules
           key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
@@ -69,7 +69,7 @@ jobs:
           VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@v3
+        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           report-json: app/vmui/packages/vmui/vmui-lint-report.json

Makefile

@@ -535,6 +535,15 @@ remove-golangci-lint:
 govulncheck: install-govulncheck
 	govulncheck ./...
 
+govulncheck-docker:
+	docker run -w $(PWD) -v $(PWD):$(PWD) \
+		-v govulncheck-gomod-cache:/root/go/pkg/mod \
+		-v govulncheck-gobuild-cache:/root/.cache/go-build \
+		-v govulncheck-go-bin:/root/go/bin \
+		--env="GOCACHE=/root/.cache/go-build" \
+		--env="GOMODCACHE=/root/go/pkg/mod" \
+		"$(GO_BUILDER_IMAGE)" /bin/sh -c "which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./..."
+
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 

app/vmagent/remotewrite/client.go

@@ -2,6 +2,7 @@ package remotewrite
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -59,6 +60,8 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	basicAuthUsernameFile = flagutil.NewArrayString("remoteWrite.basicAuth.usernameFile", "Optional path to basic auth username to use for the corresponding -remoteWrite.url. "+
+		"The file is re-read every second")
 	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
 		"The file is re-read every second")
@@ -223,12 +226,14 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 		hdrs = strings.Split(headersValue, "^^")
 	}
 	username := basicAuthUsername.GetOptionalArg(argIdx)
+	usernameFile := basicAuthUsernameFile.GetOptionalArg(argIdx)
 	password := basicAuthPassword.GetOptionalArg(argIdx)
 	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
 	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || password != "" || passwordFile != "" {
+	if username != "" || usernameFile != "" || password != "" || passwordFile != "" {
 		basicAuthCfg = &promauth.BasicAuthConfig{
 			Username:     username,
+			UsernameFile: usernameFile,
 			Password:     promauth.NewSecret(password),
 			PasswordFile: passwordFile,
 		}
@@ -306,11 +311,6 @@ func (c *client) runWorker() {
 		if !ok {
 			return
 		}
-		if len(block) == 0 {
-			// skip empty data blocks from sending
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241
-			continue
-		}
 		go func() {
 			startTime := time.Now()
 			ch <- c.sendBlock(block)
@@ -326,15 +326,20 @@ func (c *client) runWorker() {
 			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			return
 		case <-c.stopCh:
-			// c must be stopped. Wait for a while in the hope the block will be sent.
-			graceDuration := 5 * time.Second
+			// c must be stopped. Wait up to 5 seconds for the in-flight request to complete.
+			// If it succeeds, drain the remaining in-memory queue before returning.
+			stopCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			defer cancel()
+
 			select {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
 					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+				} else {
+					c.drainInMemoryQueue(stopCtx, block[:0])
 				}
-			case <-time.After(graceDuration):
+			case <-stopCtx.Done():
 				// Return unsent block to the queue.
 				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			}
@@ -466,7 +471,7 @@ again:
 				goto again
 			}
 
-			logger.Warnf("failed to repack zstd block (%s bytes) to snappy: %s; The block will be rejected. "+
+			logger.Warnf("failed to repack zstd block (%d bytes) to snappy: %s; The block will be rejected. "+
 				"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
 				zstdBlockLen, err)
 		}
@@ -504,6 +509,32 @@ again:
 	goto again
 }
 
+func (c *client) drainInMemoryQueue(stopCtx context.Context, block []byte) {
+	var ok bool
+	for {
+		select {
+		case <-stopCtx.Done():
+			return
+		default:
+		}
+
+		block, ok = c.fq.MustReadInMemoryBlock(block[:0])
+		if !ok {
+			// The in memory queue has already been drained,
+			// or persisted queue is being used.
+			// In this case it is guaranteed that fq will be empty
+			return
+		}
+
+		// at this stage c.stopCh should be closed
+		// so sendBlock function should not perform retries
+		if ok := c.sendBlock(block); !ok {
+			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			return
+		}
+	}
+}
+
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 

app/vmagent/remotewrite/client_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 func TestParseRetryAfterHeader(t *testing.T) {
@@ -36,6 +37,40 @@ func TestParseRetryAfterHeader(t *testing.T) {
 	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
 }
 
+func TestInitSecretFlags(t *testing.T) {
+	showRemoteWriteURLOrig := *showRemoteWriteURL
+	defer func() {
+		*showRemoteWriteURL = showRemoteWriteURLOrig
+		flagutil.UnregisterAllSecretFlags()
+	}()
+
+	flagutil.UnregisterAllSecretFlags()
+	*showRemoteWriteURL = false
+	InitSecretFlags()
+	if !flagutil.IsSecretFlag("remotewrite.url") {
+		t.Fatalf("expecting remoteWrite.url to be secret")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.headers") {
+		t.Fatalf("expecting remoteWrite.headers to be secret")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.proxyurl") {
+		t.Fatalf("expecting remoteWrite.proxyURL to be secret")
+	}
+
+	flagutil.UnregisterAllSecretFlags()
+	*showRemoteWriteURL = true
+	InitSecretFlags()
+	if flagutil.IsSecretFlag("remotewrite.url") {
+		t.Fatalf("remoteWrite.url must remain visible when -remoteWrite.showURL is set")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.headers") {
+		t.Fatalf("expecting remoteWrite.headers to remain secret")
+	}
+	if !flagutil.IsSecretFlag("remotewrite.proxyurl") {
+		t.Fatalf("expecting remoteWrite.proxyURL to remain secret")
+	}
+}
+
 func TestRepackBlockFromZstdToSnappy(t *testing.T) {
 	expectedPlainBlock := []byte(`foobar`)
 

app/vmagent/remotewrite/remotewrite.go

@@ -151,6 +151,10 @@ func InitSecretFlags() {
 		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
 		flagutil.RegisterSecretFlag("remoteWrite.url")
 	}
+	// remoteWrite.proxyURL can contain authentication codes.
+	flagutil.RegisterSecretFlag("remoteWrite.proxyURL")
+	// remoteWrite.headers can contain auth headers such as Authorization and API keys.
+	flagutil.RegisterSecretFlag("remoteWrite.headers")
 }
 
 var (
@@ -167,6 +171,18 @@ func Init() {
 	if len(*remoteWriteURLs) == 0 {
 		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
 	}
+	if *shardByURL && len(*disableOnDiskQueue) > 1 {
+		disableOnDiskQueues := *disableOnDiskQueue
+
+		firstValue := disableOnDiskQueues[0]
+		for _, v := range disableOnDiskQueues[1:] {
+			if firstValue != v {
+				logger.Fatalf("all -remoteWrite.url targets must have the same -remoteWrite.disableOnDiskQueue setting when -remoteWrite.shardByURL is enabled; " +
+					"either enable or disable -remoteWrite.disableOnDiskQueue for all targets")
+			}
+		}
+	}
+
 	if limit := getMaxHourlySeries(); limit > 0 {
 		hourlySeriesLimiter = bloomfilter.NewLimiter(limit, time.Hour)
 		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
@@ -499,7 +515,9 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 //
 // calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
 func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
-	if !disableOnDiskQueueAny {
+	// When -remoteWrite.shardByURL=true always use all configured remote writes to preserve stable metrics distribution across shards.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507
+	if !disableOnDiskQueueAny || *shardByURL {
 		return rwctxsGlobal, true
 	}
 
@@ -514,12 +532,6 @@ func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailu
 				return nil, false
 			}
 			rowsCount := getRowsCount(tss)
-			if *shardByURL {
-				// Todo: When shardByURL is enabled, the following metrics won't be 100% accurate. Because vmagent don't know
-				// which rwctx should data be pushed to yet. Let's consider the hashing algorithm fair and will distribute
-				// data to all rwctxs evenly.
-				rowsCount = rowsCount / len(rwctxsGlobal)
-			}
 			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 		}
 	}
@@ -699,7 +711,7 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
 			}
 			tmpLabels.Labels = hashLabels
 		}
-		h := getLabelsHash(hashLabels)
+		h := getLabelsHashForShard(hashLabels)
 
 		// Get the rwctxIdx through consistent hashing and then map it to the index in shards.
 		// The rwctxIdx is not always equal to the shardIdx, for example, when some rwctx are not available.
@@ -790,11 +802,28 @@ var (
 	dailySeriesLimitRowsDropped  = metrics.NewCounter(`vmagent_daily_series_limit_rows_dropped_total`)
 )
 
+// getLabelsHashForShard is a separate function from getLabelsHash because
+// it omits the '=' separator between label name and value for backward compatibility.
+// Changing it would re-shard all series across remoteWrite targets.
+func getLabelsHashForShard(labels []prompb.Label) uint64 {
+	bb := labelsHashBufPool.Get()
+	b := bb.B[:0]
+	for _, label := range labels {
+		b = append(b, label.Name...)
+		b = append(b, label.Value...)
+	}
+	h := xxhash.Sum64(b)
+	bb.B = b
+	labelsHashBufPool.Put(bb)
+	return h
+}
+
 func getLabelsHash(labels []prompb.Label) uint64 {
 	bb := labelsHashBufPool.Get()
 	b := bb.B[:0]
 	for _, label := range labels {
 		b = append(b, label.Name...)
+		b = append(b, '=')
 		b = append(b, label.Value...)
 	}
 	h := xxhash.Sum64(b)

app/vmagent/remotewrite/remotewrite_test.go

@@ -25,7 +25,7 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		t.Helper()
 
 		// Distribute itemsCount hashes returned by getLabelsHash() across bucketsCount buckets.
-		itemsCount := 1_000 * bucketsCount
+		itemsCount := 10_000 * bucketsCount
 		m := make([]int, bucketsCount)
 		var labels []prompb.Label
 		for i := range itemsCount {
@@ -44,10 +44,12 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		}
 
 		// Verify that the distribution is even
-		expectedItemsPerBucket := itemsCount / bucketsCount
+		expectedItemsPerBucket := float64(itemsCount / bucketsCount)
+		allowedDeviation := math.Round(float64(expectedItemsPerBucket) * 0.04)
 		for _, n := range m {
-			if math.Abs(1-float64(n)/float64(expectedItemsPerBucket)) > 0.04 {
-				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want around %d", bucketsCount, n, expectedItemsPerBucket)
+			if math.Abs(expectedItemsPerBucket-float64(n)) > allowedDeviation {
+				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want in range [%.0f, %.0f]",
+					bucketsCount, n, expectedItemsPerBucket-allowedDeviation, expectedItemsPerBucket+allowedDeviation)
 			}
 		}
 	}

app/vmalert-tool/unittest/unittest.go

@@ -61,7 +61,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	}
 	eu, err := url.Parse(externalURL)
 	if err != nil {
-		logger.Fatalf("failed to parse external URL: %w", err)
+		logger.Fatalf("failed to parse external URL: %s", err)
 	}
 	if err := templates.Load([]string{}, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)

app/vmalert/datasource/client_test.go

@@ -772,7 +772,7 @@ func TestHeaders(t *testing.T) {
 
 	// basic auth
 	f(func() *Client {
-		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
+		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "", "bar", ""))
 		if err != nil {
 			t.Fatalf("Error get auth config: %s", err)
 		}
@@ -817,7 +817,7 @@ func TestHeaders(t *testing.T) {
 
 	// custom header overrides basic auth
 	f(func() *Client {
-		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
+		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "", "bar", ""))
 		if err != nil {
 			t.Fatalf("Error get auth config: %s", err)
 		}

app/vmalert/datasource/init.go

@@ -27,6 +27,7 @@ var (
 		"Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
+	basicAuthUsernameFile = flag.String("datasource.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
 	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
 
@@ -63,6 +64,7 @@ func InitSecretFlags() {
 	if !*showDatasourceURL {
 		flagutil.RegisterSecretFlag("datasource.url")
 	}
+	flagutil.RegisterSecretFlag("datasource.headers")
 }
 
 // ShowDatasourceURL whether to show -datasource.url with sensitive information
@@ -105,7 +107,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -datasource.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/notifier/alertmanager.go

@@ -191,7 +191,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 
 	aCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
+		vmalertutil.WithBasicAuth(ba.Username, ba.UsernameFile, ba.Password.String(), ba.PasswordFile),
 		vmalertutil.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
 		vmalertutil.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
 		vmalertutil.WithHeaders(strings.Join(authCfg.Headers, "^^")),

app/vmalert/notifier/config_watcher.go

@@ -105,7 +105,7 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, targetsFn
 			}
 			targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
 			for _, err := range errors {
-				logger.Errorf("failed to init notifier for %q: %w", typeK, err)
+				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
 			}
 			cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 		}
@@ -274,7 +274,7 @@ func (cw *configWatcher) updateTargets(key TargetType, targetMts map[string]targ
 	for addr, metadata := range targetMts {
 		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, metadata.alertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
-			logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
+			logger.Errorf("failed to init %s notifier with addr %q: %s", key, addr, err)
 			continue
 		}
 		updatedTargets = append(updatedTargets, Target{

app/vmalert/notifier/init.go

@@ -36,6 +36,7 @@ var (
 		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
 		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
+	basicAuthUsernameFile = flagutil.NewArrayString("notifier.basicAuth.usernameFile", "Optional path to basic auth username file for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
 
@@ -193,6 +194,7 @@ func InitSecretFlags() {
 	if !*showNotifierURL {
 		flagutil.RegisterSecretFlag("notifier.url")
 	}
+	flagutil.RegisterSecretFlag("notifier.headers")
 }
 
 func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
@@ -213,6 +215,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			},
 			BasicAuth: &promauth.BasicAuthConfig{
 				Username:     basicAuthUsername.GetOptionalArg(i),
+				UsernameFile: basicAuthUsernameFile.GetOptionalArg(i),
 				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
 				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
 			},

app/vmalert/remoteread/init.go

@@ -28,6 +28,7 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
+	basicAuthUsernameFile = flag.String("remoteRead.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
 	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
 
@@ -58,6 +59,7 @@ func InitSecretFlags() {
 	if !*showRemoteReadURL {
 		flagutil.RegisterSecretFlag("remoteRead.url")
 	}
+	flagutil.RegisterSecretFlag("remoteRead.headers")
 }
 
 // Init creates a Querier from provided flag values.
@@ -80,7 +82,7 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/remotewrite/client.go

@@ -11,6 +11,7 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/cespare/xxhash/v2"
@@ -18,6 +19,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -57,6 +60,11 @@ type Client struct {
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
+
+	// Whether to encode the write request with VictoriaMetrics remote write protocol.
+	// It is set to true by default, and will be switched to false if the client
+	// receives specific errors indicating that the remote storage doesn't support VictoriaMetrics remote write protocol.
+	isVMRemoteWrite atomic.Bool
 }
 
 // Config is config for remote write client.
@@ -116,6 +124,7 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		doneCh:        make(chan struct{}),
 		input:         make(chan prompb.TimeSeries, cfg.MaxQueueSize),
 	}
+	c.isVMRemoteWrite.Store(true)
 
 	for i := 0; i < cc; i++ {
 		c.wg.Go(func() {
@@ -265,8 +274,16 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	defer wr.Reset()
 	defer bufferFlushDuration.UpdateDuration(time.Now())
 
-	data := wr.MarshalProtobuf(nil)
-	b := snappy.Encode(nil, data)
+	bb := writeRequestBufPool.Get()
+	bb.B = wr.MarshalProtobuf(bb.B[:0])
+	zb := compressBufPool.Get()
+	defer compressBufPool.Put(zb)
+	if c.isVMRemoteWrite.Load() {
+		zb.B = zstd.CompressLevel(zb.B[:0], bb.B, 0)
+	} else {
+		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
+	}
+	writeRequestBufPool.Put(bb)
 
 	maxRetryInterval := *retryMaxTime
 	bt := timeutil.NewBackoffTimer(*retryMinInterval, maxRetryInterval)
@@ -278,17 +295,17 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	attempts := 0
 L:
 	for {
-		err := c.send(ctx, b)
+		err := c.send(ctx, zb.B)
 		if err != nil && (errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err)) {
 			// Something in the middle between client and destination might be closing
 			// the connection. So we do a one more attempt in hope request will succeed.
-			err = c.send(ctx, b)
+			err = c.send(ctx, zb.B)
 		}
 		if err == nil {
 			sentRows.Add(len(wr.Timeseries))
-			sentBytes.Add(len(b))
+			sentBytes.Add(len(zb.B))
 			flushedRows.Update(float64(len(wr.Timeseries)))
-			flushedBytes.Update(float64(len(b)))
+			flushedBytes.Update(float64(len(zb.B)))
 			return
 		}
 
@@ -340,12 +357,16 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
 
-	// RFC standard compliant headers
-	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("User-Agent", "vmalert")
 	req.Header.Set("Content-Type", "application/x-protobuf")
 
-	// Prometheus compliant headers
-	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	if encoding.IsZstd(data) {
+		req.Header.Set("Content-Encoding", "zstd")
+		req.Header.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
+	} else {
+		req.Header.Set("Content-Encoding", "snappy")
+		req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	}
 
 	if c.authCfg != nil {
 		err = c.authCfg.SetHeaders(req, true)
@@ -374,6 +395,29 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 		// respond with HTTP 2xx status code when write is successful.
 		return nil
 	case 4:
+		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
+		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
+		// - Real-world implementations of v1 use both 400 and 415 status codes.
+		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
+		if resp.StatusCode == http.StatusUnsupportedMediaType || resp.StatusCode == http.StatusBadRequest {
+			if encoding.IsZstd(data) {
+				logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
+					"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", req.URL.Redacted())
+				zstdBlockLen := len(data)
+				data, err = repackBlockFromZstdToSnappy(data)
+				if err == nil {
+					logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
+						"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", req.URL.Redacted())
+					c.isVMRemoteWrite.Store(false)
+					return c.send(ctx, data)
+				}
+
+				logger.Warnf("failed to repack zstd block (%d bytes) to snappy: %s; The block will be rejected. "+
+					"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
+					zstdBlockLen, err)
+			}
+		}
+
 		if resp.StatusCode != http.StatusTooManyRequests {
 			// MUST NOT retry write requests on HTTP 4xx responses other than 429
 			return &nonRetriableError{
@@ -394,3 +438,19 @@ type nonRetriableError struct {
 func (e *nonRetriableError) Error() string {
 	return e.err.Error()
 }
+
+var (
+	writeRequestBufPool bytesutil.ByteBufferPool
+	compressBufPool     bytesutil.ByteBufferPool
+)
+
+// repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
+func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
+	plainBlock := make([]byte, 0, len(zstdBlock)*2)
+	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
+	if err != nil {
+		return nil, err
+	}
+
+	return snappy.Encode(nil, plainBlock), nil
+}

app/vmalert/remotewrite/client_test.go

@@ -12,8 +12,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang/snappy"
-
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
@@ -159,8 +158,8 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	h := r.Header.Get("Content-Encoding")
-	if h != "snappy" {
-		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not snappy (%q)", h))
+	if h != "zstd" {
+		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not zstd (%q)", h))
 	}
 
 	h = r.Header.Get("Content-Type")
@@ -168,9 +167,9 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("header read error: Content-Type is not x-protobuf (%q)", h))
 	}
 
-	h = r.Header.Get("X-Prometheus-Remote-Write-Version")
-	if h != "0.1.0" {
-		rw.err(w, fmt.Errorf("header read error: X-Prometheus-Remote-Write-Version is not 0.1.0 (%q)", h))
+	h = r.Header.Get("X-VictoriaMetrics-Remote-Write-Version")
+	if h != "1" {
+		rw.err(w, fmt.Errorf("header read error: X-VictoriaMetrics-Remote-Write-Version is not 1 (%q)", h))
 	}
 
 	data, err := io.ReadAll(r.Body)
@@ -180,7 +179,7 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 	}
 	defer func() { _ = r.Body.Close() }()
 
-	b, err := snappy.Decode(nil, data)
+	b, err := zstd.Decompress(nil, data)
 	if err != nil {
 		rw.err(w, fmt.Errorf("decode err: %w", err))
 		return

app/vmalert/remotewrite/debug_client.go

@@ -9,8 +9,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/golang/snappy"
-
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -64,19 +63,17 @@ func (c *DebugClient) Close() error {
 }
 
 func (c *DebugClient) send(data []byte) error {
-	b := snappy.Encode(nil, data)
+	b := zstd.CompressLevel(nil, data, 0)
 	r := bytes.NewReader(b)
 	req, err := http.NewRequest(http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
 
-	// RFC standard compliant headers
-	req.Header.Set("Content-Encoding", "snappy")
+	req.Header.Set("Content-Encoding", "zstd")
 	req.Header.Set("Content-Type", "application/x-protobuf")
 
-	// Prometheus compliant headers
-	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
+	req.Header.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
 
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")

app/vmalert/remotewrite/init.go

@@ -13,8 +13,8 @@ import (
 )
 
 var (
-	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
-		"and recording rules results in form of timeseries. "+
+	addr = flag.String("remoteWrite.url", "", "Optional URL to persist alerts state and recording rules results in form of timeseries. "+
+		"It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. "+
 		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
@@ -26,6 +26,7 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
+	basicAuthUsernameFile = flag.String("remoteWrite.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
 	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
 
@@ -61,6 +62,7 @@ func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		flagutil.RegisterSecretFlag("remoteWrite.url")
 	}
+	flagutil.RegisterSecretFlag("remoteWrite.headers")
 }
 
 // Init creates Client object from given flags.
@@ -83,7 +85,7 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/vmalertutil/auth.go

@@ -20,11 +20,12 @@ func AuthConfig(filterOptions ...AuthConfigOptions) (*promauth.Config, error) {
 }
 
 // WithBasicAuth returns AuthConfigOptions and initialized promauth.BasicAuthConfig based on given params
-func WithBasicAuth(username, password, passwordFile string) AuthConfigOptions {
+func WithBasicAuth(username, usernameFile, password, passwordFile string) AuthConfigOptions {
 	return func(config *promauth.HTTPClientConfig) {
-		if username != "" || password != "" || passwordFile != "" {
+		if username != "" || usernameFile != "" || password != "" || passwordFile != "" {
 			config.BasicAuth = &promauth.BasicAuthConfig{
 				Username:     username,
+				UsernameFile: usernameFile,
 				Password:     promauth.NewSecret(password),
 				PasswordFile: passwordFile,
 			}

app/vmauth/example_config.yml

@@ -130,6 +130,16 @@ users:
   - "http://vmselect1:8481/select/{{.MetricsTenant}}/prometheus"
   - "http://vmselect2:8481/select/{{.MetricsTenant}}/prometheus"
 
+# JWT-based routing using header-based tenant identification (VictoriaMetrics cluster)
+# The AccountID and ProjectID from JWT vm_access claims are injected as HTTP headers.
+- name: jwt-header-tenant
+  jwt:
+    skip_verify: true
+  headers:
+  - "AccountID: {{.MetricsAccountID}}"
+  - "ProjectID: {{.MetricsProjectID}}"
+  url_prefix: "http://vminsert:8480/insert/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -17,6 +17,8 @@ import (
 
 const (
 	metricsTenantPlaceholder       = `{{.MetricsTenant}}`
+	metricsAccountIDPlaceholder    = `{{.MetricsAccountID}}`
+	metricsProjectIDPlaceholder    = `{{.MetricsProjectID}}`
 	metricsExtraLabelsPlaceholder  = `{{.MetricsExtraLabels}}`
 	metricsExtraFiltersPlaceholder = `{{.MetricsExtraFilters}}`
 
@@ -30,6 +32,8 @@ const (
 
 var allPlaceholders = []string{
 	metricsTenantPlaceholder,
+	metricsAccountIDPlaceholder,
+	metricsProjectIDPlaceholder,
 	metricsExtraLabelsPlaceholder,
 	metricsExtraFiltersPlaceholder,
 	logsAccountIDPlaceholder,
@@ -40,6 +44,8 @@ var allPlaceholders = []string{
 
 var urlPathPlaceHolders = []string{
 	metricsTenantPlaceholder,
+	metricsAccountIDPlaceholder,
+	metricsProjectIDPlaceholder,
 	logsAccountIDPlaceholder,
 	logsProjectIDPlaceholder,
 }
@@ -371,6 +377,8 @@ func jwtClaimsData(vma *jwt.VMAccessClaim) map[string][]string {
 	data := map[string][]string{
 		// TODO: optimize at parsing stage
 		metricsTenantPlaceholder:       {fmt.Sprintf("%d:%d", vma.MetricsAccountID, vma.MetricsProjectID)},
+		metricsAccountIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsAccountID)},
+		metricsProjectIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsProjectID)},
 		metricsExtraLabelsPlaceholder:  vma.MetricsExtraLabels,
 		metricsExtraFiltersPlaceholder: vma.MetricsExtraFilters,
 

app/vmauth/jwt_test.go

@@ -170,13 +170,13 @@ users:
   url_prefix: http://foo.bar
 `, "cannot parse public key from file \""+publicKeyFile+"\": failed to parse key \"invalidPEM\": failed to decode PEM block containing public key")
 
-	// unsupported placeholder in a header
+	// unsupported placeholder in a URL path
 	f(`
 users:
 - jwt: 
     skip_verify: true
   url_prefix: http://foo.bar/{{.UnsupportedPlaceholder}}/foo`,
-		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 	// unsupported placeholder in a header
 	f(`
@@ -187,7 +187,7 @@ users:
     - "AccountID: {{.UnsupportedPlaceholder}}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// spaces in templating not allowed
@@ -199,7 +199,19 @@ users:
     - "AccountID: {{ .LogsAccountID }}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+	)
+
+	// placeholder must match the entire header value
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+  headers:
+    - "AccountID: foo {{.MetricsAccountID}}"
+  url_prefix: http://foo.bar
+`,
+		"request header: \"AccountID\" has unsupported placeholder: \"foo {{.MetricsAccountID}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// oidc is not an object
@@ -364,10 +376,25 @@ users:
   url_prefix: http://foo.bar
 `, validRSAPublicKey, validECDSAPublicKey))
 
+	// metrics header placeholders
 	f(`
 users:
 - jwt:
     skip_verify: true
+  headers:
+  - "MetricsAccountID: {{.MetricsAccountID}}"
+  - "MetricsProjectID: {{.MetricsProjectID}}"
+  url_prefix: http://foo.bar
+`)
+
+	// logs header placeholders
+	f(`
+users:
+- jwt:
+    skip_verify: true
+  headers:
+  - "LogsAccountID: {{.LogsAccountID}}"
+  - "LogsProjectID: {{.LogsProjectID}}"
   url_prefix: http://foo.bar
 `)
 

app/vmauth/main_test.go

@@ -851,6 +851,30 @@ users:
 		responseExpected,
 	)
 
+	// test header injection and URL templating with individual placeholders
+	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
+	request.Header.Set(`Authorization`, `Bearer `+fullToken)
+	responseExpected = `
+statusCode=200
+path: /select/123/234/api/v1/query
+query:
+headers:
+    AccountID=123
+    ProjectID=234`
+	f(fmt.Sprintf(
+		`
+users:
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: {BACKEND}/select/{{.MetricsAccountID}}/{{.MetricsProjectID}}
+  headers:
+  - "AccountID: {{.MetricsAccountID}}"
+  - "ProjectID: {{.MetricsProjectID}}"`, string(publicKeyPEM)),
+		request,
+		responseExpected,
+	)
+
 	// extra_label and extra_filters from vm_access claim merged with statically defined
 	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
 	request.Header.Set(`Authorization`, `Bearer `+fullToken)

app/vmctl/vmctlutil/time_test.go

@@ -20,6 +20,9 @@ func TestGetTime_Failure(t *testing.T) {
 
 	// negative time
 	f("-292273086-05-16T16:47:06Z")
+
+	// relative duration that resolves to a timestamp before 1970
+	f("-9223372036.855")
 }
 
 func TestGetTime_Success(t *testing.T) {
@@ -77,9 +80,6 @@ func TestGetTime_Success(t *testing.T) {
 	// float timestamp representation",
 	f("1562529662.324", time.Date(2019, 7, 7, 20, 01, 02, 324e6, time.UTC))
 
-	// negative timestamp
-	f("-9223372036.855", time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC))
-
 	// big timestamp
 	f("1223372036855", time.Date(2008, 10, 7, 9, 33, 56, 855e6, time.UTC))
 

app/vmselect/main.go

@@ -69,6 +69,8 @@ func Init() {
 	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
 	initVMUIConfig()
 	initVMAlertProxy()
+
+	flagutil.RegisterSecretFlag("vmalert.proxyURL")
 }
 
 // Stop stops vmselect

app/vmselect/netstorage/netstorage.go

@@ -990,9 +990,6 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 		return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
 	}
 	tr := sq.GetTimeRange()
-	if err := vmstorage.CheckTimeRange(tr); err != nil {
-		return err
-	}
 	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
 	if err != nil {
 		return err
@@ -1098,9 +1095,6 @@ func SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline
 
 	// Setup search.
 	tr := sq.GetTimeRange()
-	if err := vmstorage.CheckTimeRange(tr); err != nil {
-		return nil, err
-	}
 	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
 	if err != nil {
 		return nil, err
@@ -1127,9 +1121,6 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 
 	// Setup search.
 	tr := sq.GetTimeRange()
-	if err := vmstorage.CheckTimeRange(tr); err != nil {
-		return nil, err
-	}
 	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
 	if err != nil {
 		return nil, err

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-C7gvW_Zn.js"></script>
+  <script type="module" crossorigin src="./assets/index-BjJ7fDL7.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-D2OEy8Ra.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-BL7jEFBa.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -56,8 +56,8 @@ var (
 
 	logNewSeries = flag.Bool("logNewSeries", false, "Whether to log new series. This option is for debug purposes only. It can lead to performance issues "+
 		"when big number of new series are ingested into VictoriaMetrics")
-	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod. "+
-		"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. "+
+	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod and -futureRetention. "+
+		"When set, then /api/v1/query_range will return an error for queries with 'from' value outside -retentionPeriod or 'to' value beyond -futureRetention. "+
 		"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee")
 	maxHourlySeries = flag.Int64("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
@@ -103,21 +103,6 @@ var (
 		"If set to 0 or a negative value, defaults to 1% of allowed memory.")
 )
 
-// CheckTimeRange returns true if the given tr is denied for querying.
-func CheckTimeRange(tr storage.TimeRange) error {
-	if !*denyQueriesOutsideRetention {
-		return nil
-	}
-	minAllowedTimestamp := int64(fasttime.UnixTimestamp()*1000) - retentionPeriod.Milliseconds()
-	if tr.MinTimestamp > minAllowedTimestamp {
-		return nil
-	}
-	return &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("the given time range %s is outside the allowed -retentionPeriod=%s according to -denyQueriesOutsideRetention", &tr, retentionPeriod),
-		StatusCode: http.StatusServiceUnavailable,
-	}
-}
-
 // Init initializes vmstorage.
 func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
@@ -151,14 +136,15 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	startTime := time.Now()
 	WG = syncwg.WaitGroup{}
 	opts := storage.OpenOptions{
-		Retention:             retentionPeriod.Duration(),
-		FutureRetention:       futureRetention.Duration(),
-		MaxHourlySeries:       getMaxHourlySeries(),
-		MaxDailySeries:        getMaxDailySeries(),
-		DisablePerDayIndex:    *disablePerDayIndex,
-		TrackMetricNamesStats: *trackMetricNamesStats,
-		IDBPrefillStart:       *idbPrefillStart,
-		LogNewSeries:          *logNewSeries,
+		Retention:                   retentionPeriod.Duration(),
+		FutureRetention:             futureRetention.Duration(),
+		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
+		MaxHourlySeries:             getMaxHourlySeries(),
+		MaxDailySeries:              getMaxDailySeries(),
+		DisablePerDayIndex:          *disablePerDayIndex,
+		TrackMetricNamesStats:       *trackMetricNamesStats,
+		IDBPrefillStart:             *idbPrefillStart,
+		LogNewSeries:                *logNewSeries,
 	}
 	strg := storage.MustOpenStorage(*DataPath, opts)
 	Storage = strg

app/vmui/packages/vmui/src/api/types.ts

@@ -3,6 +3,7 @@ export interface MetricBase {
   metric: {
     [key: string]: string;
   };
+  nullTimestamps?: number[];
 }
 
 export interface MetricResult extends MetricBase {

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -16,6 +16,7 @@ export interface ChartTooltipProps {
   point: { top: number, left: number };
   unit?: string;
   statsFormatted?: SeriesItemStatsFormatted;
+  description?: ReactNode;
   isSticky?: boolean;
   info?: ReactNode;
   marker?: string;
@@ -34,6 +35,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
   unit = "",
   info,
   statsFormatted,
+  description,
   isSticky,
   marker,
   duplicateCount = 0,
@@ -173,6 +175,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
           ))}
         </table>
       )}
+      {description && <p className="vm-chart-tooltip__description">{description}</p>}
       {info && <p className="vm-chart-tooltip__info">{info}</p>}
     </div>
   ), u.root);

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -143,4 +143,10 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     word-break: break-all;
     white-space: pre-wrap;
   }
+
+  &__description {
+    word-break: break-word;
+    white-space: pre-wrap;
+    opacity: 0.85;
+  }
 }

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -4,7 +4,7 @@ import { ChartTooltipProps } from "../../components/Chart/ChartTooltip/ChartTool
 import { SeriesItem } from "../../types";
 import dayjs from "dayjs";
 import { DATE_FULL_TIMEZONE_FORMAT } from "../../constants/date";
-import { formatPrettyNumber, getMetricName } from "../../utils/uplot";
+import { getMetricName } from "../../utils/uplot";
 import { MetricResult } from "../../api/types";
 import useEventListener from "../useEventListener";
 
@@ -15,19 +15,65 @@ interface LineTooltipHook {
   unit?: string;
 }
 
+// Pixel proximity for detecting hover over null-timestamp X markers drawn at chart bottom.
+const NULL_HOVER_PROX = 8;
+// Half the visual marker height in CSS px (BASE_POINT_SIZE * 1.4 / 2 from scatter.ts).
+// scatter.ts lifts the marker center by this amount above yMin so the icon sits inside
+// the plot area; the hover y-anchor must match that offset.
+const NULL_MARKER_HALF_CSS = 2.8;
+
+interface NullHover {
+  seriesIdx: number;
+  timestamp: number;
+}
+
+const findNullHover = (u: uPlot): NullHover | null => {
+  const cursorLeft = u.cursor.left ?? -1;
+  const cursorTop = u.cursor.top ?? -1;
+  if (cursorLeft < 0 || cursorTop < 0) return null;
+
+  const scaleY = u.scales["1"];
+  if (!scaleY || scaleY.min == null) return null;
+  const yPos = u.valToPos(scaleY.min, "1") - NULL_MARKER_HALF_CSS;
+  if (Math.abs(cursorTop - yPos) > NULL_HOVER_PROX) return null;
+
+  let best: { seriesIdx: number; timestamp: number; dist: number } | null = null;
+  for (let s = 1; s < u.series.length; s++) {
+    const seriesItem = u.series[s] as SeriesItem;
+    if (!seriesItem.show) continue;
+    const nullTs = seriesItem.nullTimestamps;
+    if (!nullTs || !nullTs.length) continue;
+
+    for (let i = 0; i < nullTs.length; i++) {
+      const t = nullTs[i];
+      const xPos = u.valToPos(t, "x");
+      const dist = Math.abs(cursorLeft - xPos);
+      if (dist < NULL_HOVER_PROX && (best === null || dist < best.dist)) {
+        best = { seriesIdx: s, timestamp: t, dist };
+      }
+    }
+  }
+  return best ? { seriesIdx: best.seriesIdx, timestamp: best.timestamp } : null;
+};
+
+const NULL_DESCRIPTION = "\"null\" can be a staleness marker or an actual NaN/null value produced by exporter.";
+
 const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   const [showTooltip, setShowTooltip] = useState(false);
   const [tooltipIdx, setTooltipIdx] = useState({ seriesIdx: -1, dataIdx: -1 });
+  const [nullTooltip, setNullTooltip] = useState<NullHover | null>(null);
   const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipProps[]>([]);
 
   const resetTooltips = () => {
     setStickyToolTips([]);
     setTooltipIdx({ seriesIdx: -1, dataIdx: -1 });
+    setNullTooltip(null);
   };
 
   const setCursor = (u: uPlot) => {
     const dataIdx = u.cursor.idx ?? -1;
     setTooltipIdx(prev => ({ ...prev, dataIdx }));
+    setNullTooltip(findNullHover(u));
   };
 
   const seriesFocus = (u: uPlot, sidx: (number | null)) => {
@@ -35,7 +81,36 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     setTooltipIdx(prev => ({ ...prev, seriesIdx }));
   };
 
+  const getNullTooltipProps = (hit: NullHover): ChartTooltipProps => {
+    const { seriesIdx, timestamp } = hit;
+    const metricItem = metrics[seriesIdx - 1];
+    const seriesItem = series[seriesIdx] as SeriesItem;
+
+    const groups = new Set(metrics.map(m => m.group));
+    const group = metricItem?.group || 0;
+
+    const yMin = u?.scales?.[1]?.min ?? 0;
+    const point = {
+      top: u ? u.valToPos(yMin, seriesItem?.scale || "1") - NULL_MARKER_HALF_CSS : 0,
+      left: u ? u.valToPos(timestamp, "x") : 0,
+    };
+
+    return {
+      u,
+      id: `null_${seriesIdx}_${timestamp}`,
+      title: groups.size > 1 ? `Query ${group}` : "",
+      dates: [dayjs(timestamp * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT)],
+      value: "null",
+      info: getMetricName(metricItem, seriesItem),
+      description: NULL_DESCRIPTION,
+      marker: `${seriesItem?.stroke}`,
+      point,
+    };
+  };
+
   const getTooltipProps = useCallback((): ChartTooltipProps => {
+    if (nullTooltip) return getNullTooltipProps(nullTooltip);
+
     const { seriesIdx, dataIdx } = tooltipIdx;
     const metricItem = metrics[seriesIdx - 1];
     const seriesItem = series[seriesIdx] as SeriesItem;
@@ -44,8 +119,6 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     const group = metricItem?.group || 0;
 
     const value = u?.data?.[seriesIdx]?.[dataIdx] || 0;
-    const min = u?.scales?.[1]?.min || 0;
-    const max = u?.scales?.[1]?.max || 1;
     const date = u?.data?.[0]?.[dataIdx] || 0;
 
     let duplicateCount = 1;
@@ -80,13 +153,13 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
       id: `${seriesIdx}_${dataIdx}`,
       title: groups.size > 1 ? `Query ${group}` : "",
       dates: [date ? dayjs(date * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT) : "-"],
-      value: formatPrettyNumber(value, min, max),
+      value: value.toLocaleString("en-US", { maximumFractionDigits: 20 }),
       info: getMetricName(metricItem, seriesItem),
       statsFormatted: seriesItem?.statsFormatted,
       marker: `${seriesItem?.stroke}`,
       duplicateCount,
     };
-  }, [u, tooltipIdx, metrics, series, unit]);
+  }, [u, tooltipIdx, metrics, series, unit, nullTooltip]);
 
   const handleClick = useCallback(() => {
     if (!showTooltip) return;
@@ -101,8 +174,9 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   };
 
   useEffect(() => {
-    setShowTooltip(tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1);
-  }, [tooltipIdx]);
+    const normalHit = tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1;
+    setShowTooltip(normalHit || nullTooltip !== null);
+  }, [tooltipIdx, nullTooltip]);
 
   useEventListener("click", handleClick);
 

app/vmui/packages/vmui/src/pages/CustomPanel/utils.test.ts

@@ -20,7 +20,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,value2");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123");
   });
 
   it("should return a valid CSV string for multiple metric entries with values", () => {
@@ -43,7 +43,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,value2\n456,value4");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123\n456,value4,1623949200,456");
   });
 
   it("should handle metric entries with multiple values field", () => {
@@ -58,7 +58,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123-456,values");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123-456,values,-,-");
   });
 
   it("should handle a combination of metric entries with value and values", () => {
@@ -81,6 +81,19 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2\n123,first\n456-789,second");
+    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,first,1623945600,123\n456-789,second,-,-");
   });
+
+  it("should return value and timestamp if metric field is empty", () => {
+    const data: InstantMetricResult[] = [
+      {
+        value: [1623945600, "123"],
+        group: 0,
+        metric: {}
+      },
+    ];
+    const result = convertMetricsDataToCSV(data);
+    expect(result).toBe("__timestamp__,__value__\n1623945600,123");
+  });
+
 });

app/vmui/packages/vmui/src/pages/CustomPanel/utils.ts

@@ -3,16 +3,22 @@ import { getColumns, MetricCategory } from "../../hooks/useSortedCategories";
 import { formatValueToCSV } from "../../utils/csv";
 
 const getHeaders = (data: InstantMetricResult[]): string => {
-  return getColumns(data).map(({ key }) => key).join(",");
+  const metricHeaders = getColumns(data).map(({ key }) => key);
+  return [...metricHeaders, "__timestamp__", "__value__"].join(",");
 };
 
 const getRows = (data: InstantMetricResult[], headers: MetricCategory[]) => {
-  return data?.map(d => headers.map(c => formatValueToCSV(d.metric[c.key] || "-")).join(","));
+  return data?.map(d => {
+    const metricPart = headers.map(c => formatValueToCSV(d.metric[c.key] || "-"));
+    const timestamp = d.value ? formatValueToCSV(String(d.value[0])) : "-";
+    const value = d.value ? formatValueToCSV(d.value[1]) : "-";
+    return [...metricPart, timestamp, value].join(",");
+  });
 };
 
 export const convertMetricsDataToCSV = (data: InstantMetricResult[]): string => {
+  if (!data.length) return "";
   const headers = getHeaders(data);
-  if (!headers.length) return "";
   const rows = getRows(data, getColumns(data));
   return [headers, ...rows].join("\n");
 };

app/vmui/packages/vmui/src/pages/RawQueryPage/hooks/useFetchExport.ts

@@ -149,15 +149,21 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
             const pointsToTake = shouldDownsample ? maxPointsPerSeries : totalPoints;
             const step = shouldDownsample ? totalPoints / maxPointsPerSeries : 1;
 
-            const values: [number, number][] = Array.from({ length: pointsToTake }, (_, i) => {
+            const values: [number, number][] = new Array(pointsToTake);
+            const nullTimestamps: number[] = [];
+            for (let i = 0; i < pointsToTake; i++) {
               const idx = shouldDownsample ? Math.floor(i * step) : i;
-              return [rawTimestamps[idx] / 1000, rawValues[idx]];
-            });
+              const ts = rawTimestamps[idx] / 1000;
+              const raw = rawValues[idx];
+              if (raw === null) nullTimestamps.push(ts);
+              values[i] = [ts, raw as number];
+            }
 
             tempData.push({
               group: counter,
               metric: jsonLine.metric,
               values,
+              nullTimestamps,
             } as MetricBase);
           }
 

app/vmui/packages/vmui/src/pages/TopQueries/TopQueryPanel/TopQueryPanel.tsx

@@ -1,4 +1,4 @@
-import { FC, useState } from "react";
+import { FC, ReactNode, useState } from "react";
 import { TopQuery } from "../../../types";
 import JsonView from "../../../components/Views/JsonView/JsonView";
 import { CodeIcon, TableIcon } from "../../../components/Main/Icons";
@@ -8,10 +8,18 @@ import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 
+export interface TopQueryColumn {
+  title?: string;
+  tooltip?: string;
+  key: keyof TopQuery;
+  sortBy?: keyof TopQuery;
+  format?: (row: TopQuery) => ReactNode;
+}
+
 export interface TopQueryPanelProps {
   rows: TopQuery[],
   title?: string,
-  columns: {title?: string, key: (keyof TopQuery), sortBy?: (keyof TopQuery)}[],
+  columns: TopQueryColumn[],
   defaultOrderBy?: keyof TopQuery,
 }
 const tabs = ["table", "JSON"].map((t, i) => ({

app/vmui/packages/vmui/src/pages/TopQueries/TopQueryTable/TopQueryTable.tsx

@@ -3,7 +3,7 @@ import { TopQuery } from "../../../types";
 import { getComparator, stableSort } from "../../../components/Table/helpers";
 import { TopQueryPanelProps } from "../TopQueryPanel/TopQueryPanel";
 import classNames from "classnames";
-import { ArrowDropDownIcon, CopyIcon, PlayCircleOutlineIcon } from "../../../components/Main/Icons";
+import { ArrowDropDownIcon, CopyIcon, InfoOutlinedIcon, PlayCircleOutlineIcon } from "../../../components/Main/Icons";
 import Button from "../../../components/Main/Button/Button";
 import Tooltip from "../../../components/Main/Tooltip/Tooltip";
 import { Link } from "react-router-dom";
@@ -35,26 +35,40 @@ const TopQueryTable:FC<TopQueryPanelProps> = ({ rows, columns, defaultOrderBy })
     <table className="vm-table">
       <thead className="vm-table-header">
         <tr className="vm-table__row vm-table__row_header">
-          {columns.map((col) => (
-            <th
-              className="vm-table-cell vm-table-cell_header vm-table-cell_sort"
-              onClick={createSortHandler(col.sortBy || col.key)}
-              key={col.key}
-            >
-              <div className="vm-table-cell__content">
-                {col.title || col.key}
-                <div
-                  className={classNames({
-                    "vm-table__sort-icon": true,
-                    "vm-table__sort-icon_active": orderBy === col.key,
-                    "vm-table__sort-icon_desc": orderDir === "desc" && orderBy === col.key
-                  })}
-                >
-                  <ArrowDropDownIcon/>
+          {columns.map((col) => {
+            const sortKey = col.sortBy || col.key;
+
+            return (
+              <th
+                className="vm-table-cell vm-table-cell_header vm-table-cell_sort"
+                onClick={createSortHandler(sortKey)}
+                key={col.key}
+              >
+                <div className="vm-table-cell__content">
+                  {col.title || col.key}
+                  {col.tooltip && (
+                    <Tooltip
+                      placement="top-center"
+                      title={col.tooltip}
+                    >
+                      <span className="vm-top-queries-table__info-icon">
+                        <InfoOutlinedIcon/>
+                      </span>
+                    </Tooltip>
+                  )}
+                  <div
+                    className={classNames({
+                      "vm-table__sort-icon": true,
+                      "vm-table__sort-icon_active": orderBy === sortKey,
+                      "vm-table__sort-icon_desc": orderDir === "desc" && orderBy === sortKey
+                    })}
+                  >
+                    <ArrowDropDownIcon/>
+                  </div>
                 </div>
-              </div>
-            </th>
-          ))}
+              </th>
+            );
+          })}
           <th className="vm-table-cell vm-table-cell_header"/> {/* empty cell for actions */}
         </tr>
       </thead>
@@ -69,7 +83,7 @@ const TopQueryTable:FC<TopQueryPanelProps> = ({ rows, columns, defaultOrderBy })
                 className="vm-table-cell"
                 key={col.key}
               >
-                {row[col.key] || "-"}
+                {col.format?.(row) ?? row[col.key] ?? "-"}
               </td>
             ))}
             <td className="vm-table-cell vm-table-cell_no-padding">

app/vmui/packages/vmui/src/pages/TopQueries/hooks/useFetchTopQueries.ts

@@ -34,7 +34,7 @@ const processResponse = (data: TopQueriesData) => {
     target.forEach(t => {
       const timeRange = getDurationFromMilliseconds(t.timeRangeSeconds*1000);
       t.url = getQueryUrl(t, timeRange);
-      t.timeRange = timeRange;
+      t.timeRange = timeRange || "instant";
     });
   });
 

app/vmui/packages/vmui/src/pages/TopQueries/hooks/useTopQueriesColumns.ts

@@ -0,0 +1,77 @@
+import { useMemo } from "react";
+import { TopQueryColumn } from "../TopQueryPanel/TopQueryPanel";
+import { humanizeSeconds } from "../../../utils/time";
+import { formatBytes } from "../../../utils/bytes";
+
+type UseTopQueriesColumns = {
+  maxLifetime: string;
+};
+
+export const useTopQueriesColumns = ({ maxLifetime }: UseTopQueriesColumns) => {
+  return useMemo(() => {
+    const queryCol: TopQueryColumn = {
+      key: "query"
+    };
+
+    const timeRangeCol: TopQueryColumn = {
+      key: "timeRange",
+      sortBy: "timeRangeSeconds",
+      title: "range",
+      tooltip: "The time range between start and end of the query request. 'instant' means the query was executed at a single point in time without a time range"
+    };
+
+    const countCol: TopQueryColumn = {
+      key: "count",
+      tooltip: `The number of times the query was executed over the last ${maxLifetime}`,
+    };
+
+    const topBySumDuration: TopQueryColumn[] = [
+      queryCol,
+      {
+        key: "sumDurationSeconds",
+        title: "duration",
+        tooltip: `Cumulative time spent executing the query across all its invocations over the last ${maxLifetime}`,
+        format: (row) => humanizeSeconds(row.sumDurationSeconds)
+      },
+      timeRangeCol,
+      countCol,
+    ];
+
+    const topByAvgDuration: TopQueryColumn[] = [
+      queryCol,
+      {
+        key: "avgDurationSeconds",
+        title: "duration",
+        tooltip: `Average time spent executing the query over the last ${maxLifetime}`,
+        format: (row) => humanizeSeconds(row.avgDurationSeconds)
+      },
+      timeRangeCol,
+      countCol,
+    ];
+
+    const topByCount: TopQueryColumn[] = [
+      queryCol,
+      timeRangeCol,
+      countCol,
+    ];
+
+    const topByAvgMemoryUsage: TopQueryColumn[] = [
+      queryCol,
+      {
+        key: "avgMemoryBytes",
+        title: "memory",
+        tooltip: `Average memory used during query execution over the last ${maxLifetime}`,
+        format: (row) => formatBytes(row.avgMemoryBytes)
+      },
+      timeRangeCol,
+      countCol,
+    ];
+
+    return {
+      topBySumDuration,
+      topByAvgDuration,
+      topByCount,
+      topByAvgMemoryUsage,
+    };
+  }, [maxLifetime]);
+};

app/vmui/packages/vmui/src/pages/TopQueries/index.tsx

@@ -15,6 +15,7 @@ import "./style.scss";
 import useDeviceDetect from "../../hooks/useDeviceDetect";
 import classNames from "classnames";
 import useStateSearchParams from "../../hooks/useStateSearchParams";
+import { useTopQueriesColumns } from "./hooks/useTopQueriesColumns";
 
 const exampleDuration = "30ms, 15s, 3d4h, 1y2w";
 
@@ -23,6 +24,7 @@ const TopQueries: FC = () => {
 
   const [topN, setTopN] = useStateSearchParams(10, "topN");
   const [maxLifetime, setMaxLifetime] = useStateSearchParams("10m", "maxLifetime");
+  const columns = useTopQueriesColumns({ maxLifetime });
 
   const { data, error, loading, fetch } = useFetchTopQueries({ topN, maxLifetime });
 
@@ -145,52 +147,33 @@ const TopQueries: FC = () => {
 
       {error && <Alert variant="error">{error}</Alert>}
 
-      {data && (<>
+      {data && (
         <div className="vm-top-queries-panels">
           <TopQueryPanel
+            title="Queries with most summary time to execute"
             rows={data.topBySumDuration}
-            title={"Queries with most summary time to execute"}
-            columns={[
-              { key: "query" },
-              { key: "sumDurationSeconds", title: "sum duration, sec" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
-            defaultOrderBy={"sumDurationSeconds"}
+            columns={columns.topBySumDuration}
+            defaultOrderBy="sumDurationSeconds"
           />
           <TopQueryPanel
+            title="Most heavy queries"
             rows={data.topByAvgDuration}
-            title={"Most heavy queries"}
-            columns={[
-              { key: "query" },
-              { key: "avgDurationSeconds", title: "avg duration, sec" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
-            defaultOrderBy={"avgDurationSeconds"}
+            columns={columns.topByAvgDuration}
+            defaultOrderBy="avgDurationSeconds"
           />
           <TopQueryPanel
+            title="Most frequently executed queries"
             rows={data.topByCount}
-            title={"Most frequently executed queries"}
-            columns={[
-              { key: "query" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
+            columns={columns.topByCount}
           />
           <TopQueryPanel
+            title="Queries with most memory to execute"
             rows={data.topByAvgMemoryUsage}
-            title={"Queries with most memory to execute"}
-            columns={[
-              { key: "query" },
-              { key: "avgMemoryBytes", title: "avg memory usage, bytes" },
-              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
-              { key: "count" }
-            ]}
-            defaultOrderBy={"avgMemoryBytes"}
+            columns={columns.topByAvgMemoryUsage}
+            defaultOrderBy="avgMemoryBytes"
           />
         </div>
-      </>)}
+      )}
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/TopQueries/style.scss

@@ -1,5 +1,19 @@
 @use "src/styles/variables" as *;
 
+.vm-top-queries-table {
+  &__info-icon {
+    display: inline-flex;
+    align-items: center;
+    color: $color-text-secondary;
+    margin-left: 4px;
+
+    svg {
+      width: 14px;
+      height: 14px;
+    }
+  }
+}
+
 .vm-top-queries {
   display: grid;
   align-items: flex-start;

app/vmui/packages/vmui/src/types/uplot.ts

@@ -11,6 +11,7 @@ export interface SeriesItem extends Series {
     statsFormatted: SeriesItemStatsFormatted;
     median: number;
     hasAlias?: boolean;
+    nullTimestamps?: number[];
 }
 
 export interface HideSeriesArgs {

app/vmui/packages/vmui/src/utils/bytes.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { formatBytes } from "./bytes";
+
+describe("formatBytes", () => {
+  it("returns null for invalid values", () => {
+    expect(formatBytes(-1)).toBeNull();
+    expect(formatBytes(Number.NaN)).toBeNull();
+    expect(formatBytes(Number.POSITIVE_INFINITY)).toBeNull();
+    expect(formatBytes(Number.NEGATIVE_INFINITY)).toBeNull();
+  });
+
+  it("formats zero bytes", () => {
+    expect(formatBytes(0)).toBe("0 B");
+  });
+
+  it("formats bytes", () => {
+    expect(formatBytes(0.5)).toBe("0.5 B");
+    expect(formatBytes(1)).toBe("1 B");
+    expect(formatBytes(512)).toBe("512 B");
+    expect(formatBytes(1023)).toBe("1023 B");
+  });
+
+  it("formats kilobytes", () => {
+    expect(formatBytes(1024)).toBe("1 KB");
+    expect(formatBytes(1536)).toBe("1.5 KB");
+  });
+
+  it("formats megabytes", () => {
+    expect(formatBytes(1024 ** 2)).toBe("1 MB");
+    expect(formatBytes(2.5 * 1024 ** 2)).toBe("2.5 MB");
+  });
+
+  it("formats gigabytes, terabytes and petabytes", () => {
+    expect(formatBytes(1024 ** 3)).toBe("1 GB");
+    expect(formatBytes(1024 ** 4)).toBe("1 TB");
+    expect(formatBytes(1024 ** 5)).toBe("1 PB");
+  });
+
+  it("caps values above PB to PB unit", () => {
+    expect(formatBytes(1024 ** 6)).toBe("1024 PB");
+  });
+
+  it("rounds to two decimals", () => {
+    expect(formatBytes(1234)).toBe("1.21 KB");
+    expect(formatBytes(1234567)).toBe("1.18 MB");
+  });
+});

app/vmui/packages/vmui/src/utils/bytes.ts

@@ -0,0 +1,14 @@
+const LOG_1024 = Math.log(1024);
+const UNITS = ["B", "KB", "MB", "GB", "TB", "PB"] as const;
+
+export const formatBytes = (bytes: number): string | null => {
+  if (!Number.isFinite(bytes) || bytes < 0) return null;
+  if (bytes === 0) return "0 B";
+
+  const unitIndex = Math.min(
+    Math.max(Math.floor(Math.log(bytes) / LOG_1024), 0),
+    UNITS.length - 1
+  );
+
+  return `${parseFloat((bytes / 1024 ** unitIndex).toFixed(2))} ${UNITS[unitIndex]}`;
+};

app/vmui/packages/vmui/src/utils/uplot/scatter.ts

@@ -103,6 +103,28 @@ export const drawPoints = (u: uPlot, seriesIdx: number) => {
     u.ctx.lineWidth = 1.4 * uPlot.pxRatio;
     u.ctx.strokeStyle = u.ctx.fillStyle;
     u.ctx.stroke(squaresPath);
+
+    const nullTs = (series as unknown as { nullTimestamps?: number[] }).nullTimestamps;
+    if (nullTs && nullTs.length) {
+      const xSize = BASE_POINT_SIZE * 1.4 * uPlot.pxRatio;
+      const xHalf = xSize / 2;
+      // Lift the marker by half its size so the entire icon sits inside the plot area
+      // (yMin maps to the plot's bottom edge, so centering on it would clip the lower half).
+      const cy = valToPosY(yMin, scaleY, yDim, yOff) - xHalf;
+      const xPath = new Path2D();
+      for (let i = 0; i < nullTs.length; i++) {
+        const t = nullTs[i];
+        if (t < xMin || t > xMax) continue;
+        const cx = valToPosX(t, scaleX, xDim, xOff);
+        xPath.moveTo(cx - xHalf, cy - xHalf);
+        xPath.lineTo(cx + xHalf, cy + xHalf);
+        xPath.moveTo(cx + xHalf, cy - xHalf);
+        xPath.lineTo(cx - xHalf, cy + xHalf);
+      }
+      u.ctx.lineWidth = 1.6 * uPlot.pxRatio;
+      u.ctx.strokeStyle = u.ctx.fillStyle;
+      u.ctx.stroke(xPath);
+    }
   };
 
   uPlot.orient(u, seriesIdx, orientCallback);

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -38,6 +38,7 @@ export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[],
       show: !includesHideSeries(label, hideSeries),
       scale: "1",
       paths: isRawQuery ? drawPoints : undefined,
+      nullTimestamps: d.nullTimestamps,
       ...getSeriesStatistics(d),
     };
   };

apptest/client.go

@@ -2,6 +2,7 @@ package apptest
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -13,6 +14,10 @@ import (
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
+	"github.com/golang/snappy"
 )
 
 // Client is used for interacting with the apps over the network.
@@ -147,27 +152,34 @@ func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
 	return string(b)
 }
 
-// ServesMetrics is used to retrieve the app's metrics.
+// metricsClient is used to retrieve the app's metrics.
 //
 // This type is expected to be embedded by the apps that serve metrics.
-type ServesMetrics struct {
-	metricsURL string
-	cli        *Client
+type metricsClient struct {
+	metricsCli *Client
+	url        string
+}
+
+func newMetricsClient(cli *Client, addr string) *metricsClient {
+	return &metricsClient{
+		metricsCli: cli,
+		url:        fmt.Sprintf("http://%s/metrics", addr),
+	}
 }
 
 // GetIntMetric retrieves the value of a metric served by an app at /metrics URL.
 // The value is then converted to int.
-func (app *ServesMetrics) GetIntMetric(t *testing.T, metricName string) int {
+func (c *metricsClient) GetIntMetric(t *testing.T, metricName string) int {
 	t.Helper()
 
-	return int(app.GetMetric(t, metricName))
+	return int(c.GetMetric(t, metricName))
 }
 
 // GetMetric retrieves the value of a metric served by an app at /metrics URL.
-func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
+func (c *metricsClient) GetMetric(t *testing.T, metricName string) float64 {
 	t.Helper()
 
-	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
+	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -188,12 +200,12 @@ func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
 
 // GetMetricsByPrefix retrieves the values of all metrics that start with given
 // prefix.
-func (app *ServesMetrics) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
+func (c *metricsClient) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
 	t.Helper()
 
 	values := []float64{}
 
-	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
+	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -217,12 +229,12 @@ func (app *ServesMetrics) GetMetricsByPrefix(t *testing.T, prefix string) []floa
 	return values
 }
 
-func (app *ServesMetrics) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []float64 {
+func (c *metricsClient) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []float64 {
 	t.Helper()
 
 	values := []float64{}
 
-	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
+	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -245,3 +257,756 @@ func (app *ServesMetrics) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []
 	}
 	return values
 }
+
+// rpcRowsSentTotal retrieves the values of all vminsert
+// `vm_rpc_rows_sent_total` metrics (there will be one for each vmstorage) and
+// returns their integer sum.
+func (c *metricsClient) rpcRowsSentTotal(t *testing.T) int {
+	total := 0.0
+	for _, v := range c.GetMetricsByPrefix(t, "vm_rpc_rows_sent_total") {
+		total += v
+	}
+	return int(total)
+}
+
+type vmselectClient struct {
+	vmselectCli              *Client
+	url                      func(op, path string, opts QueryOpts) string
+	metricNamesStatsResetURL string
+	tenantsURL               string
+}
+
+// PrometheusAPIV1Export is a test helper function that performs the export of
+// raw samples in JSON line format by sending a request to
+// /prometheus/api/v1/export endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
+func (c *vmselectClient) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/export", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1ExportNative is a test helper function that performs the export of
+// raw samples in native binary format by sending a request to
+// /prometheus/api/v1/export/native endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
+func (c *vmselectClient) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/export/native", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return []byte(res)
+}
+
+// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
+// instant query by sending a request to /prometheus/api/v1/query endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
+func (c *vmselectClient) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/query", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1QueryRange is a test helper function that performs
+// PromQL/MetricsQL range query by sending a request to
+// /prometheus/api/v1/query_range endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
+func (c *vmselectClient) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/query_range", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1Series retrieves list of time series that match the query by
+// sending a request to /prometheus/api/v1/series endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (c *vmselectClient) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/series", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesResponse(t, res)
+}
+
+// PrometheusAPIV1SeriesCount retrieves the total number of time series by
+// sending a request to /prometheus/api/v1/series/count endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (c *vmselectClient) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/series/count", opts)
+	values := opts.asURLValues()
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesCountResponse(t, res)
+}
+
+// PrometheusAPIV1Labels retrieves the label names for time series that match a
+// query by sending a request to /prometheus/api/v1/labels endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
+func (c *vmselectClient) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/labels", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1LabelsResponse(t, res)
+}
+
+// PrometheusAPIV1LabelValues retrieves the labels values for the metrics that
+// match the query by sending a request to /prometheus/api/v1/label/.../values
+// endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
+func (c *vmselectClient) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
+	t.Helper()
+	path := fmt.Sprintf("prometheus/api/v1/label/%s/values", labelName)
+	url := c.url("select", path, opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1LabelValuesResponse(t, res)
+}
+
+// PrometheusAPIV1Metadata retrieves metadata for the given metric by sending a
+// request to /prometheus/api/v1/metadata endpoint.
+func (c *vmselectClient) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/metadata", opts)
+	values := opts.asURLValues()
+	values.Add("metric", metric)
+	values.Add("limit", strconv.Itoa(limit))
+	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	return NewPrometheusAPIV1Metadata(t, res)
+}
+
+// PrometheusAPIV1AdminTSDBDeleteSeries deletes the series that match the query
+// by sending a request to /prometheus/api/v1/admin/tsdb/delete_series.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
+func (c *vmselectClient) PrometheusAPIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("delete", "prometheus/api/v1/admin/tsdb/delete_series", opts)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// PrometheusAPIV1StatusMetricNamesStats sends a query to
+// /prometheus/api/v1/status/metric_names_stats endpoint and returns the metric
+// usage stats response for given params.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (c *vmselectClient) PrometheusAPIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/status/metric_names_stats", opts)
+	values := opts.asURLValues()
+	values.Add("limit", limit)
+	values.Add("le", le)
+	values.Add("match_pattern", matchPattern)
+	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+	var resp MetricNamesStatsResponse
+	if err := json.Unmarshal([]byte(res), &resp); err != nil {
+		t.Fatalf("could not unmarshal metric names stats response data:\n%s\n err: %v", res, err)
+	}
+	return resp
+}
+
+// PrometheusAPIV1StatusTSDB retrieves the TSDB status for the time series that
+// match the query on the given date by sending a request to
+// /prometheus/api/v1/status/tsdb endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
+func (c *vmselectClient) PrometheusAPIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
+	t.Helper()
+	url := c.url("select", "prometheus/api/v1/status/tsdb", opts)
+	values := opts.asURLValues()
+	addNonEmpty := func(name, value string) {
+		if len(value) == 0 {
+			return
+		}
+		values.Add(name, value)
+	}
+	addNonEmpty("match[]", matchQuery)
+	addNonEmpty("topN", topN)
+	addNonEmpty("date", date)
+	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var status TSDBStatusResponse
+	if err := json.Unmarshal([]byte(res), &status); err != nil {
+		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
+	}
+	status.Sort()
+	return status
+}
+
+// GraphiteMetricsIndex retrieves the list of all metrics by sending a request
+// to /graphite/metrics/index.json endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+func (c *vmselectClient) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/metrics/index.json", opts)
+	res, statusCode := c.vmselectCli.Get(t, url, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var index GraphiteMetricsIndexResponse
+	if err := json.Unmarshal([]byte(res), &index); err != nil {
+		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
+	}
+	return index
+}
+
+// GraphiteMetricsFind finds metrics under a given path by sending a request
+// to /metrics/find endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+// and https://graphite.readthedocs.io/en/latest/metrics_api.html#metrics-find
+func (c *vmselectClient) GraphiteMetricsFind(t *testing.T, query string, opts QueryOpts) GraphiteMetricsFindResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/metrics/find", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
+	}
+
+	var res GraphiteMetricsFindResponse
+	if err := json.Unmarshal([]byte(resText), &res); err != nil {
+		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
+	}
+	return res
+}
+
+// GraphiteMetricsExpand expands the given query with matching paths by sending
+// a request to /graphite/metrics/expand endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+// and https://graphite.readthedocs.io/en/latest/metrics_api.html#metrics-expand
+func (c *vmselectClient) GraphiteMetricsExpand(t *testing.T, query string, opts QueryOpts) GraphiteMetricsExpandResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/metrics/expand", opts)
+	values := opts.asURLValues()
+	values.Add("query", query)
+	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
+	}
+
+	var res GraphiteMetricsExpandResponse
+	if err := json.Unmarshal([]byte(resText), &res); err != nil {
+		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
+	}
+	return res
+}
+
+// GraphiteRender retrieves the raw metric data by sending a request to
+// /graphite/render endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#render-api
+// and https://graphite-api.readthedocs.io/en/latest/api.html#the-render-api-render
+func (c *vmselectClient) GraphiteRender(t *testing.T, target string, opts QueryOpts) GraphiteRenderResponse {
+	t.Helper()
+
+	url := c.url("select", "graphite/render", opts)
+	values := opts.asURLValues()
+	values.Add("format", "json")
+	values.Add("target", target)
+	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
+	}
+
+	var res GraphiteRenderResponse
+	if err := json.Unmarshal([]byte(resText), &res); err != nil {
+		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
+	}
+	return res
+}
+
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a request to /graphite/tags/tagSeries
+// endpoint.
+func (c *vmselectClient) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("select", "graphite/tags/tagSeries", opts)
+	values := opts.asURLValues()
+	values.Add("path", record)
+	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+// GraphiteTagsTagMultiSeries is a test helper function that registers Graphite
+// tags for a multiple time series by sending a request to
+// /graphite/tags/tagSeries endpoint.
+func (c *vmselectClient) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("select", "graphite/tags/tagMultiSeries", opts)
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+// PrometheusAPIV1AdminStatusMetricNamesStatsReset resets the metric name usage
+// stats by sending a request to
+// /prometheus/api/v1/admin/status/metric_names_stats/reset endpoint
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (c *vmselectClient) PrometheusAPIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts QueryOpts) {
+	t.Helper()
+	values := opts.asURLValues()
+	res, statusCode := c.vmselectCli.PostForm(t, c.metricNamesStatsResetURL, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// APIV1AdminTenants retrieves the list of tenants by sending a request to
+// /admin/tenants endpoint.
+func (c *vmselectClient) APIV1AdminTenants(t *testing.T, opts QueryOpts) *AdminTenantsResponse {
+	t.Helper()
+	res, statusCode := c.vmselectCli.Get(t, c.tenantsURL, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	tenants := &AdminTenantsResponse{}
+	if err := json.Unmarshal([]byte(res), tenants); err != nil {
+		t.Fatalf("could not unmarshal tenants response data:\n%s\n err: %v", res, err)
+	}
+
+	return tenants
+}
+
+type vminsertClient struct {
+	vminsertCli        *Client
+	url                func(op, path string, opts QueryOpts) string
+	openTSDBURL        func(op, path string, opts QueryOpts) string
+	graphiteListenAddr string
+	sendBlocking       func(t *testing.T, numRecordsToSend int, send func())
+}
+
+// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
+// of records in CSV format for the given tenant by sending an HTTP POST
+// request to prometheus/api/v1/import/csv vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (c *vminsertClient) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/import/csv", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, len(records), func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
+// of records in Native format for the given tenant by sending an HTTP POST
+// request to prometheus/api/v1/import/native vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (c *vminsertClient) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/import/native", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, 1, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1Write is a test helper function that inserts a
+// collection of records in Prometheus remote-write format by sending a HTTP
+// POST request to /prometheus/api/v1/write vminsert endpoint.
+func (c *vminsertClient) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/write", opts)
+	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
+	recordsCount := len(wr.Timeseries)
+	if prommetadata.IsEnabled() {
+		recordsCount += len(wr.Metadata)
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/x-protobuf")
+	c.sendBlocking(t, recordsCount, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// collection of records in Prometheus text exposition format for the given
+// tenant by sending a HTTP POST request to
+// /prometheus/api/v1/import/prometheus vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
+func (c *vminsertClient) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "prometheus/api/v1/import/prometheus", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	var recordsCount int
+	var metadataRecords int
+	uniqueMetadataMetricNames := make(map[string]struct{})
+	for _, record := range records {
+		// metric metadata has the following format:
+		//# HELP importprometheus_series
+		//# TYPE importprometheus_series
+		// it results into single metadata record
+		if strings.HasPrefix(record, "# ") {
+			metadataItems := strings.Split(record, " ")
+			if len(metadataItems) < 3 {
+				t.Fatalf("BUG: unexpected metadata format=%q", record)
+			}
+			metricName := metadataItems[2]
+			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
+				continue
+			}
+			uniqueMetadataMetricNames[metricName] = struct{}{}
+			metadataRecords++
+			continue
+		}
+		recordsCount++
+	}
+	if prommetadata.IsEnabled() {
+		recordsCount += metadataRecords
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, recordsCount, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// InfluxWrite is a test helper function that inserts a collection of records in
+// Influx line format by sending a HTTP POST request to /influx/write endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
+func (c *vminsertClient) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "influx/write", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	c.sendBlocking(t, len(records), func() {
+		t.Helper()
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// OpentelemetryV1Metrics is a test helper function that inserts a
+// collection of records in Opentelemetry protocol format by sending a HTTP
+// POST request to /opentelemetry/v1/metrics vminsert endpoint.
+func (c *vminsertClient) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsData, opts QueryOpts) {
+	t.Helper()
+
+	var recordsCount int
+	for _, rss := range md.ResourceMetrics {
+		for _, sm := range rss.ScopeMetrics {
+			recordsCount += len(sm.Metrics)
+			for _, m := range sm.Metrics {
+				if prommetadata.IsEnabled() {
+					recordsCount += len(m.Metadata)
+				}
+			}
+		}
+	}
+	url := c.url("insert", "opentelemetry/v1/metrics", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := md.MarshalProtobuf(nil)
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/x-protobuf")
+	c.sendBlocking(t, recordsCount, func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusOK {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+		}
+	})
+}
+
+// OpenTSDBAPIPut is a test helper function that inserts a collection of
+// records in OpenTSDB format for the given tenant by sending an HTTP POST
+// request to /opentsdb/api/put vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (c *vminsertClient) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.openTSDBURL("insert", "opentsdb/api/put", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte("[" + strings.Join(records, ",") + "]")
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/json")
+	c.sendBlocking(t, len(records), func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// ZabbixConnectorHistory is a test helper function that inserts a
+// collection of records in zabbixconnector  format by sending a HTTP
+// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
+func (c *vminsertClient) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := c.url("insert", "zabbixconnector/api/v1/history", opts)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/json")
+	c.sendBlocking(t, len(records), func() {
+		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		if statusCode != http.StatusOK {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+		}
+	})
+
+}
+
+// GraphiteWrite is a test helper function that sends a
+// collection of records to graphiteListenAddr port.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
+func (c *vminsertClient) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
+	t.Helper()
+	c.vminsertCli.Write(t, c.graphiteListenAddr, records)
+}
+
+type vmstorageClient struct {
+	vmstorageCli   *Client
+	httpListenAddr string
+}
+
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
+func (c *vmstorageClient) ForceFlush(t *testing.T) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/internal/force_flush", c.httpListenAddr)
+	_, statusCode := c.vmstorageCli.Get(t, url, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// ForceMerge is a test helper function that forces the merging of parts.
+func (c *vmstorageClient) ForceMerge(t *testing.T) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/internal/force_merge", c.httpListenAddr)
+	_, statusCode := c.vmstorageCli.Get(t, url, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// SnapshotCreate creates a database snapshot by sending a query to the
+// /snapshot/create endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
+	t.Helper()
+
+	data, statusCode := c.vmstorageCli.Post(t, c.SnapshotCreateURL(), nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotCreateResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotCreateURL returns the URL for creating snapshots.
+func (c *vmstorageClient) SnapshotCreateURL() string {
+	return fmt.Sprintf("http://%s/snapshot/create", c.httpListenAddr)
+}
+
+// APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
+// /api/v1/admin/tsdb/snapshot endpoint.
+//
+// See https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot.
+func (c *vmstorageClient) APIV1AdminTSDBSnapshot(t *testing.T) *APIV1AdminTSDBSnapshotResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/api/v1/admin/tsdb/snapshot", c.httpListenAddr)
+	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res APIV1AdminTSDBSnapshotResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal prometheus snapshot create response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotList lists existing database snapshots by sending a query to the
+// /snapshot/list endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotList(t *testing.T) *SnapshotListResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/snapshot/list", c.httpListenAddr)
+	data, statusCode := c.vmstorageCli.Get(t, url, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotListResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotDelete deletes a snapshot by sending a query to the
+// /snapshot/delete endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", c.httpListenAddr, snapshotName)
+	data, statusCode := c.vmstorageCli.Delete(t, url)
+	wantStatusCodes := map[int]bool{
+		http.StatusOK:                  true,
+		http.StatusInternalServerError: true,
+	}
+	if !wantStatusCodes[statusCode] {
+		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
+	}
+
+	var res SnapshotDeleteResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotDeleteAll deletes all snapshots by sending a query to the
+// /snapshot/delete_all endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (c *vmstorageClient) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/snapshot/delete_all", c.httpListenAddr)
+	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotDeleteAllResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}

apptest/model.go

@@ -27,13 +27,16 @@ type PrometheusQuerier interface {
 	PrometheusAPIV1LabelValues(t *testing.T, labelName, query string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse
 	PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
 	PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata
-
-	APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts)
+	PrometheusAPIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse
+	PrometheusAPIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts)
 
 	// TODO(@rtm0): Prometheus does not provide this API. Either move it to a
 	// separate interface or rename this interface to allow for multiple querier
 	// types.
 	GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse
+	GraphiteMetricsFind(t *testing.T, query string, opts QueryOpts) GraphiteMetricsFindResponse
+	GraphiteMetricsExpand(t *testing.T, query string, opts QueryOpts) GraphiteMetricsExpandResponse
+	GraphiteRender(t *testing.T, target string, opts QueryOpts) GraphiteRenderResponse
 	GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts)
 	GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts)
 }
@@ -91,6 +94,9 @@ type QueryOpts struct {
 	Format         string
 	NoCache        string
 	Headers        http.Header
+	From           string
+	Until          string
+	StorageStep    string
 }
 
 func (qos *QueryOpts) getHeaders() http.Header {
@@ -123,6 +129,9 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("latency_offset", qos.LatencyOffset)
 	addNonEmpty("format", qos.Format)
 	addNonEmpty("nocache", qos.NoCache)
+	addNonEmpty("from", qos.From)
+	addNonEmpty("until", qos.Until)
+	addNonEmpty("storage_step", qos.StorageStep)
 
 	return uv
 }
@@ -480,10 +489,6 @@ type TSDBStatusResponse struct {
 	Data      TSDBStatusResponseData
 }
 
-// GraphiteMetricsIndexResponse is an in-memory representation of the json response
-// returned by the /graphite/metrics/index.json endpoint.
-type GraphiteMetricsIndexResponse = []string
-
 // AdminTenantsResponse is an in-memory representation of the json response
 // returned by the /api/v1/admin/tenants endpoint.
 type AdminTenantsResponse struct {
@@ -533,3 +538,32 @@ func sortTSDBStatusResponseEntries(entries []TSDBStatusResponseEntry) {
 		return left.Count < right.Count
 	})
 }
+
+// GraphiteMetricsIndexResponse is an in-memory representation of the json response
+// returned by the /graphite/metrics/index.json endpoint.
+type GraphiteMetricsIndexResponse = []string
+
+type GraphiteMetric struct {
+	Id            string
+	Text          string
+	AllowChildren int
+	Expandable    int
+	Leaf          int
+}
+
+// GraphiteMetricsIndexResponse is an in-memory representation of the json response
+// returned by the /graphite/metrics/find endpoint.
+type GraphiteMetricsFindResponse = []GraphiteMetric
+
+// GraphiteMetricsExpandResponse is an in-memory representation of the json response
+// returned by the /graphite/metrics/expand endpoint.
+type GraphiteMetricsExpandResponse = []string
+
+type GraphiteRenderedTarget struct {
+	Target     string
+	Datapoints [][2]float64
+}
+
+// GraphiteRenderResponse is an in-memory representation of the json response
+// returned by the /graphite/render endpoint.
+type GraphiteRenderResponse = []GraphiteRenderedTarget

apptest/tests/legacy_indexdb_test.go

@@ -191,7 +191,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - start legacy vmsingle
 	// - insert data1
-	// - confirm that metric names and samples are searcheable
+	// - confirm that metric names and samples are searchable
 	// - stop legacy vmsingle
 	const step = 24 * 3600 * 1000 // 24h
 	start1 := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
@@ -204,17 +204,17 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 	opts.stopLegacySUT()
 
 	// - start new vmsingle
-	// - confirm that data1 metric names and samples are searcheable
+	// - confirm that data1 metric names and samples are searchable
 	// - delete data1
-	// - confirm that data1 metric names and samples are not searcheable anymore
+	// - confirm that data1 metric names and samples are not searchable anymore
 	// - insert data2 (same metric names, different dates)
-	// - confirm that metric names become searcheable again
-	// - confirm that data1 samples are not searchable and data2 samples are searcheable
+	// - confirm that metric names become searchable again
+	// - confirm that data1 samples are not searchable and data2 samples are searchable
 
 	newSUT := opts.startNewSUT()
 	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
 
-	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	newSUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	wantNoResults := &want{
 		series:       []map[string]string{},
 		queryResults: []*at.QueryResult{},
@@ -230,7 +230,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - restart new vmsingle
 	// - confirm that metric names still searchable, data1 samples are not
-	//   searchable, and data2 samples are searcheable
+	//   searchable, and data2 samples are searchable
 
 	opts.stopNewSUT()
 	newSUT = opts.startNewSUT()
@@ -877,7 +877,7 @@ func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
 	// Ingest legacy2 records, ensure the queries return only legacy2.
 	legacySUT = opts.startLegacySUT()
 	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1, numMetrics)
-	legacySUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	legacySUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	assertQueries(legacySUT, `{__name__=~".*"}`, wantEmpty, numMetrics)
 	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy2Data, at.QueryOpts{})
 	legacySUT.ForceFlush(t)
@@ -891,7 +891,7 @@ func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
 	newSUT = opts.startNewSUT()
 	// series count includes deleted metrics
 	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy2New1, 3*numMetrics)
-	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	newSUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	// series count includes deleted metrics
 	assertQueries(newSUT, `{__name__=~".*"}`, wantEmpty, 3*numMetrics)
 	opts.stopNewSUT()

apptest/tests/metric_names_stats_test.go

@@ -48,7 +48,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3"},
 		},
 	}
-	got := sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got := sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -63,7 +63,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -90,7 +90,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 		},
 	}
 	expectedStatsResponse.Sort()
-	gotStatus := sut.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{})
+	gotStatus := sut.PrometheusAPIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
 		t.Errorf("unexpected APIV1StatusTSDB response (-want, +got):\n%s", diff)
 	}
@@ -105,7 +105,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -118,17 +118,17 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "2", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "2", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
 	// reset state and check empty request response
-	sut.APIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
+	sut.PrometheusAPIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
 	expected = apptest.MetricNamesStatsResponse{
 		Records: []apptest.MetricNamesStatsRecord{},
 	}
-	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -158,7 +158,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 		fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VmselectAddr(), vmstorage2.VmselectAddr()),
 	})
 	// verify empty stats
-	resp := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "0:0"})
+	resp := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "0:0"})
 	if len(resp.Records) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Records), 0)
 	}
@@ -198,7 +198,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 				{MetricName: "metric_name_3"},
 			},
 		}
-		gotStats := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
+		gotStats := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expected, gotStats); diff != "" {
 			t.Errorf("unexpected response (-want, +got):\n%s", diff)
 		}
@@ -216,7 +216,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 				{MetricName: "metric_name_1", QueryRequestsCount: 3},
 			},
 		}
-		gotStats = vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
+		gotStats = vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expected, gotStats); diff != "" {
 			t.Errorf("unexpected response tenant: %s (-want, +got):\n%s", tenantID, diff)
 		}
@@ -243,9 +243,9 @@ func TestClusterMetricNamesStats(t *testing.T) {
 			},
 		}
 		expectedStatsResponse.Sort()
-		gotStatus := vmselect.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{Tenant: tenantID})
+		gotStatus := vmselect.PrometheusAPIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
-			t.Errorf("unexpected APIV1StatusTSDB response tenant: %s (-want, +got):\n%s", tenantID, diff)
+			t.Errorf("unexpected TSDB status for tenant %s (-want, +got):\n%s", tenantID, diff)
 		}
 	}
 
@@ -258,14 +258,14 @@ func TestClusterMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_1", QueryRequestsCount: 9},
 		},
 	}
-	gotStats := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
+	gotStats := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
 	if diff := cmp.Diff(expected, gotStats); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
 	// reset cache and check empty state
-	vmselect.MetricNamesStatsReset(t, apptest.QueryOpts{})
-	resp = vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
+	vmselect.PrometheusAPIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
+	resp = vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
 	if len(resp.Records) != 0 {
 		t.Fatalf("want 0 records, got: %d", len(resp.Records))
 	}

apptest/tests/multitenancy_via_headers_test.go

@@ -219,7 +219,7 @@ func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
 	tenantID := make(http.Header)
 	tenantID.Set("AccountID", "5")
 	tenantID.Set("ProjectID", "15")
-	vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
 		Headers: tenantID,
 	})
 	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
@@ -244,7 +244,7 @@ func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
 	}
 
 	// Delete series for multitenant with tenant filter
-	vmselect.APIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
 		Headers: multitenant,
 	})
 

apptest/tests/multitenant_test.go

@@ -192,7 +192,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	}
 
 	// Delete series from specific tenant
-	vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "5:15",
 	})
 	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
@@ -215,7 +215,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	}
 
 	// Delete series for multitenant with tenant filter
-	vmselect.APIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
 		Tenant: "multitenant",
 	})
 

apptest/vmagent.go

@@ -19,9 +19,11 @@ import (
 // Vmagent holds the state of a vmagent app and provides vmagent-specific functions
 type Vmagent struct {
 	*app
-	*ServesMetrics
+	*metricsClient
 
 	httpListenAddr string
+
+	cli *Client
 }
 
 // StartVmagent starts an instance of vmagent with the given flags. It also
@@ -46,12 +48,10 @@ func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfig
 	}
 
 	return &Vmagent{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
-		},
+		app:            app,
+		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
 		httpListenAddr: stderrExtracts[0],
+		cli:            cli,
 	}, nil
 }
 

apptest/vmauth.go

@@ -1,7 +1,6 @@
 package apptest
 
 import (
-	"fmt"
 	"io"
 	"regexp"
 	"syscall"
@@ -17,7 +16,7 @@ var httpBuilitinListenAddrRE = regexp.MustCompile(`pprof handlers are exposed at
 // functions.
 type Vmauth struct {
 	*app
-	*ServesMetrics
+	*metricsClient
 
 	httpListenAddr string
 	configFilePath string
@@ -45,11 +44,8 @@ func StartVmauth(instance string, flags []string, cli *Client, configFilePath st
 	}
 
 	return &Vmauth{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
-		},
+		app:            app,
+		metricsClient:  newMetricsClient(cli, stderrExtracts[0]),
 		httpListenAddr: stderrExtracts[0],
 		configFilePath: configFilePath,
 		cli:            cli,

apptest/vminsert.go

@@ -3,31 +3,21 @@ package apptest
 import (
 	"fmt"
 	"io"
-	"net/http"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
-
-	"github.com/golang/snappy"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 )
 
 // Vminsert holds the state of a vminsert app and provides vminsert-specific
 // functions.
 type Vminsert struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vminsertClient
 
 	httpListenAddr          string
 	clusternativeListenAddr string
-	graphiteListenAddr      string
-	openTSDBListenAddr      string
-
-	cli *Client
 }
 
 // storageNodes returns the storage node addresses passed to vminsert via
@@ -73,17 +63,26 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
+	metricsClient := newMetricsClient(cli, stderrExtracts[0])
 	return &Vminsert{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
+		app:           app,
+		metricsClient: metricsClient,
+		vminsertClient: &vminsertClient{
+			vminsertCli: cli,
+			url: func(op, path string, opts QueryOpts) string {
+				return getClusterPath(stderrExtracts[0], op, path, opts)
+			},
+			openTSDBURL: func(op, path string, opts QueryOpts) string {
+				return getClusterPath(stderrExtracts[3], op, path, opts)
+			},
+			graphiteListenAddr: stderrExtracts[2],
+			sendBlocking: func(t *testing.T, numRecordsToSend int, send func()) {
+				t.Helper()
+				sendBlocking(t, metricsClient, numRecordsToSend, send)
+			},
 		},
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
-		graphiteListenAddr:      stderrExtracts[2],
-		openTSDBListenAddr:      stderrExtracts[3],
-		cli:                     cli,
 	}, nil
 }
 
@@ -99,247 +98,6 @@ func (app *Vminsert) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
-// InfluxWrite is a test helper function that inserts a
-// collection of records in Influx line format by sending a HTTP
-// POST request to /influx/write vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
-func (app *Vminsert) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "influx/write", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// GraphiteWrite is a test helper function that sends a
-// collection of records to graphiteListenAddr port.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
-func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
-	t.Helper()
-	app.cli.Write(t, app.graphiteListenAddr, records)
-}
-
-// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
-// of records in CSV format for the given tenant by sending an HTTP POST
-// request to prometheus/api/v1/import/csv vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/csv", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
-// of records in Native format for the given tenant by sending an HTTP POST
-// request to prometheus/api/v1/import/native vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/native", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, 1, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// OpenTSDBAPIPut is a test helper function that inserts a collection of
-// records in OpenTSDB format for the given tenant by sending an HTTP POST
-// request to /opentsdb/api/put vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.openTSDBListenAddr, "insert", "opentsdb/api/put", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte("[" + strings.Join(records, ",") + "]")
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1Write is a test helper function that inserts a
-// collection of records in Prometheus remote-write format by sending a HTTP
-// POST request to /prometheus/api/v1/write vminsert endpoint.
-func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/write", opts)
-	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
-	recordsCount := len(wr.Timeseries)
-	if prommetadata.IsEnabled() {
-		recordsCount += len(wr.Metadata)
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	app.sendBlocking(t, recordsCount, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
-// collection of records in Prometheus text exposition format for the given
-// tenant by sending a HTTP POST request to
-// /prometheus/api/v1/import/prometheus vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
-func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "prometheus/api/v1/import/prometheus", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	var recordsCount int
-	var metadataRecords int
-	uniqueMetadataMetricNames := make(map[string]struct{})
-	for _, record := range records {
-		// metric metadata has the following format:
-		//# HELP importprometheus_series
-		//# TYPE importprometheus_series
-		// it results into single metadata record
-		if strings.HasPrefix(record, "# ") {
-			metadataItems := strings.Split(record, " ")
-			if len(metadataItems) < 2 {
-				t.Fatalf("BUG: unexpected metadata format=%q", record)
-			}
-			metricName := metadataItems[2]
-			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
-				continue
-			}
-			uniqueMetadataMetricNames[metricName] = struct{}{}
-			metadataRecords++
-			continue
-		}
-		recordsCount++
-	}
-	if prommetadata.IsEnabled() {
-		recordsCount += metadataRecords
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	app.sendBlocking(t, recordsCount, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// ZabbixConnectorHistory is a test helper function that inserts a
-// collection of records in zabbixconnector  format by sending a HTTP
-// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
-func (app *Vminsert) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "insert", "zabbixconnector/api/v1/history", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	app.sendBlocking(t, len(records), func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusOK {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-		}
-	})
-
-}
-
-// OpentelemetryV1Metrics is a test helper function that inserts a
-// collection of records in Opentelemetry protocol format by sending a HTTP
-// POST request to /opentelemetry/v1/metrics vminsert endpoint.
-func (app *Vminsert) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsData, opts QueryOpts) {
-	t.Helper()
-
-	var recordsCount int
-	for _, rss := range md.ResourceMetrics {
-		for _, sm := range rss.ScopeMetrics {
-			recordsCount += len(sm.Metrics)
-			for _, m := range sm.Metrics {
-				if prommetadata.IsEnabled() {
-					recordsCount += len(m.Metadata)
-				}
-			}
-		}
-	}
-	url := getClusterPath(app.httpListenAddr, "insert", "opentelemetry/v1/metrics", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := md.MarshalProtobuf(nil)
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	app.sendBlocking(t, recordsCount, func() {
-		_, statusCode := app.cli.Post(t, url, data, headers)
-		if statusCode != http.StatusOK {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-		}
-	})
-}
-
 // String returns the string representation of the vminsert app state.
 func (app *Vminsert) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)
@@ -355,13 +113,10 @@ func (app *Vminsert) String() string {
 // Waiting is implemented a retrieving the value of `vm_rpc_rows_sent_total`
 // metric and checking whether it is equal or greater than the wanted value.
 // If it is, then the data has been sent to vmstorage.
-//
-// Unreliable if the records are inserted concurrently.
-// TODO(rtm0): Put sending and waiting into a critical section to make reliable?
-func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
+func sendBlocking(t *testing.T, c *metricsClient, numRecordsToSend int, send func()) {
 	t.Helper()
 
-	wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
+	wantRowsSentCount := c.rpcRowsSentTotal(t) + numRecordsToSend
 
 	send()
 
@@ -370,7 +125,7 @@ func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func(
 		period  = 100 * time.Millisecond
 	)
 	for range retries {
-		d := app.rpcRowsSentTotal(t)
+		d := c.rpcRowsSentTotal(t)
 		if d >= wantRowsSentCount {
 			return
 		}
@@ -378,14 +133,3 @@ func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func(
 	}
 	t.Fatalf("timed out while waiting for inserted rows to be sent to vmstorage")
 }
-
-// rpcRowsSentTotal retrieves the values of all vminsert
-// `vm_rpc_rows_sent_total` metrics (there will be one for each vmstorage) and
-// returns their integer sum.
-func (app *Vminsert) rpcRowsSentTotal(t *testing.T) int {
-	total := 0.0
-	for _, v := range app.GetMetricsByPrefix(t, "vm_rpc_rows_sent_total") {
-		total += v
-	}
-	return int(total)
-}

apptest/vmselect.go

@@ -1,20 +1,17 @@
 package apptest
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"regexp"
-	"strconv"
-	"testing"
 )
 
 // Vmselect holds the state of a vmselect app and provides vmselect-specific
 // functions.
 type Vmselect struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vmselectClient
 
 	httpListenAddr          string
 	clusternativeListenAddr string
@@ -41,10 +38,15 @@ func StartVmselect(instance string, flags []string, cli *Client, output io.Write
 	}
 
 	return &Vmselect{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
-			cli:        cli,
+		app:           app,
+		metricsClient: newMetricsClient(cli, stderrExtracts[0]),
+		vmselectClient: &vmselectClient{
+			vmselectCli: cli,
+			url: func(op, path string, opts QueryOpts) string {
+				return getClusterPath(stderrExtracts[0], op, path, opts)
+			},
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[0]),
+			tenantsURL:               fmt.Sprintf("http://%s/admin/tenants", stderrExtracts[0]),
 		},
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
@@ -64,299 +66,6 @@ func (app *Vmselect) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
-// PrometheusAPIV1Export is a test helper function that performs the export of
-// raw samples in JSON line format by sending a HTTP POST request to
-// /prometheus/api/v1/export vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
-func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-	res, _ := app.cli.PostForm(t, exportURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1ExportNative is a test helper function that performs the export of
-// raw samples in native binary format by sending an HTTP POST request to
-// /prometheus/api/v1/export/native vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
-func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
-	t.Helper()
-
-	exportURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/export/native", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-	res, _ := app.cli.PostForm(t, exportURL, values, opts.Headers)
-	return []byte(res)
-}
-
-// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
-// instant query by sending a HTTP POST request to /prometheus/api/v1/query
-// vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
-func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1QueryRange is a test helper function that performs
-// PromQL/MetricsQL range query by sending a HTTP POST request to
-// /prometheus/api/v1/query_range vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
-func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/query_range", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
-// and returns the list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
-	t.Helper()
-
-	seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesResponse(t, res)
-}
-
-// PrometheusAPIV1SeriesCount sends a query to a /prometheus/api/v1/series/count endpoint
-// and returns the total number of time series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmselect) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
-	t.Helper()
-
-	seriesURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/series/count", opts)
-	values := opts.asURLValues()
-
-	res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesCountResponse(t, res)
-}
-
-// PrometheusAPIV1Labels sends a query to a /prometheus/api/v1/labels endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
-func (app *Vmselect) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/labels", opts)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelsResponse(t, res)
-}
-
-// PrometheusAPIV1LabelValues sends a query to a /prometheus/api/v1/label/.../values endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
-func (app *Vmselect) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	suffix := fmt.Sprintf("prometheus/api/v1/label/%s/values", labelName)
-	queryURL := getClusterPath(app.httpListenAddr, "select", suffix, opts)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelValuesResponse(t, res)
-}
-
-// PrometheusAPIV1Metadata sends a query to a /prometheus/api/v1/metadata endpoint
-// and returns the results.
-func (app *Vmselect) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("metric", metric)
-	values.Add("limit", strconv.Itoa(limit))
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/metadata", opts)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1Metadata(t, res)
-}
-
-// APIV1AdminTSDBDeleteSeries deletes the series that match the query by sending
-// a request to /api/v1/admin/tsdb/delete_series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
-func (app *Vmselect) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
-	t.Helper()
-
-	queryURL := getClusterPath(app.httpListenAddr, "delete", "prometheus/api/v1/admin/tsdb/delete_series", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// MetricNamesStats sends a query to a /select/tenant/prometheus/api/v1/status/metric_names_stats endpoint
-// and returns the statistics response for given params.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmselect) MetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("limit", limit)
-	values.Add("le", le)
-	values.Add("match_pattern", matchPattern)
-	queryURL := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/metric_names_stats", opts)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-	var resp MetricNamesStatsResponse
-	if err := json.Unmarshal([]byte(res), &resp); err != nil {
-		t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", res, err)
-	}
-	return resp
-}
-
-// MetricNamesStatsReset sends a query to a /admin/api/v1/status/metric_names_stats/reset endpoint
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmselect) MetricNamesStatsReset(t *testing.T, opts QueryOpts) {
-	t.Helper()
-
-	values := opts.asURLValues()
-	queryURL := fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", app.httpListenAddr)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
-// //
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
-func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "prometheus/api/v1/status/tsdb", opts)
-	values := opts.asURLValues()
-	addNonEmpty := func(name, value string) {
-		if len(value) == 0 {
-			return
-		}
-		values.Add(name, value)
-	}
-	addNonEmpty("match[]", matchQuery)
-	addNonEmpty("topN", topN)
-	addNonEmpty("date", date)
-
-	res, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var status TSDBStatusResponse
-	if err := json.Unmarshal([]byte(res), &status); err != nil {
-		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
-	}
-	status.Sort()
-	return status
-}
-
-// GraphiteMetricsIndex sends a query to a /graphite/metrics/index.json
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
-func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "graphite/metrics/index.json", opts)
-	res, statusCode := app.cli.Get(t, url, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var index GraphiteMetricsIndexResponse
-	if err := json.Unmarshal([]byte(res), &index); err != nil {
-		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
-	}
-	return index
-}
-
-// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
-// for a single time series by sending a HTTP POST request to
-// /graphite/tags/tagSeries vmsingle endpoint.
-func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagSeries", opts)
-	values := opts.asURLValues()
-	values.Add("path", record)
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-func (app *Vmselect) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := getClusterPath(app.httpListenAddr, "select", "graphite/tags/tagMultiSeries", opts)
-	values := opts.asURLValues()
-	for _, rec := range records {
-		values.Add("path", rec)
-	}
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-// APIV1AdminTenants sends a query to a /admin/tenants endpoint
-func (app *Vmselect) APIV1AdminTenants(t *testing.T) *AdminTenantsResponse {
-	t.Helper()
-
-	tenantsURL := fmt.Sprintf("http://%s/admin/tenants", app.httpListenAddr)
-	res, statusCode := app.cli.Get(t, tenantsURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var tenants *AdminTenantsResponse
-	if err := json.Unmarshal([]byte(res), tenants); err != nil {
-		t.Fatalf("could not unmarshal tenants response data:\n%s\n err: %v", res, err)
-	}
-
-	return tenants
-}
-
 // String returns the string representation of the vmselect app state.
 func (app *Vmselect) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)

apptest/vmsingle.go

@@ -1,49 +1,25 @@
 package apptest
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"regexp"
-	"strconv"
-	"strings"
 	"testing"
 	"time"
-
-	"github.com/golang/snappy"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 )
 
 // Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
 // functions.
 type Vmsingle struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vmstorageClient
+	*vmselectClient
+	*vminsertClient
 
 	storageDataPath string
 	httpListenAddr  string
-
-	// vmstorage URLs.
-	forceFlushURL string
-	forceMergeURL string
-
-	// vminsert URLs.
-	influxLineWriteURL                 string
-	graphiteWriteAddr                  string
-	openTSDBHTTPURL                    string
-	prometheusAPIV1ImportPrometheusURL string
-	prometheusAPIV1WriteURL            string
-
-	// vmselect URLs.
-	prometheusAPIV1ExportURL       string
-	prometheusAPIV1ExportNativeURL string
-	prometheusAPIV1QueryURL        string
-	prometheusAPIV1QueryRangeURL   string
-	prometheusAPIV1SeriesURL       string
 }
 
 // StartVmsingleAt starts an instance of vmsingle with the given flags. It also
@@ -70,617 +46,39 @@ func StartVmsingleAt(instance, binary string, flags []string, cli *Client, outpu
 	}
 
 	return &Vmsingle{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
-			cli:        cli,
+		app:           app,
+		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
+		vmstorageClient: &vmstorageClient{
+			vmstorageCli:   cli,
+			httpListenAddr: stderrExtracts[1],
+		},
+		vmselectClient: &vmselectClient{
+			vmselectCli: cli,
+			url: func(op, path string, opts QueryOpts) string {
+				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+			},
+			metricNamesStatsResetURL: fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", stderrExtracts[1]),
+			tenantsURL:               "vmsingle-does-not-serve-tenants",
+		},
+		vminsertClient: &vminsertClient{
+			vminsertCli: cli,
+			url: func(_, path string, _ QueryOpts) string {
+				return fmt.Sprintf("http://%s/%s", stderrExtracts[1], path)
+			},
+			openTSDBURL: func(_, path string, _ QueryOpts) string {
+				return fmt.Sprintf("http://%s/%s", stderrExtracts[3], path)
+			},
+			graphiteListenAddr: stderrExtracts[2],
+			sendBlocking: func(t *testing.T, _ int, send func()) {
+				t.Helper()
+				send()
+			},
 		},
 		storageDataPath: stderrExtracts[0],
 		httpListenAddr:  stderrExtracts[1],
-
-		forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
-		forceMergeURL: fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]),
-
-		influxLineWriteURL:                 fmt.Sprintf("http://%s/influx/write", stderrExtracts[1]),
-		graphiteWriteAddr:                  stderrExtracts[2],
-		openTSDBHTTPURL:                    fmt.Sprintf("http://%s", stderrExtracts[3]),
-		prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
-		prometheusAPIV1WriteURL:            fmt.Sprintf("http://%s/prometheus/api/v1/write", stderrExtracts[1]),
-		prometheusAPIV1ExportURL:           fmt.Sprintf("http://%s/prometheus/api/v1/export", stderrExtracts[1]),
-		prometheusAPIV1ExportNativeURL:     fmt.Sprintf("http://%s/prometheus/api/v1/export/native", stderrExtracts[1]),
-		prometheusAPIV1QueryURL:            fmt.Sprintf("http://%s/prometheus/api/v1/query", stderrExtracts[1]),
-		prometheusAPIV1QueryRangeURL:       fmt.Sprintf("http://%s/prometheus/api/v1/query_range", stderrExtracts[1]),
-		prometheusAPIV1SeriesURL:           fmt.Sprintf("http://%s/prometheus/api/v1/series", stderrExtracts[1]),
 	}, nil
 }
 
-// ForceFlush is a test helper function that forces the flushing of inserted
-// data, so it becomes available for searching immediately.
-func (app *Vmsingle) ForceFlush(t *testing.T) {
-	t.Helper()
-
-	_, statusCode := app.cli.Get(t, app.forceFlushURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// ForceMerge is a test helper function that forces the merging of parts.
-func (app *Vmsingle) ForceMerge(t *testing.T) {
-	t.Helper()
-
-	_, statusCode := app.cli.Get(t, app.forceMergeURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// InfluxWrite is a test helper function that inserts a
-// collection of records in Influx line format by sending a HTTP
-// POST request to /influx/write vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
-func (app *Vmsingle) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	data := []byte(strings.Join(records, "\n"))
-
-	url := app.influxLineWriteURL
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// GraphiteWrite is a test helper function that sends a collection of records
-// to graphiteListenAddr port.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
-func (app *Vmsingle) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
-	t.Helper()
-	app.cli.Write(t, app.graphiteWriteAddr, records)
-}
-
-// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
-// of records in CSV format for the given tenant by sending an HTTP POST
-// request to /api/v1/import/csv vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-import-csv-data
-func (app *Vmsingle) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/api/v1/import/csv", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
-// of records in native format for the given tenant by sending an HTTP POST
-// request to /api/v1/import/native vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-native-format
-func (app *Vmsingle) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/api/v1/import/native", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// OpenTSDBAPIPut is a test helper function that inserts a collection of
-// records in OpenTSDB format for the given tenant by sending an HTTP POST
-// request to /api/put vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http
-func (app *Vmsingle) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	// add extra label
-	url := app.openTSDBHTTPURL + "/api/put"
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte("[" + strings.Join(records, ",") + "]")
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1Write is a test helper function that inserts a
-// collection of records in Prometheus remote-write format by sending a HTTP
-// POST request to /prometheus/api/v1/write vmsingle endpoint.
-func (app *Vmsingle) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
-	t.Helper()
-
-	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	_, statusCode := app.cli.Post(t, app.prometheusAPIV1WriteURL, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
-// collection of records in Prometheus text exposition format by sending a HTTP
-// POST request to /prometheus/api/v1/import/prometheus vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
-func (app *Vmsingle) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	// add extra label
-	url := app.prometheusAPIV1ImportPrometheusURL
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	data := []byte(strings.Join(records, "\n"))
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-	}
-}
-
-// PrometheusAPIV1Export is a test helper function that performs the export of
-// raw samples in JSON line format by sending a HTTP POST request to
-// /prometheus/api/v1/export vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
-func (app *Vmsingle) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1ExportURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1ExportNative is a test helper function that performs the export of
-// raw samples in native binary format by sending an HTTP POST request to
-// /prometheus/api/v1/export/native vmselect endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
-func (app *Vmsingle) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
-	t.Helper()
-
-	t.Helper()
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1ExportNativeURL, values, opts.Headers)
-	return []byte(res)
-}
-
-// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
-// instant query by sending a HTTP POST request to /prometheus/api/v1/query
-// vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
-func (app *Vmsingle) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("query", query)
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1QueryRange is a test helper function that performs
-// PromQL/MetricsQL range query by sending a HTTP POST request to
-// /prometheus/api/v1/query_range vmsingle endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
-func (app *Vmsingle) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("query", query)
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryRangeURL, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
-// and returns the list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmsingle) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, _ := app.cli.PostForm(t, app.prometheusAPIV1SeriesURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesResponse(t, res)
-}
-
-// PrometheusAPIV1SeriesCount sends a query to a /prometheus/api/v1/series/count endpoint
-// and returns the total number of time series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (app *Vmsingle) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/series/count", app.httpListenAddr)
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesCountResponse(t, res)
-}
-
-// PrometheusAPIV1Labels sends a query to a /prometheus/api/v1/labels endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
-func (app *Vmsingle) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/labels", app.httpListenAddr)
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelsResponse(t, res)
-}
-
-// PrometheusAPIV1LabelValues sends a query to a /prometheus/api/v1/label/.../values endpoint
-// and returns the label names list of time series that match the query.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
-func (app *Vmsingle) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/label/%s/values", app.httpListenAddr, labelName)
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1LabelValuesResponse(t, res)
-}
-
-// PrometheusAPIV1Metadata sends a query to a /prometheus/api/v1/metadata endpoint
-// and returns the results.
-func (app *Vmsingle) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("metric", metric)
-	values.Add("limit", strconv.Itoa(limit))
-	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/metadata", app.httpListenAddr)
-
-	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	return NewPrometheusAPIV1Metadata(t, res)
-}
-
-// APIV1AdminTSDBDeleteSeries deletes the series that match the query by sending
-// a request to /api/v1/admin/tsdb/delete_series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
-func (app *Vmsingle) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/api/v1/admin/tsdb/delete_series", app.httpListenAddr)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// GraphiteMetricsIndex sends a query to a /metrics/index.json
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
-func (app *Vmsingle) GraphiteMetricsIndex(t *testing.T, _ QueryOpts) GraphiteMetricsIndexResponse {
-	t.Helper()
-
-	seriesURL := fmt.Sprintf("http://%s/metrics/index.json", app.httpListenAddr)
-	res, statusCode := app.cli.Get(t, seriesURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var index GraphiteMetricsIndexResponse
-	if err := json.Unmarshal([]byte(res), &index); err != nil {
-		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
-	}
-	return index
-}
-
-// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
-// for a single time series by sending a HTTP POST request to
-// /graphite/tags/tagSeries vmsingle endpoint.
-func (app *Vmsingle) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/graphite/tags/tagSeries", app.httpListenAddr)
-	values := opts.asURLValues()
-	values.Add("path", record)
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-func (app *Vmsingle) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/graphite/tags/tagMultiSeries", app.httpListenAddr)
-	values := opts.asURLValues()
-	for _, rec := range records {
-		values.Add("path", rec)
-	}
-
-	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-// APIV1StatusMetricNamesStats sends a query to a /api/v1/status/metric_names_stats endpoint
-// and returns the statistics response for given params.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmsingle) APIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
-	t.Helper()
-
-	values := opts.asURLValues()
-	values.Add("limit", limit)
-	values.Add("le", le)
-	values.Add("match_pattern", matchPattern)
-	queryURL := fmt.Sprintf("http://%s/api/v1/status/metric_names_stats", app.httpListenAddr)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-	var resp MetricNamesStatsResponse
-	if err := json.Unmarshal([]byte(res), &resp); err != nil {
-		t.Fatalf("could not unmarshal metric names stats response data:\n%s\n err: %v", res, err)
-	}
-	return resp
-}
-
-// APIV1AdminStatusMetricNamesStatsReset sends a query to a /api/v1/admin/status/metric_names_stats/reset endpoint
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (app *Vmsingle) APIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts QueryOpts) {
-	t.Helper()
-
-	values := opts.asURLValues()
-	queryURL := fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", app.httpListenAddr)
-
-	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// SnapshotCreate creates a database snapshot by sending a query to the
-// /snapshot/create endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
-	t.Helper()
-
-	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotCreateResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotCreateURL returns the URL for creating snapshots.
-func (app *Vmsingle) SnapshotCreateURL() string {
-	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-}
-
-// APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
-// /api/v1/admin/tsdb/snapshot endpoint.
-//
-// See https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot.
-func (app *Vmsingle) APIV1AdminTSDBSnapshot(t *testing.T) *APIV1AdminTSDBSnapshotResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/api/v1/admin/tsdb/snapshot", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res APIV1AdminTSDBSnapshotResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal prometheus snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotList lists existing database snapshots by sending a query to the
-// /snapshot/list endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotList(t *testing.T) *SnapshotListResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/list", app.httpListenAddr)
-	data, statusCode := app.cli.Get(t, queryURL, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotListResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDelete deletes a snapshot by sending a query to the
-// /snapshot/delete endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", app.httpListenAddr, snapshotName)
-	data, statusCode := app.cli.Delete(t, queryURL)
-	wantStatusCodes := map[int]bool{
-		http.StatusOK:                  true,
-		http.StatusInternalServerError: true,
-	}
-	if !wantStatusCodes[statusCode] {
-		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
-	}
-
-	var res SnapshotDeleteResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDeleteAll deletes all snapshots by sending a query to the
-// /snapshot/delete_all endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmsingle) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete_all", app.httpListenAddr)
-	data, statusCode := app.cli.Get(t, queryURL, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotDeleteAllResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
-// //
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
-func (app *Vmsingle) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
-	t.Helper()
-
-	seriesURL := fmt.Sprintf("http://%s/prometheus/api/v1/status/tsdb", app.httpListenAddr)
-	values := opts.asURLValues()
-	addNonEmpty := func(name, value string) {
-		if len(value) == 0 {
-			return
-		}
-		values.Add(name, value)
-	}
-	addNonEmpty("match[]", matchQuery)
-	addNonEmpty("topN", topN)
-	addNonEmpty("date", date)
-
-	res, statusCode := app.cli.PostForm(t, seriesURL, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var status TSDBStatusResponse
-	if err := json.Unmarshal([]byte(res), &status); err != nil {
-		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
-	}
-	status.Sort()
-	return status
-}
-
-// ZabbixConnectorHistory is a test helper function that inserts a
-// collection of records in zabbixconnector  format by sending a HTTP
-// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
-func (app *Vmsingle) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/zabbixconnector/api/v1/history", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// OpentelemetryV1Metrics is a test helper function that inserts a
-// collection of records in Opentelemetry protocol format by sending a HTTP
-// POST request to /opentelemetry/v1/metrics vmsingle endpoint.
-func (app *Vmsingle) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsData, opts QueryOpts) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/opentelemetry/v1/metrics", app.httpListenAddr)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := md.MarshalProtobuf(nil)
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	_, statusCode := app.cli.Post(t, url, data, headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
 // HTTPAddr returns the address at which the vminsert process is
 // listening for incoming HTTP requests.
 func (app *Vmsingle) HTTPAddr() string {

apptest/vmstorage.go

@@ -1,13 +1,10 @@
 package apptest
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"regexp"
-	"testing"
 	"time"
 )
 
@@ -15,7 +12,8 @@ import (
 // functions.
 type Vmstorage struct {
 	*app
-	*ServesMetrics
+	*metricsClient
+	*vmstorageClient
 
 	storageDataPath string
 	httpListenAddr  string
@@ -47,10 +45,11 @@ func StartVmstorageAt(instance, binary string, flags []string, cli *Client, outp
 	}
 
 	return &Vmstorage{
-		app: app,
-		ServesMetrics: &ServesMetrics{
-			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
-			cli:        cli,
+		app:           app,
+		metricsClient: newMetricsClient(cli, stderrExtracts[1]),
+		vmstorageClient: &vmstorageClient{
+			vmstorageCli:   cli,
+			httpListenAddr: stderrExtracts[1],
 		},
 		storageDataPath: stderrExtracts[0],
 		httpListenAddr:  stderrExtracts[1],
@@ -71,121 +70,6 @@ func (app *Vmstorage) VmselectAddr() string {
 	return app.vmselectAddr
 }
 
-// ForceFlush is a test helper function that forces the flushing of inserted
-// data, so it becomes available for searching immediately.
-func (app *Vmstorage) ForceFlush(t *testing.T) {
-	t.Helper()
-
-	forceFlushURL := fmt.Sprintf("http://%s/internal/force_flush", app.httpListenAddr)
-	_, statusCode := app.cli.Get(t, forceFlushURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// ForceMerge is a test helper function that forces the merging of parts.
-func (app *Vmstorage) ForceMerge(t *testing.T) {
-	t.Helper()
-
-	forceMergeURL := fmt.Sprintf("http://%s/internal/force_merge", app.httpListenAddr)
-	_, statusCode := app.cli.Get(t, forceMergeURL, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// SnapshotCreate creates a database snapshot by sending a query to the
-// /snapshot/create endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
-	t.Helper()
-
-	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotCreateResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotCreateURL returns the URL for creating snapshots.
-func (app *Vmstorage) SnapshotCreateURL() string {
-	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-}
-
-// SnapshotList lists existing database snapshots by sending a query to the
-// /snapshot/list endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotList(t *testing.T) *SnapshotListResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/list", app.httpListenAddr)
-	data, statusCode := app.cli.Get(t, queryURL, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotListResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDelete deletes a snapshot by sending a query to the
-// /snapshot/delete endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", app.httpListenAddr, snapshotName)
-	data, statusCode := app.cli.Delete(t, queryURL)
-	wantStatusCodes := map[int]bool{
-		http.StatusOK:                  true,
-		http.StatusInternalServerError: true,
-	}
-	if !wantStatusCodes[statusCode] {
-		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
-	}
-
-	var res SnapshotDeleteResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDeleteAll deletes all snapshots by sending a query to the
-// /snapshot/delete_all endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (app *Vmstorage) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
-	t.Helper()
-
-	queryURL := fmt.Sprintf("http://%s/snapshot/delete_all", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotDeleteAllResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
 // String returns the string representation of the vmstorage app state.
 func (app *Vmstorage) String() string {
 	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q vminsertAddr: %q vmselectAddr: %q}", []any{

dashboards/victoriametrics-cluster.json

@@ -3109,7 +3109,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/victoriametrics.json

@@ -3406,7 +3406,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/victoriametrics-cluster.json

@@ -3110,7 +3110,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/victoriametrics.json

@@ -3407,7 +3407,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/vmagent.json

@@ -2946,7 +2946,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/vmalert.json

@@ -2324,7 +2324,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vmagent.json

@@ -2945,7 +2945,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vmalert.json

@@ -2323,7 +2323,6 @@
                   "mode": "off"
                 }
               },
-              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.142.0
+    image: victoriametrics/vmauth:v1.143.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     depends_on:
       - "victoriametrics"
     ports:

docs/Makefile

@@ -61,11 +61,11 @@ docs-update-vmsingle-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) victoria-metrics && \
-	./bin/victoria-metrics -help > /tmp/victoria_metrics_common_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) victoria-metrics)
+	(cd /tmp/vm-opensource && ./bin/victoria-metrics -help > /tmp/victoria_metrics_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise && $(MAKE) victoria-metrics && \
-	./bin/victoria-metrics -help > /tmp/victoria_metrics_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise && $(MAKE) victoria-metrics)
+	(cd /tmp/vm-enterprise && ./bin/victoria-metrics -help > /tmp/victoria_metrics_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/victoria_metrics_common_flags.md && \
 	cat /tmp/victoria_metrics_common_flags_tmp.md >> docs/victoriametrics/victoria_metrics_common_flags.md && \
@@ -90,10 +90,11 @@ docs-update-vmauth-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmauth && \
-	./bin/vmauth -help > /tmp/vmauth_common_flags_tmp.md
-	git checkout $(TAG)-enterprise && $(MAKE) vmauth && \
-	./bin/vmauth -help > /tmp/vmauth_enterprise_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmauth)
+	(cd /tmp/vm-opensource && ./bin/vmauth -help > /tmp/vmauth_common_flags_tmp.md)
+
+	(cd /tmp/vm-enterprise && $(MAKE) vmauth)
+	(cd /tmp/vm-enterprise && ./bin/vmauth -help > /tmp/vmauth_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmauth_common_flags.md
 	cat /tmp/vmauth_common_flags_tmp.md >> docs/victoriametrics/vmauth_common_flags.md
@@ -115,11 +116,11 @@ docs-update-vmagent-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmagent && \
-	./bin/vmagent -help > /tmp/vmagent_common_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmagent)
+	(cd /tmp/vm-opensource && ./bin/vmagent -help > /tmp/vmagent_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise && $(MAKE) vmagent && \
-	./bin/vmagent -help > /tmp/vmagent_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise && $(MAKE) vmagent)
+	(cd /tmp/vm-enterprise && ./bin/vmagent -help > /tmp/vmagent_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmagent_common_flags.md && \
 	cat /tmp/vmagent_common_flags_tmp.md >> docs/victoriametrics/vmagent_common_flags.md && \
@@ -143,11 +144,11 @@ docs-update-vmalert-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmalert && \
-	./bin/vmalert -help > /tmp/vmalert_common_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmalert)
+	(cd /tmp/vm-opensource && ./bin/vmalert -help > /tmp/vmalert_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise && $(MAKE) vmalert && \
-	./bin/vmalert -help > /tmp/vmalert_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise && $(MAKE) vmalert)
+	(cd /tmp/vm-enterprise && ./bin/vmalert -help > /tmp/vmalert_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmalert_common_flags.md && \
 	cat /tmp/vmalert_common_flags_tmp.md >> docs/victoriametrics/vmalert_common_flags.md && \
@@ -170,11 +171,11 @@ docs-update-vmselect-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG)-cluster && $(MAKE) vmselect && \
-	./bin/vmselect -help > /tmp/vmselect_common_flags_tmp.md
+	(cd /tmp/vm-opensource-cluster && $(MAKE) vmselect)
+	(cd /tmp/vm-opensource-cluster && ./bin/vmselect -help > /tmp/vmselect_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise-cluster && $(MAKE) vmselect && \
-	./bin/vmselect -help > /tmp/vmselect_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise-cluster && $(MAKE) vmselect)
+	(cd /tmp/vm-enterprise-cluster && ./bin/vmselect -help > /tmp/vmselect_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmselect_common_flags.md && \
 	cat /tmp/vmselect_common_flags_tmp.md >> docs/victoriametrics/vmselect_common_flags.md && \
@@ -199,11 +200,11 @@ docs-update-vminsert-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG)-cluster && $(MAKE) vminsert && \
-	./bin/vminsert -help > /tmp/vminsert_common_flags_tmp.md
+	(cd /tmp/vm-opensource-cluster && $(MAKE) vminsert)
+	(cd /tmp/vm-opensource-cluster && ./bin/vminsert -help > /tmp/vminsert_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise-cluster && $(MAKE) vminsert && \
-	./bin/vminsert -help > /tmp/vminsert_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise-cluster && $(MAKE) vminsert)
+	(cd /tmp/vm-enterprise-cluster && ./bin/vminsert -help > /tmp/vminsert_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vminsert_common_flags.md && \
 	cat /tmp/vminsert_common_flags_tmp.md >> docs/victoriametrics/vminsert_common_flags.md && \
@@ -235,11 +236,11 @@ docs-update-vmstorage-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG)-cluster && $(MAKE) vmstorage && \
-	./bin/vmstorage -help > /tmp/vmstorage_common_flags_tmp.md
+	(cd /tmp/vm-opensource-cluster && $(MAKE) vmstorage)
+	(cd /tmp/vm-opensource-cluster && ./bin/vmstorage -help > /tmp/vmstorage_common_flags_tmp.md)
 
-	git checkout $(TAG)-enterprise-cluster && $(MAKE) vmstorage && \
-	./bin/vmstorage -help > /tmp/vmstorage_enterprise_flags_tmp.md
+	(cd /tmp/vm-enterprise-cluster && $(MAKE) vmstorage)
+	(cd /tmp/vm-enterprise-cluster && ./bin/vmstorage -help > /tmp/vmstorage_enterprise_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmstorage_common_flags.md && \
 	cat /tmp/vmstorage_common_flags_tmp.md >> docs/victoriametrics/vmstorage_common_flags.md && \
@@ -263,13 +264,13 @@ docs-update-vmctl-flags:
 ifndef TAG
 	$(error TAG must be provided to update flags in docs)
 endif
-	git checkout $(TAG) && $(MAKE) vmctl && \
-	./bin/vmctl -help > /tmp/vmctl_flags_tmp.md && \
-	./bin/vmctl opentsdb -help > /tmp/vmctl_opentsdb_flags_tmp.md && \
-	./bin/vmctl influx -help > /tmp/vmctl_influx_flags_tmp.md && \
-	./bin/vmctl remote-read -help > /tmp/vmctl_remote-read_flags_tmp.md && \
-	./bin/vmctl prometheus -help > /tmp/vmctl_prometheus_flags_tmp.md && \
-	./bin/vmctl vm-native -help > /tmp/vmctl_vm-native_flags_tmp.md
+	(cd /tmp/vm-opensource && $(MAKE) vmctl)
+	(cd /tmp/vm-opensource && ./bin/vmctl -help > /tmp/vmctl_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl opentsdb -help > /tmp/vmctl_opentsdb_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl influx -help > /tmp/vmctl_influx_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl remote-read -help > /tmp/vmctl_remote-read_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl prometheus -help > /tmp/vmctl_prometheus_flags_tmp.md)
+	(cd /tmp/vm-opensource && ./bin/vmctl vm-native -help > /tmp/vmctl_vm-native_flags_tmp.md)
 
 	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_flags.md && \
 	cat /tmp/vmctl_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_flags.md && \
@@ -306,6 +307,10 @@ endif
 	# remove Version line and the actual version line from vmctl_flags.md to reduce diffs noise
 	sed -i '/^VERSION:/,+1d' docs/victoriametrics/vmctl/vmctl_flags.md
 
+	# adjust flags with dynamic default values
+	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
+	sed -i '/prom-tmp-dir-path/ s|(default: "[^"]*")|(default: os.TempDir())|' docs/victoriametrics/vmctl/vmctl_prometheus_flags.md
+
 # docs-update-flags updates flags in the documentation
 # using the actual binaries compiled from the provided $TAG.
 # The command also normalizes the output a bit.
@@ -324,12 +329,27 @@ endif
 	# Install using: brew install gnu-sed gawk
 	# Add tools to PATH see how in `brew info gnu-sed` and `brew info gawk
 
-	orig_branch=$$(git rev-parse --abbrev-ref HEAD); \
-	$(MAKE) docs-update-vmctl-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmsingle-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmalert-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmauth-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmagent-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmselect-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vminsert-flags && git checkout "$$orig_branch" && \
-	$(MAKE) docs-update-vmstorage-flags && git checkout "$$orig_branch"
+	rm -rf /tmp/vm-enterprise
+	git worktree remove /tmp/vm-enterprise || true
+	git worktree add /tmp/vm-enterprise $(TAG)-enterprise
+
+	rm -rf /tmp/vm-enterprise-cluster
+	git worktree remove /tmp/vm-enterprise-cluster || true
+	git worktree add /tmp/vm-enterprise-cluster $(TAG)-enterprise-cluster
+
+	rm -rf /tmp/vm-opensource
+	git worktree remove /tmp/vm-opensource || true
+	git worktree add /tmp/vm-opensource $(TAG)
+
+	rm -rf /tmp/vm-opensource-cluster
+	git worktree remove /tmp/vm-opensource-cluster || true
+	git worktree add /tmp/vm-opensource-cluster $(TAG)-cluster
+
+	$(MAKE) docs-update-vmctl-flags
+	$(MAKE) docs-update-vmsingle-flags
+	$(MAKE) docs-update-vmalert-flags
+	$(MAKE) docs-update-vmauth-flags
+	$(MAKE) docs-update-vmagent-flags
+	$(MAKE) docs-update-vmselect-flags
+	$(MAKE) docs-update-vminsert-flags
+	$(MAKE) docs-update-vmstorage-flags

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,15 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+## v1.29.4
+Released: 2026-05-15
+
+- IMPROVEMENT: Optimized [BacktestingScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) exact mode for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) (which are also used in [UI](https://docs.victoriametrics.com/anomaly-detection/ui/)-triggered tasks), leading to 2-200x faster results and more responsive UI interactions.
+
+- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.6.1](https://docs.victoriametrics.com/anomaly-detection/ui/#v161) to [v1.7.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v170), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v170) for details. Notable changes include iterative anomaly detection with progress bar, model inference speedups, improved AI Copilot UX and various bug fixes.
+
+- BUGFIX: Made model-level `queries` and `schedulers` attachment semantics explicit and safer. Omitting these args still keeps the legacy default of attaching a model to all queries or schedulers with a warning, while `queries: null`, `queries: []`, `schedulers: null`, and `schedulers: []` now mean the model is intentionally unattached and will be dropped from active execution. Use `queries: all` or `schedulers: all` to request all attachments explicitly. Orphan models, queries, and schedulers are logged and removed during config validation, including hot reload.
+
 ## v1.29.3
 Released: 2026-04-16
 

docs/anomaly-detection/FAQ.md

@@ -423,7 +423,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     # ...
     restart: always
     volumes:
@@ -641,7 +641,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.3 && docker image tag victoriametrics/vmanomaly:v1.29.3 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.29.4 && docker image tag victoriametrics/vmanomaly:v1.29.4 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/QuickStart.md

@@ -122,7 +122,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.3
+docker pull victoriametrics/vmanomaly:v1.29.4
 ```
 
 2. Create the license file with your license key.
@@ -142,7 +142,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.3 \
+    victoriametrics/vmanomaly:v1.29.4 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -159,7 +159,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.3 \
+    victoriametrics/vmanomaly:v1.29.4 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -172,7 +172,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     # ...
     restart: always
     volumes:

docs/anomaly-detection/UI.md

@@ -315,7 +315,7 @@ docker run -it --rm \
   -e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
   -p 8080:8080 \
   -p 8490:8490 \
-  victoriametrics/vmanomaly:v1.29.3 \
+  victoriametrics/vmanomaly:v1.29.4 \
   vmanomaly_config.yaml
 ```
 
@@ -640,6 +640,25 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
+### v1.7.0
+Released: 2026-05-15
+
+vmanomaly version: [v1.29.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1294)
+
+- FEATURE: Improved [AI Copilot](#ai-assistance) to 
+  - support textual attachments, such as configuration files, query examples, model outputs, etc.
+  - operate with current time when suggesting time range changes, e.g. "current month" or "from the beginning of the last month until now"
+- FEATURE: improve infer jobs scheduled from UI to
+  - run 2-100x faster for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) depending on a configuration vs [1.6.1](#v161) timings
+  - show stage-aware progress bar, e.g. "getting data", "fitting model", "inferring on chunk".
+- IMPROVEMENT: "Advanced Options" design is improved, section is collapsed by default to save vertical space, yet can be encoded as always open in UI state URL.
+- IMPROVEMENT: dropdowns in UI are now searchable and constrained in size for better UX, especially when many options are available (e.g. models list, tenants list, etc.).
+- BUGFIX: in generated config ("Show Config" menu)
+  - special YAML values (like `-.inf`, `.inf`, `.nan`) are now properly quoted and formatted to avoid JSON converting issues when copied to Kubernetes CRs, which previously led to rejection of the manifest before vmanomaly can consume it.
+  - with Data Source selected as "Logs/Traces" `reader.class` is now correctly set to `vlogs` (previously `vm`) in generated config, when "model only" toggle is turned off.
+- BUGFIX: now "Hide Common Labels" toggle in "Table View" works correctly and does not show common labels in the legend columns when turned on.
+- BUGFIX: "Prettify query" now works correctly for "Metrics" datasources.
+
 ### v1.6.1
 Released: 2026-04-16
 

docs/anomaly-detection/components/models.md

@@ -1265,7 +1265,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.2
+docker pull victoriametrics/vmanomaly:v1.29.4
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -1279,7 +1279,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.29.2 /config.yaml \
+victoriametrics/vmanomaly:v1.29.4 /config.yaml \
 --licenseFile=/license
 --watch
 ```

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -395,7 +395,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.3
+    image: victoriametrics/vmanomaly:v1.29.4
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -240,23 +240,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -308,7 +308,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.142.0
+    image: docker.io/victoriametrics/vmauth:v1.143.0
     ports:
       - 8427:8427
     volumes:

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.142.0-cluster
+    image: victoriametrics/vmstorage:v1.143.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.142.0-cluster
+    image: victoriametrics/vminsert:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.142.0-cluster
+    image: victoriametrics/vmselect:v1.143.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.142.0-enterprise
+    image: victoriametrics/vmauth:v1.143.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.142.0
+    image: victoriametrics/vmagent:v1.143.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.142.0
+    image: victoriametrics/vmalert:v1.143.0
     depends_on:
       - victoriametrics
       - alertmanager

docs/victoriametrics/Articles.md

@@ -33,6 +33,7 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [Percona: How do We Keep Metrics for a Long Time in VictoriaMetrics](https://www.youtube.com/watch?v=SGZjY7xgDwE)
 * [Miro: Prometheus High Availability and Fault Tolerance strategy, long term storage with VictoriaMetrics](https://medium.com/miro-engineering/prometheus-high-availability-and-fault-tolerance-strategy-long-term-storage-with-victoriametrics-82f6f3f0409e)
 * [ZERODHA: Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
+* [ZERODHA: Monitoring K8s with Victoriametrics](https://www.youtube.com/watch?v=ZJQYW-cFOms)
 * [Criteo: VictoriaMetrics, a stress-free Prometheus Remote Storage for 1 Billion metrics](https://medium.com/criteo-engineering/victoriametrics-a-prometheus-remote-storage-solution-57081a3d8e61)
 * [Abios Gaming: Choosing a Time Series Database for High Cardinality Aggregations](https://abiosgaming.com/press/high-cardinality-aggregations/)
 * [Cybozu: Monitoring Kubernetes clusters with VictoriaMetrics and Grafana](https://blog.cybozu.io/entry/2021/03/18/115743)

docs/victoriametrics/CONTRIBUTING.md

@@ -4,8 +4,8 @@ title: Contributing
 menu:
   docs:
     identifier: vm-contributing
-    parent: victoriametrics
     weight: 400
+    pageRef: "/victoriametrics/contributing/"
 tags: []
 aliases:
 - /CONTRIBUTING.html

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -566,7 +566,7 @@ The following optional command-line flags related to mTLS are supported:
 - `-cluster.tlsCAFile` can be set at `vminsert`, `vmselect` and `vmstorage` for verifying peer certificates issued with custom [certificate authority](https://en.wikipedia.org/wiki/Certificate_authority). By default, system-wide certificate authority is used for peer certificate verification.
 - `-cluster.tlsCipherSuites` can be set to the list of supported TLS cipher suites at `vmstorage`. See [the list of supported TLS cipher suites](https://pkg.go.dev/crypto/tls#pkg-constants).
 
-When `vmselect` runs with `-clusternativeListenAddr` command-line option, then it can be configured with `-clusternative.tls*` options similar to `-cluster.tls*` for accepting `mTLS` connections from top-level `vmselect` nodes in [multi-level cluster setup](#multi-level-cluster-setup).
+When `vmselect` or `vminsert` runs with `-clusternativeListenAddr` command-line option, then it can be configured with `-clusternative.tls*` options similar to `-cluster.tls*` for accepting `mTLS` connections from top-level `vmselect` or `vminsert` nodes in [multi-level cluster setup](#multi-level-cluster-setup).
 
 See [these docs](https://gist.github.com/f41gh7/76ed8e5fb1ebb9737fe746bae9175ee6) on how to set up mTLS in VictoriaMetrics cluster.
 
@@ -700,6 +700,7 @@ Also in the cluster version the `/prometheus/api/v1` endpoint ingests  `jsonl`,
 
 - `vmstorage` nodes provide the following HTTP endpoints on `8482` port:
   - `/internal/force_merge` - initiate [forced compactions](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-merge) on the given `vmstorage` node.
+  - `/internal/force_flush` - [flush in-memory buffers](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-flush) for recently ingested samples into searchable parts on the given `vmstorage` node.
   - `/snapshot/create` - create [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282),
     which can be used for backups in background. Snapshots are created in `<storageDataPath>/snapshots` folder, where `<storageDataPath>` is the corresponding
     command-line flag value.

docs/victoriametrics/Quick-Start.md

@@ -43,8 +43,8 @@ Just download VictoriaMetrics and follow [these instructions](https://docs.victo
 See [available integrations](https://docs.victoriametrics.com/victoriametrics/integrations/) with other systems like
 [Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/) or [Grafana](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/).
 
-VictoriaMetrics is developed at a fast pace, so it is recommended periodically checking the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/)
-and performing [regular upgrades](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
+VictoriaMetrics is developed at a fast pace, so it is recommended to periodically check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/)
+and perform [regular upgrades](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
 
 ### Starting VictoriaMetrics Single Node or Cluster on VictoriaMetrics Cloud {id="starting-vm-on-cloud"}
 
@@ -58,12 +58,12 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.142.0
+docker pull victoriametrics/victoria-metrics:v1.143.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.142.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.143.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
-_For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
+_For Enterprise images, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
 
 You should see:
 
@@ -113,7 +113,7 @@ See more details about [cluster architecture](https://docs.victoriametrics.com/v
 ### Starting VictoriaMetrics Single Node from a Binary {id="starting-vm-single-from-a-binary"}
 
 1. Download the correct binary for your OS and architecture from [GitHub](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
-For Enterprise binaries see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases).
+For Enterprise binaries, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases).
 
 2. Extract the archive to /usr/local/bin by running:
 
@@ -147,7 +147,7 @@ After=network.target
 Type=simple
 User=victoriametrics
 Group=victoriametrics
-ExecStart=/usr/local/bin/victoria-metrics-prod -storageDataPath=/var/lib/victoria-metrics -retentionPeriod=90d -selfScrapeInterval=10s
+ExecStart=/usr/local/bin/victoria-metrics-prod -storageDataPath=/var/lib/victoria-metrics -selfScrapeInterval=10s
 SyslogIdentifier=victoriametrics
 Restart=always
 
@@ -164,7 +164,7 @@ END'
 
 Extra [command-line flags](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#list-of-command-line-flags) can be added to `ExecStart` line.
 
-If you want to deploy VictoriaMetrics Single Node as a Windows Service review the [running as a Windows service docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#running-as-windows-service).
+If you want to deploy VictoriaMetrics Single Node as a Windows Service, review the [running as a Windows service docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#running-as-windows-service).
 
 > Please note, `victoriametrics` service is listening on `:8428` for HTTP connections (see `-httpListenAddr` flag).
 
@@ -174,7 +174,7 @@ If you want to deploy VictoriaMetrics Single Node as a Windows Service review th
 sudo systemctl daemon-reload && sudo systemctl enable --now victoriametrics.service
 ```
 
-7. Check that service started successfully:
+7. Check that the service started successfully:
 
 ```sh
 sudo systemctl status victoriametrics.service
@@ -187,12 +187,12 @@ by going to `http://<ip_or_hostname>:8428/vmui`.
 
 VictoriaMetrics cluster consists of [3 components](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#architecture-overview).
 It is recommended to run these components in the same private network (for [security reasons](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#security)),
-but on the separate physical nodes for the best performance.
+but on separate physical nodes for the best performance.
 
-On all nodes you will need to do the following:
+On all nodes, you will need to do the following:
 
 1. Download the correct binary for your OS and architecture with `-cluster` suffix from [GitHub](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
-For Enterprise binaries see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases).
+For Enterprise binaries, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases).
 
 2. Extract the archive to /usr/local/bin by running:
 
@@ -231,7 +231,7 @@ Type=simple
 User=victoriametrics
 Group=victoriametrics
 Restart=always
-ExecStart=/usr/local/bin/vmstorage-prod -retentionPeriod=90d -storageDataPath=/var/lib/vmstorage
+ExecStart=/usr/local/bin/vmstorage-prod -storageDataPath=/var/lib/vmstorage
 
 PrivateTmp=yes
 NoNewPrivileges=yes
@@ -254,7 +254,7 @@ for vmstorage can be added to `ExecStart` line.
 sudo systemctl daemon-reload && sudo systemctl enable --now vmstorage
 ```
 
-4. Check that service started successfully:
+4. Check that the service started successfully:
 
 ```sh
 sudo systemctl status vmstorage
@@ -301,14 +301,14 @@ in one flag. See more details in `-storageNode` flag description in [vminsert fl
 sudo systemctl daemon-reload && sudo systemctl enable --now vminsert.service
 ```
 
-3. Check that service started successfully:
+3. Check that the service started successfully:
 
 ```sh
 sudo systemctl status vminsert.service
 ```
 
 4. After `vminsert` is in `Running` state, confirm the service is healthy by visiting `http://<ip_or_hostname>:8480/-/healthy` link.
-It should say "VictoriaMetrics is Healthy"
+It should say "VictoriaMetrics is Healthy."
 
 #### Installing vmselect
 
@@ -344,7 +344,7 @@ END'
 ```
 
 Replace `<list of vmstorages>` with addresses of previously configured `vmstorage` services.
-To specify multiple addresses you can repeat the flag multiple times, or separate addresses with commas
+To specify multiple addresses, you can repeat the flag multiple times or separate addresses with commas
 in one flag. See more details in `-storageNode` flag description [vminsert flags](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#list-of-command-line-flags-for-vminsert).
 
 > Please note, `vmselect` service is listening on `:8481` for HTTP connections (see `-httpListenAddr` flag).
@@ -362,12 +362,12 @@ sudo systemctl status vmselect.service
 ```
 
 5. After `vmselect` is in `Running` state, confirm the service is healthy by visiting `http://<ip_or_hostname>:8481/select/0/vmui` link.
-It should open [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) page.
+It should open the [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) page.
 
 ## Write data
 
 There are two main models in monitoring for data collection: [push](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#push-model)
-and [pull](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#pull-model). Both are used in modern monitoring and both are
+and [pull](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#pull-model). Both are used in modern monitoring, and both are
 supported by VictoriaMetrics.
 
 See more details on [key concepts of writing data here](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#write-data).
@@ -389,7 +389,7 @@ and [other integrations](https://docs.victoriametrics.com/victoriametrics/integr
 ## Alerting
 
 To run periodic conditions checks use [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/).
-It allows creating set of conditions using MetricsQL expressions and send notifications to [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)
+It allows creating a set of conditions using MetricsQL expressions and sending notifications to [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)
 when such conditions are met.
 
 See [vmalert quick start](https://docs.victoriametrics.com/victoriametrics/vmalert/#quickstart).
@@ -413,7 +413,7 @@ command line tool. It supports the following databases for migration to Victoria
 
 ## Productionization
 
-When going to production with VictoriaMetrics we recommend following the recommendations below.
+When moving to production with VictoriaMetrics, we recommend following these best practices.
 
 ### Monitoring
 
@@ -429,11 +429,19 @@ Using the [recommended alerting rules](https://github.com/VictoriaMetrics/Victor
 will help to identify unwanted issues.
 
 The rule of thumb is to have a separate installation of VictoriaMetrics or any other monitoring system to monitor the
-production installation of VictoriaMetrics. This would make monitoring independent and will help identify problems with
+production installation of VictoriaMetrics. This would make monitoring independent and help identify problems with
 the main monitoring installation.
 
 See more details in the article [VictoriaMetrics Monitoring](https://victoriametrics.com/blog/victoriametrics-monitoring/).
 
+### Retention
+
+VictoriaMetrics Single-node and `vmstorage` in VictoriaMetrics Cluster retain data for 1 month by default.
+Data older than the retention period will be automatically deleted. To change the retention period, use the `-retentionPeriod` flag (e.g. `-retentionPeriod=90d`).
+See the [retention](https://docs.victoriametrics.com/victoriametrics/#retention) documentation for more details. 
+
+If free disk space falls below `-storage.minFreeDiskSpaceBytes`, VictoriaMetrics Single-node or `vmstorage` switches to read-only mode and stops accepting new data. To prevent this, ensure proper [capacity planning](#capacity-planning) and set up monitoring and alerting for disk usage.
+
 ### Capacity planning
 
 See capacity planning sections in [docs](https://docs.victoriametrics.com) for
@@ -457,7 +465,7 @@ For backup configuration, please refer to [vmbackup documentation](https://docs.
 
 ### Configuring limits
 
-To avoid excessive resource usage or performance degradation limits must be in place:
+To avoid excessive resource usage or performance degradation, limits must be in place:
 
 * [Resource usage limits](https://docs.victoriametrics.com/victoriametrics/faq/#how-to-set-a-memory-limit-for-victoriametrics-components);
 * [Cardinality limiter](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter).

docs/victoriametrics/README.md

@@ -701,18 +701,37 @@ It's better to use the `-retentionPeriod` command-line flag for efficient prunin
 
 ## Forced merge
 
-VictoriaMetrics performs [data compactions in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
-in order to keep good performance characteristics when accepting new data. These compactions (merges) are performed independently on per-month partitions.
-This means that compactions are stopped for per-month partitions if no new data is ingested into these partitions.
-Sometimes it is necessary to trigger compactions for old partitions. For instance, in order to free up disk space occupied by [deleted time series](#how-to-delete-time-series).
-In this case forced compaction may be initiated on the specified per-month partition by sending request to `/internal/force_merge?partition_prefix=YYYY_MM`,
-where `YYYY_MM` is per-month partition name. For example, `http://victoriametrics:8428/internal/force_merge?partition_prefix=2020_08` would initiate forced
-merge for August 2020 partition. The call to `/internal/force_merge` returns immediately, while the corresponding forced merge continues running in background.
+VictoriaMetrics performs [data compaction in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
+to keep high performance during ingestion. These compactions (merges) are performed independently on per-month partitions.
+Partitions that don't receive new data will eventually stop being merged, as they reach an optimal state.
+Sometimes it is necessary to trigger merges for old partitions. For example, to free up disk space occupied by [deleted time series](https://docs.victoriametrics.com/victoriametrics/#how-to-delete-time-series).
+The user can **force** compaction on the specified per-month partition by sending a request to `/internal/force_merge?partition_prefix=YYYY_MM`,
+where `YYYY_MM` is the per-month partition name. For example, `http://victoriametrics:8428/internal/force_merge?partition_prefix=2020_08` would initiate a forced
+merge for the August 2020 partition. The call to `/internal/force_merge` returns immediately, while the corresponding forced merge continues running in the background.
 
-Forced merges may require additional CPU, disk IO and storage space resources. It is unnecessary to run forced merge under normal conditions,
+Forced merges may require additional CPU, disk I/O, and storage space. It is unnecessary to run a forced merge under normal conditions,
 since VictoriaMetrics automatically performs [optimal merges in background](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
 when new data is ingested into it.
 
+The `/internal/force_merge` endpoint can be protected from unauthorized access via the `--forceMergeAuthKey` command-line flag.
+See [General security recommendations](https://docs.victoriametrics.com/victoriametrics/#general-security-recommendations) for more details.
+
+## Forced flush
+
+VictoriaMetrics puts the recently [ingested samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#write-data) into in-memory buffers,
+which aren't available for [querying](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data) for up to a few seconds.
+If you need to query the samples immediately after they are ingested, then call the `/internal/force_flush` HTTP endpoint before running your query.
+This endpoint converts in-memory buffers containing recently ingested samples into searchable data blocks.
+
+It isn't recommended to force flush on a regular basis, as this increases CPU usage and slows data ingestion.
+The `/internal/force_flush` endpoint exists for debug and test purposes (for instance, for automated tests) and should be avoided in production.
+
+> VictoriaMetrics may intentionally hide samples with timestamps close to the current time during querying.
+> This behavior is controlled via `-search.latencyOffset` command-line flag. See more details in [Query latency](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-latency) documentation.
+
+The `/internal/force_flush` endpoint can be protected from unauthorized access via `-forceFlushAuthKey` command-line flag.
+See [General security recommendations](https://docs.victoriametrics.com/victoriametrics/#general-security-recommendations) for more details.
+
 ## How to export time series
 
 VictoriaMetrics provides the following handlers for exporting data:
@@ -1736,16 +1755,16 @@ VictoriaMetrics provides the following security-related command-line flags:
 * `-mtls` and `-mtlsCAFile` for enabling [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication) for requests to `-httpListenAddr`. See [these docs](#mtls-protection).
 * `-httpAuth.username` and `-httpAuth.password` for protecting all the HTTP endpoints
   with [HTTP Basic Authentication](https://en.wikipedia.org/wiki/Basic_access_authentication).
-* `-deleteAuthKey` for protecting `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
-* `-snapshotAuthKey` for protecting `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
-* `-forceFlushAuthKey` for protecting `/internal/force_flush` endpoint. See [these docs](#troubleshooting).
-* `-forceMergeAuthKey` for protecting `/internal/force_merge` endpoint. See [force merge docs](#forced-merge).
-* `-search.resetCacheAuthKey` for protecting `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
-* `-reloadAuthKey` for protecting `/-/reload` endpoint, which is used for force reloading of [`-promscrape.config`](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
-* `-configAuthKey` for protecting `/config` endpoint, since it may contain sensitive information such as passwords.
-* `-flagsAuthKey` for protecting `/flags` endpoint.
-* `-pprofAuthKey` for protecting `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
-* `-metricNamesStatsResetAuthKey` for protecting `/api/v1/admin/status/metric_names_stats/reset` endpoint, used for [Metric Names Tracker](#track-ingested-metrics-usage).
+* `-deleteAuthKey` for protecting the `/api/v1/admin/tsdb/delete_series` endpoint. See [how to delete time series](#how-to-delete-time-series).
+* `-snapshotAuthKey` for protecting the `/snapshot*` endpoints. See [how to work with snapshots](#how-to-work-with-snapshots).
+* `-forceFlushAuthKey` for protecting the `/internal/force_flush` endpoint. See [force flush docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-flush).
+* `-forceMergeAuthKey` for protecting the `/internal/force_merge` endpoint. See [force merge docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#forced-merge).
+* `-search.resetCacheAuthKey` for protecting the `/internal/resetRollupResultCache` endpoint. See [backfilling](#backfilling) for more details.
+* `-reloadAuthKey` for protecting the `/-/reload` endpoint, which is used for force reloading of [`-promscrape.config`](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
+* `-configAuthKey` for protecting the `/config` endpoint, since it may contain sensitive information such as passwords.
+* `-flagsAuthKey` for protecting the `/flags` endpoint.
+* `-pprofAuthKey` for protecting the `/debug/pprof/*` endpoints, which can be used for [profiling](#profiling).
+* `-metricNamesStatsResetAuthKey` for protecting the `/api/v1/admin/status/metric_names_stats/reset` endpoint, used for [Metric Names Tracker](#track-ingested-metrics-usage).
 * `-denyQueryTracing` for disallowing [query tracing](#query-tracing).
 * `-http.header.hsts`, `-http.header.csp`, and `-http.header.frameOptions` for serving `Strict-Transport-Security`, `Content-Security-Policy`
   and `X-Frame-Options` HTTP response headers.

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,36 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
+
+Released at 2026-05-22
+
+* FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
+* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `clusternative.tls` `vminsert` configuration flags for [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). See [#10958](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10958).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
+* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Improve [slowness-based rerouting](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) to prevent rerouting storms under high cluster load. Previously, rerouting could cascade across storage nodes when the whole cluster was saturated, making the situation worse. Now rerouting only activates when the cluster p90 saturation is below 60%, and the slowest node is more than 20% slower than p90. See [#10876](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10876).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `{{.MetricsAccountID}}` and `{{.MetricsProjectID}}` [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) for use in `headers` and `url_prefix` config fields. Previously, only the combined `{{.MetricsTenant}}` (`accountID:projectID`) JWT placeholder was supported, making it impossible to configure [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers). See [#10927](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10927). Thanks to @Vinayak9769 for the contribution.
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
+
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive credentials.
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): hide values passed to `-remoteWrite.headers`,`remoteRead.headers`, `datasource.headers` and `notifier.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly establish [mtls](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection) connection between vmstorage and vminsert. Regression was introduced in v1.130.0 release for the enterprise version of vmstorage. See [#10972](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10972).
+* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
+* BUGFIX: all VictoriaMetrics components: fix int64 overflow when parsing [timestamp parameters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats) with relative durations. See [#10880](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
+
 ## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)
 
 Released at 2026-05-08
@@ -39,6 +69,7 @@ Released at 2026-05-08
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
 * FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Kafka (Enterprise)` row with panels for monitoring traffic (bytes), messages in/out, producer and consumer errors. See [#10728](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10728).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support sending data to the configured `-remoteWrite.url` via [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol). See [#10929](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10929).
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly obtain `__meta_hetzner_hcloud_location` and `__meta_hetzner_hcloud_location_network_zone` labels for [hetzner_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs). Hetzner changed discovery [API response](https://docs.hetzner.cloud/changelog#2025-12-16-phasing-out-datacenters) and returns `location` information from different field. See [#10909](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10909). Thanks to @juliusrickert for contribution.
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly discover addresses of storage nodes with [enterprise automatic-vmstorage-discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery). Bug was introduced at [v1.141.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11410).

docs/victoriametrics/changelog/CHANGELOG_2022.md

@@ -711,7 +711,7 @@ Released at 2022-06-20
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): expose `/api/v1/status/config` endpoint in the same way as Prometheus does. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#config).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-promscrape.suppressScrapeErrorsDelay` command-line flag, which can be used for delaying and aggregating the logging of per-target scrape errors. This may reduce the amounts of logs when `vmagent` scrapes many unreliable targets. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2575). Thanks to @jelmd for [the initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2576).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-promscrape.cluster.name` command-line flag, which allows proper data de-duplication when the same target is scraped from multiple [vmagent clusters](https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2679).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `action: graphite` relabeling rules optimized for extracting labels from Graphite-style metric names. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmagent/#graphite-relabeling) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2737).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `action: graphite` relabeling rules optimized for extracting labels from Graphite-style metric names. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#graphite-relabeling) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2737).
 * FEATURE: [VictoriaMetrics Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/): expose `vm_downsampling_partitions_scheduled` and `vm_downsampling_partitions_scheduled_size_bytes` metrics, which can be used for tracking the progress of initial [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling) for historical data. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2612).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): do not spend up to 5 seconds when trying to connect to unavailable `vmstorage` nodes. This should improve query latency when some of `vmstorage` nodes aren't available. Expose `vm_tcpdialer_addr_available{addr="..."}` metric at `http://vmselect:8481/metrics` for determining whether the given `addr` is available for establishing new connections. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711#issuecomment-1160363187).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-vmstorageDialTimeout` command-line flags to `vmselect` and `vminsert` for tuning the maximum duration for connection establishing to `vmstorage` nodes. This should help resolving [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/711).

docs/victoriametrics/changelog/CHANGELOG_2023.md

@@ -402,7 +402,7 @@ so rolling back to the previous versions of VictoriaMetrics may result in partia
 the following samples to the configured remote storage by default:
 
 - aggregated samples;
-- the original input samples, which match zero `match` options from the provided [config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config).
+- the original input samples, which match zero `match` options from the provided [config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
 
 Previously only aggregated samples were written to the storage by default.
 The previous behavior can be restored in the following ways:
@@ -429,8 +429,8 @@ The previous behavior can be restored in the following ways:
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow sharding outgoing time series among the configured remote storage systems. This can be useful for building horizontally scalable [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/), when samples for the same time series must be aggregated by the same `vmagent` instance at the second level. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4637) for details.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow configuring staleness interval in [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) config. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4667) for details.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow specifying a list of [series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) inside `if` option of relabeling rules. The corresponding relabeling rule is executed when at least a single series selector matches. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-enhancements).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): allow specifying a list of [series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) inside `match` option of [stream aggregation configs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). The input sample is aggregated when at least a single series selector matches. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4635).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): preserve input samples, which match zero `match` options from the [configured aggregations](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). Previously all the input samples were dropped by default, so only the aggregated samples are written to the output storage. The previous behavior can be restored by passing `-streamAggr.dropInput` command-line flag to single-node VictoriaMetrics or by passing `-remoteWrite.streamAggr.dropInput` command-line flag to `vmagent`.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): allow specifying a list of [series selectors](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering) inside `match` option of [stream aggregation configs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). The input sample is aggregated when at least a single series selector matches. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4635).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): preserve input samples, which match zero `match` options from the [configured aggregations](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). Previously all the input samples were dropped by default, so only the aggregated samples are written to the output storage. The previous behavior can be restored by passing `-streamAggr.dropInput` command-line flag to single-node VictoriaMetrics or by passing `-remoteWrite.streamAggr.dropInput` command-line flag to `vmagent`.
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add verbose output for docker installations or when TTY isn't available. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4081).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): interrupt backoff retries when import process is cancelled. The change makes vmctl more responsive in case of errors during the import. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4442).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): update backoff policy on retries to reduce probability of overloading for `source` or `destination` databases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4402).

docs/victoriametrics/changelog/CHANGELOG_2024.md

@@ -75,7 +75,7 @@ Released at 2024-11-29
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `dryRun` flag to validate configuration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7505) for details.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `removeXFFHTTPHeaderValue` flag to remove content of `X-Forwarded-For` HTTP Header before proxy it to the backend. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6883) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): revert the default value of `-remoteWrite.maxQueueSize` from `1_000_000` to `100_000`. It was bumped in [v1.104.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.104.0), which increases memory usage and is not needed for most setups. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7471).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `ignore_first_sample_interval` param to [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). It allows users to control the time interval when aggregation skips sending aggregated samples to avoid unexpected spikes in values. By default, this interval is set to x2 of `staleness_interval`. The new setting is applicable only to `total`, `total_prometheus`, `increase`, `increase_prometheus` and `histogram_bucket` outputs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7116) for details. Thanks to @iyuroch for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7313).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `ignore_first_sample_interval` param to [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). It allows users to control the time interval when aggregation skips sending aggregated samples to avoid unexpected spikes in values. By default, this interval is set to x2 of `staleness_interval`. The new setting is applicable only to `total`, `total_prometheus`, `increase`, `increase_prometheus` and `histogram_bucket` outputs. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7116) for details. Thanks to @iyuroch for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7313).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add support for template alias in predefined panels. This allows creating more readable metric names in the legend using constructions like `{{label_name}}`, where `label_name` is the name of the label. [See this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/116101da78a4dee8bd7c4ba0e66458fd05a10469#diff-95141489b32468cf852d2705d96eaa48c50a8b1cdd0424a29e7ca289912a6dcbR140-R151)
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): properly parse `multitenant` token value for multitenant endpoints. Before, pushing data into vmagent using [multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format) resulted in unsupported path. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7694).
@@ -231,7 +231,7 @@ Released at 2024-08-28
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add command-line flag `-notifier.headers` to allow configuring additional headers for all requests sent to the corresponding `-notifier.url`.
 * FEATURE: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): add `-external.label` and `-external.url` command-line flags, in the same way as these flags are supported by vmalert. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735).
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/), [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): use exponential backoff for retries when uploading or downloading data from S3. This should reduce the number of failed uploads and downloads when S3 is temporarily unavailable. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6732).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): do not allow enabling `-streamAggr.keepInput` and `keep_metric_names` options together in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config), as it may result in time series collision.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): do not allow enabling `-streamAggr.keepInput` and `keep_metric_names` options together in [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config), as it may result in time series collision.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add search functionality to the column display settings in the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the ability to select all columns in the column display settings of the table. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6668). Thanks to @yincongcyincong for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6680).
 * FEATURE: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add command-line flag `-search.inmemoryBufSizeBytes` for configuring size of in-memory buffers used by vmselect during processing of vmstorage responses. A new summary metric `vm_tmp_blocks_inmemory_file_size_bytes` is exposed to show the size of the buffer during requests processing. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6851) for details. Thanks to @tydhot for implementation.
@@ -427,7 +427,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
 
 **Update note 1: support for snap packages was removed due to lack of interest from community. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6543) for details. Please read about supported package types [here](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#install).**
 
-**Update note 2: [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config) now prevents setting multiple identical outputs. For example, `outputs: [total, total]` will fail the validation phase. In addition, `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` will fail the validation as well - use `outputs: ["quantiles(0.5, 0.9)"]` instead.**
+**Update note 2: [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config) now prevents setting multiple identical outputs. For example, `outputs: [total, total]` will fail the validation phase. In addition, `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` will fail the validation as well - use `outputs: ["quantiles(0.5, 0.9)"]` instead.**
 
 * SECURITY: upgrade Go builder from Go1.22.4 to Go1.22.5. See the list of issues addressed in [Go1.22.5](https://github.com/golang/go/issues?q=milestone%3AGo1.22.5+label%3ACherryPickApproved).
 * SECURITY: upgrade base docker image (Alpine) from 3.20.0 to 3.20.1. See [alpine 3.20.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.1-released.html).
@@ -437,11 +437,11 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
   * `vm_streamaggr_output_samples_total` - the number of output samples produced by the corresponding aggregation rule
   * `vm_streamaggr_samples_lag_seconds` - [histogram](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) with the lag between the current time and the timestamp seen in the aggregated input samples
 * FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add new labels to `vm_streamaggr_*` metrics:
-  * `name` - the name of the streaming aggregation rule, which can be configured via `name` option - see [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config).
+  * `name` - the name of the streaming aggregation rule, which can be configured via `name` option - see [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
   * `url` - `-remoteWrite.url` for the corresponding `-remoteWrite.streamAggr.config`
   * `path` - path to the corresponding streaming aggregation config file
   * `position` - the position of the aggregation rule in the corresponding streaming aggregation config file
-* FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent having duplicated aggregation function as `outputs` in one [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). It also prevents using `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` instead of  `outputs: ["quantiles(0.5, 0.9)"]`, as the former has higher computation cost for producing the same result.
+* FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent having duplicated aggregation function as `outputs` in one [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). It also prevents using `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` instead of  `outputs: ["quantiles(0.5, 0.9)"]`, as the former has higher computation cost for producing the same result.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-graphite.sanitizeMetricName` command-line flag for sanitizing metrics ingested via [Graphite protocol](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6077).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): [`yandexcloud_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs): add support for obtaining IAM token in [GCE format](https://yandex.cloud/en-ru/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm) additionally to the [deprecated Amazon EC2 IMDSv1 format](https://yandex.cloud/en/docs/security/standard/authentication#aws-token). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5513).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): make `-replay.timeTo` optional in [replay mode](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling). When omitted, the current timestamp will be used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6492).
@@ -485,7 +485,7 @@ Released at 2024-06-24
 * BUGFIX: all VictoriaMetrics components: prioritize `-configAuthKey` and `-reloadAuthKey` over `-httpAuth.*` settings. This change aligns behavior of mentioned flags with other auth flags like `-metricsAuthKey`, `-flagsAuthKey`, `-pprofAuthKey`. Check [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6329).
 * BUGFIX: all VictoriaMetrics components: do not trim trailing spaces when reading content from `*.passwordFile` and similar flags. Previously, trailing spaces were trimmed from the content of the password file, which could lead to unexpected authentication errors.
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--disable-progress-bar` global command-line flag. It can be used for disabling dynamic progress bar for all migration modes. `--vm-disable-progress-bar`  command-line flag is deprecated and will be removed in the future releases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6367).
-* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_avg) producing `NaN` results for stale time series. Before, when series matched for aggregation became stale or weren't updated during aggregation interval, the `rate_sum` or `rate_avg` could produce data point with `NaN` value. During visualization, such aggregation results would be displayed as gaps in time series.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_avg) producing `NaN` results for stale time series. Before, when series matched for aggregation became stale or weren't updated during aggregation interval, the `rate_sum` or `rate_avg` could produce data point with `NaN` value. During visualization, such aggregation results would be displayed as gaps in time series.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix path for system links printed on default vmalert's UI page when `-http.pathPrefix` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6433).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): exit replay mode with non-zero code if generated samples are not successfully written into remoteWrite url. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6512).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/): properly configure authentication with S3 when `-configFilePath` cmd-line flag is specified. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6353).
@@ -520,7 +520,7 @@ Released at 2024-06-07
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add service discovery support for [Vultr](https://www.vultr.com/). See [these docs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#vultr_sd_configs) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6041).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): allow specifying `-remoteWrite.disableOnDiskQueue` command-line flag per each `-remoteWrite.url`. If multiple `-remoteWrite.disableOnDiskQueue` command-line flags are configured, then the `-remoteWrite.dropSamplesOnOverload` is automatically set to true, so samples are automatically dropped if they cannot be sent to the corresponding `-remoteWrite.url` in a timely manner. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6065). Thanks to @rbizos for implementation!
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `path` and `url` labels to `vmagent_remotewrite_push_failures_total` and `vmagent_remotewrite_samples_dropped_total` [metrics](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring), so the number of failed pushes and dropped samples can be tracked per each `-remoteWrite.url`.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#rate_avg) aggregation outputs.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [rate_sum](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_sum) and [rate_avg](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#rate_avg) aggregation outputs.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): reduce the number of allocated objects in heap during deduplication and aggregation. The change supposed to reduce pressure on Garbage Collector, as it will need to scan less objects. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6402).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-datasource.idleConnTimeout`, `-remoteWrite.idleConnTimeout` and `-remoteRead.idleConnTimeout` flags. These flags are set to 50s by default and should reduce the probability of `broken pipe` or `connection reset by peer` errors in vmalert logs. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5661) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add auto request retry for trivial network errors, such as `broken pipe` and `connection reset` for requests to `remoteRead`, `remoteWrite` and `datasource` URLs. See this [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5661) for details.
@@ -531,7 +531,7 @@ Released at 2024-06-07
 * FEATURE: expose metric `vm_indexdb_items_dropped_total` to track the number of IndexDB records that had to be dropped during ingestion. The reason of dropping the record will be annotated in `reason` label of the exposed metric. This change also comes with a new [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) to track changes of this metric.
 * FEATURE: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): add new alerting rules `TooLongLabelValues` and `TooLongLabelNames` to notify about truncation of label values or names respectively.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): expose `vm_streamaggr_ignored_samples_total` [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter) at [`/metrics` page](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring), which can be used for detecting amount of too old or NaN valued ignored samples. Expose also `vm_streamaggr_samples_lag_seconds` [histogram](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) to monitor aggregated samples lag.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): improve filtering speed of the received data samples if [match](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config) field is matching only [metric name](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#structure-of-a-metric).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): improve filtering speed of the received data samples if [match](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config) field is matching only [metric name](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#structure-of-a-metric).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): adds `-relabelConfigCheckInterval` flag and `/-/reload` endpoint for reloading relabeling rules. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3923) for details.
 
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix bug that prevents the first query trace from expanding on click event. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6186). The issue was introduced in [v1.100.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11000) release.
@@ -540,7 +540,7 @@ Released at 2024-06-07
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix handling of URL params for browser history navigation (back and forward buttons). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6126) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5516#issuecomment-1867507232).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent potential panic during [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) when more than one `-remoteWrite.url` command-line flags are passed to `vmagent` together with non-zero `-remoteWrite.streamAggr.dedupInterval` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6205).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): skip empty data blocks before sending to the remote write destination. Thanks to @viperstars for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241).
-* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): set correct suffix `<output>_prometheus` for aggregation outputs [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total_prometheus). Before, outputs `total` and `total_prometheus` or `increase` and `increase_prometheus` had the same suffix.
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): set correct suffix `<output>_prometheus` for aggregation outputs [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total_prometheus). Before, outputs `total` and `total_prometheus` or `increase` and `increase_prometheus` had the same suffix.
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent from excessive resource usage when stream aggregation config file is empty.
 * BUGFIX: properly estimate the needed memory for query execution if it has the format [`aggr_func`](https://docs.victoriametrics.com/victoriametrics/metricsql/#aggregate-functions)([`rollup_func[d]`](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions) (for example, `sum(rate(request_duration_seconds_bucket[5m]))`). This should allow performing aggregations over bigger number of time series when VictoriaMetrics runs in environments with small amounts of available memory. The issue has been introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/5138eaeea0791caa34bcfab410e0ca9cd253cd8f) in [v1.83.0](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2022/#v1830).
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly apply `-inmemoryDataFlushInterval` when it's set to minimum supported value 1s.
@@ -616,14 +616,14 @@ Released at 2024-04-04
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): reduce memory usage by up to 5x when aggregating over big number of unique [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series). The memory usage reduction is most visible when [stream deduplication](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) is enabled.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): allow using `-streamAggr.dedupInterval` and `-remoteWrite.streamAggr.dedupInterval` command-line flags without the need to specify `-streamAggr.config` and `-remoteWrite.streamAggr.config`. See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `-streamAggr.dropInputLabels` command-line flag, which can be used for dropping the listed labels from input samples before applying stream [de-duplication](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) and aggregation. This is faster and easier to use alternative to [input_relabel_configs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#relabeling). See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `dedup_interval` option, which allows configuring individual [deduplication intervals](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) per each [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `dedup_interval` option, which allows configuring individual [deduplication intervals](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) per each [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): use the same logic in [stream deduplication](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication) as in [the deduplication at VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5643).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): ignore out of order samples when calculating [`increase`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase), [`increase_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase_prometheus), [`total`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total) and [`total_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total_prometheus) outputs. Thanks to @edma2 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5931).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): ignore out of order samples when calculating [`increase`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase), [`increase_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase_prometheus), [`total`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total) and [`total_prometheus`](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total_prometheus) outputs. Thanks to @edma2 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5931).
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add an ability to ignore input samples with old timestamps outside the current aggregation interval. See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples) for details.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `keep_metric_names` option, which can be set at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config) in order to keep the original metric names in the output aggregated samples instead of using [the default output metric naming scheme](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#output-metric-names).
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): align the time of aggregated data flush to the specified aggregation `interval`. For example, if `interval` is set to `1m`, then the aggregated data will be flushed at the end of every minute. The alignment can be disabled by setting `no_align_flush_to_interval: true` option at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#stream-aggregation-config). See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#flush-time-alignment) for details.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [unique_samples](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#unique_samples) output, which can be used for calculating the number of unique sample values over the given `interval`.
-* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#total_prometheus) outputs, which can be used for `increase` and `total` aggregations when the first sample of every new [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) must be ignored.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add `keep_metric_names` option, which can be set at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config) in order to keep the original metric names in the output aggregated samples instead of using [the default output metric naming scheme](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#output-metric-names).
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): align the time of aggregated data flush to the specified aggregation `interval`. For example, if `interval` is set to `1m`, then the aggregated data will be flushed at the end of every minute. The alignment can be disabled by setting `no_align_flush_to_interval: true` option at [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). See [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#flush-time-alignment) for details.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [unique_samples](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#unique_samples) output, which can be used for calculating the number of unique sample values over the given `interval`.
+* FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): add [increase_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#increase_prometheus) and [total_prometheus](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#total_prometheus) outputs, which can be used for `increase` and `total` aggregations when the first sample of every new [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series) must be ignored.
 * FEATURE: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): expose `vm_streamaggr_flush_timeouts_total` and `vm_streamaggr_dedup_flush_timeouts_total` [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter) at [`/metrics` page](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring), which can be used for detecting flush timeouts for stream aggregation states. Expose also `vm_streamaggr_flush_duration_seconds` and `vm_streamaggr_dedup_flush_duration_seconds` [histograms](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram) for monitoring the real flush durations of stream aggregation states.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve trace display for better visual separation of branches. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5926).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): ability to limit the ingestion rate via `-maxIngestionRate` command-line flag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5900).
@@ -989,7 +989,7 @@ The v1.97.x line will be supported for at least 12 months since [v1.97.0](https:
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduce initial memory usage when scraping targets, which return huge responses (for example, [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) in large Kubernetes cluster may return 100MB+ responses), without the need to explicitly enable [stream parsing mode](https://docs.victoriametrics.com/victoriametrics/vmagent/#stream-parsing-mode). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5567).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): expose ability to set OAuth2 endpoint parameters per each `-remoteWrite.url` via the command-line flag `-remoteWrite.oauth2.endpointParams`. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmagent/#advanced-usage). Thanks to @mhill-holoplot for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5427).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add ability to set `attach_metadata.node=true` option for all the [`kubernetes_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs) defined at [`-promscrape.config`](https://docs.victoriametrics.com/victoriametrics/vmagent/#quick-start) via `-promscrape.kubernetes.attachNodeMetadataAll` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4640). Thanks to @wasim-nihal for [the initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5593).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): do not send unfinished [aggregation state](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) on shutdown or [hot config reload](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#configuration-update) by default, as it tend to produce unexpected anomalies with lower values. The old behavior can be restored by specifying `flush_on_shutdown: true` setting in streaming aggregation config. See more details [here](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-outputs).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): do not send unfinished [aggregation state](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/) on shutdown or [hot config reload](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#configuration-update) by default, as it tend to produce unexpected anomalies with lower values. The old behavior can be restored by specifying `flush_on_shutdown: true` setting in streaming aggregation config. See more details [here](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#aggregation-outputs).
 * FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): expand `%{ENV_VAR}` placeholders in config files with the corresponding environment variable values.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmagent/): expose ability to set OAuth2 endpoint parameters via the following command-line flags:
   * `-datasource.oauth2.endpointParams` for `-datasource.url`

docs/victoriametrics/enterprise.md

@@ -15,25 +15,28 @@ aliases:
 - /enterprise/
 ---
 
-VictoriaMetrics and [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) components are provided
+VictoriaMetrics, [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) components are provided
 in two kinds - [Community edition](https://victoriametrics.com/products/open-source/) and [Enterprise edition](https://victoriametrics.com/products/enterprise/).
 
-VictoriaMetrics and VictoriaLogs community components are open source and are free to use:
+The community components are open source and are free to use:
 
 - See [VictoriaMetrics source code](https://github.com/VictoriaMetrics/VictoriaMetrics/) and [VictoriaMetrics license](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE).
 - See [VictoriaLogs source code](https://github.com/VictoriaMetrics/VictoriaLogs/) and [VictoriaLogs license](https://github.com/VictoriaMetrics/VictoriaLogs/blob/master/LICENSE).
+- See [VictoriaTraces source code](https://github.com/VictoriaMetrics/VictoriaTraces/) and [VictoriaLogs license](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/LICENSE).
 
-Enterprise components of VictoriaMetrics and VictoriaLogs are available at the following places:
+Both open-source and enterprise components are available at the following places:
 
-- Binary executables are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-  and [the release page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
+- Binary executables are available at the corresponding releases pages:
+  - [releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+  - [releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
+  - [releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest)
 - Docker images are available at [Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay](https://quay.io/organization/victoriametrics).
 
 Enterprise executables and Docker images have `enterprise` suffix in their names and tags.
 
 ## Valid cases for VictoriaMetrics Enterprise
 
-The use of Enterprise components of VictoriaMetrics and VictoriaLogs is permitted in the following cases:
+The use of Enterprise components is permitted in the following cases:
 
 - Evaluation use in non-production setups. Please, request a [trial license](https://victoriametrics.com/products/enterprise/trial/)
   and then pass it via `-license` or `-licenseFile` command-line flags as described [in these docs](#running-victoriametrics-enterprise).
@@ -104,25 +107,26 @@ Contact us via [this page](https://victoriametrics.com/products/enterprise/) if
 
 ## Running VictoriaMetrics Enterprise
 
-Enterprise components of VictoriaMetrics and VictoriaLogs are available in the following forms:
+Enterprise components are available in the following forms:
 
-- [Binary releases](#binary-releases)
-- [Docker images](#docker-images)
-- [Helm charts](#helm-charts)
-- [Kubernetes operator](#kubernetes-operator)
+- [Binary releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases)
+- [Docker images](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)
+- [Helm charts](https://docs.victoriametrics.com/victoriametrics/enterprise/#helm-charts)
+- [Kubernetes operator](https://docs.victoriametrics.com/victoriametrics/enterprise/#kubernetes-operator)
 
 ### Binary releases
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
-Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz`.
+Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
+[the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
+and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
 
-All the Enterprise components of VictoriaMetrics and VictoriaLogs require specifying the following command-line flags:
+All the Enterprise components require specifying the following command-line flags:
 
 - `-license` - this flag accepts VictoriaMetrics Enterprise license key, which can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/)
 - `-licenseFile` - this flag accepts a path to file with VictoriaMetrics Enterprise license key,
@@ -135,8 +139,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.143.0/victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -148,26 +152,26 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 
 ### Docker images
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.142.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.143.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
-flag as described in the [binary-releases](#binary-releases) section.
+flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
 
 Enterprise license key can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/).
 
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.142.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.143.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.142.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.143.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -177,7 +181,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.142.0
+    image: victoriametrics/victoria-metrics:v1.143.0
     ports:
       - 8428:8428
     volumes:
@@ -194,12 +198,12 @@ The example assumes that the license file is stored at `/vm-license` on the host
 
 ### Helm charts
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in cases [listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Helm charts for Enterprise components are available in our VictoriaMetrics [Helm repository](https://github.com/VictoriaMetrics/helm-charts).
 
 In order to run Enterprise helm chart it is required to provide the license key via `license` value in `values.yaml` file
-and adjust the image tag to the Enterprise one as described in the [docker-images](#docker-images) section.
+and adjust the image tag to the Enterprise one as described in the [docker-images](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images) section.
 
 Enterprise license key can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/).
 
@@ -209,7 +213,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -220,7 +224,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 
 license:
   secret:
@@ -250,7 +254,7 @@ Note that the license key provided by using secret is mounted in a file. This al
 
 ### Kubernetes operator
 
-It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Enterprise components can be deployed via [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/).
 In order to use Enterprise components it is required to provide the license key via `license` field and adjust the image tag to the enterprise one.
@@ -270,7 +274,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -287,7 +291,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.142.0-enterprise
+    tag: v1.143.0-enterprise
 ```
 
 Example secret with license key:
@@ -313,7 +317,7 @@ See the full list of the CRD specifications in the [Operator API](https://docs.v
 
 ### Updating the license key
 
-Updating the license key for VictoriaMetrics and VictoriaLogs Enterprise components depends on the way
+Updating the license key for Enterprise components depends on the way
 the license key is provided to the component:
 - If the license key is provided via `-license` command-line flag, then the component should be restarted
   with the new license key.
@@ -338,7 +342,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz`
 
 Includes:
 
@@ -347,11 +351,23 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.142.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.143.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+
+## What Happens to Licensed Components When a License Expires
+
+When a license expires, all licensed components continue to function normally until a restart occurs.
+
+License checks happen only at startup. If a license expires while the component is running, nothing changes; the component continues to run until the next restart. 
+
+This means you don't need to restart components to install a new license. The component automatically picks up the new license the next time it restarts. The exception is when the `-license` flag is used, because the license is supplied at startup and changing it requires restarting VictoriaMetrics with the updated flag value.
+
+If your license has expired and you decide to not renew it, you can switch to the VictoriaMetrics Open Source version without data loss, as both versions share the same data model. In doing so, however, you will lose access to the [VictoriaMetrics Enterprise features](https://docs.victoriametrics.com/victoriametrics/enterprise/#victoriametrics-enterprise-features).
+
+See [updating the license key](https://docs.victoriametrics.com/victoriametrics/enterprise/#updating-the-license-key) for more details.
 
 ## Monitoring license expiration
 
-All the VictoriaMetrics and VictoriaLogs Enterprise components expose the following metrics at the `/metrics` page:
+All the Enterprise components expose the following metrics at the `/metrics` page:
 
 - `vm_license_expires_at` - license expiration date in unix timestamp format
 - `vm_license_expires_in_seconds` - the number of seconds left until the license expires

docs/victoriametrics/goals.md

@@ -1,10 +1,11 @@
 ---
 weight: 500
-title: Development goals
+title: Development Goals
 menu:
   docs:
-    parent: 'victoriametrics'
     weight: 500
+    identifier: goals
+    pageRef: "/victoriametrics/goals/"
 tags: []
 aliases:
   - /goals/index.html
@@ -12,21 +13,21 @@ aliases:
 ---
 ## Goals
 
-1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics and VictoriaLogs component in the most efficient way**.
+1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics products in the most efficient way**.
 1. Fixing bugs in the essential functionality of VictoriaMetrics components. Small usability bugs are usually the most annoying,
    so they **must be fixed first**. Bugs, which affect a small number of users at some rare edge cases, can be fixed later.
-1. Improving [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) for VictoriaMetrics components,
-   so users could find answers to their questions via Google or [Perplexity](https://www.perplexity.ai/) without the need
+1. Improving [public docs for VictoriaMetrics products](https://docs.victoriametrics.com).
+   so users could find answers to their questions via Google or any other AI-powered web search without the need
    to ask these questions at our [support channels](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#community-and-contributions).
-1. Simplifying usage of VictoriaMetrics components without breaking backwards compatibility, so users could regularly
-   upgrade to [the latest available release](https://docs.victoriametrics.com/victoriametrics/changelog/) and remain happy.
-1. Improving usability for the existing functionality of VictoriaMetrics components.
+1. Simplifying usage of VictoriaMetrics products without breaking backwards compatibility, so users could regularly
+   upgrade to [the latest available release](https://docs.victoriametrics.com/victoriametrics/changelog/) and become happier.
+1. Improving the usability for the existing functionality of VictoriaMetrics components.
 1. Improving the readability and maintainability of the code base by removing unnecessary abstractions and simplifying the code whenever possible.
 1. Improving development velocity by optimizing and simplifying CI/CD tasks, so they take less time to execute and debug.
 
 ## Non-goals
 
-1. Convincing people to use VictoriaMetrics components when there are better suited solutions exist for their tasks,
+1. Convincing people to use VictoriaMetrics products when there are better suited solutions exist for their tasks,
    since users will become angry at VictoriaMetrics after they discover better solutions.
 1. Breaking links to [VictoriaMetrics docs](https://docs.victoriametrics.com/victoriametrics/), since users will be unhappy seeing 404 page
    or unexpected results after they click some old link somewhere on the Internet or in the internal knowledge base.
@@ -35,25 +36,29 @@ aliases:
    since these features may break the essential functionality of VictoriaMetrics components, so a big share
    of the existing users may become unhappy after the upgrade.
 1. Adding unnecessary abstractions, since they may worsen project maintainability in the future.
-1. Implementing all the features users ask. These features should fit [the goals](#goals) of VictoriaMetrics.
+1. Implementing all the features users ask. These features should fit [the goals](https://docs.victoriametrics.com/victoriametrics/goals/#goals) of VictoriaMetrics.
    Other feature requests must be closed as `won't implement`, with the link to this page.
-1. Merging all the pull requests users submit. These pull requests should fit [the goals](#goals) of VictoriaMetrics.
+1. Merging all the pull requests users submit. These pull requests should fit [the goals](https://docs.victoriametrics.com/victoriametrics/goals/#goals) of VictoriaMetrics.
    Other pull requests must be closed as `won't merge`, with the link to this page.
 1. Slowing down and complicating CI/CD pipelines with non-essential tasks, since this results in development velocity slowdown.
 1. Introducing non-essential requirements, since this slows down development velocity.
 
 ## VictoriaMetrics proverbs
 
-- **Small usability fix is better than non-trivial feature.** Usability fix makes happy existing users.
-  Non-trivial feature may make happy some new users, while it may make upset a big share of existing users
-  if the feature breaks some essential functionality of VictoriaMetrics components or makes it less efficient.
+- **A small usability improvement is more valuable than a major new feature.**
+  The usability fix makes happy existing users. Non-trivial feature may make happy some new users,
+  while it may make upset a big share of existing users if the feature breaks the essential functionality
+  of VictoriaMetrics products or makes it less efficient.
 
-- **Good docs are better than new shiny feature.** Good docs help users discovering new functionality and dealing
-  with VictoriaMetrics components in the most efficient way. Nobody uses new shiny feature if it isn't documented properly.
+- **Having clean and concise documentation is more valuable than having a lot of great but undocumented features.**
+  Good docs help users discovering new functionality and dealing with VictoriaMetrics products in the most efficient way.
+  Nobody uses new shiny features if they aren't documented properly.
 
-- **Happy users are more important than the momentary revenue.** Happy users spread the word about VictoriaMetrics,
-  so more people convert to VictoriaMetrics users. Happy users are eager to become happy [customers](https://docs.victoriametrics.com/victoriametrics/enterprise/).
-  This increases long-term revenue.
+- **A simple solution is better than a smart one.**
+  The simple solution is easier to setup, operate, debug and troubleshoot than the smart solution. This saves users' time, costs and nerve cells.
 
-- **Simple solution is better than smart solution.** Simple solution is easier to setup, operate, debug and troubleshoot than the smart solution.
-  This saves users' time, costs and nerve cells.
+- **Happy users are more important than the short-term profit.**
+  Happy users spread the word about VictoriaMetrics, so more people convert to VictoriaMetrics users.
+  Happy users are eager to become happy [customers](https://docs.victoriametrics.com/victoriametrics/enterprise/)
+  over time. This increases long-term profit.
+  Upset users may be forced to become customers, but they will constantily search for the opportunity to switch to competing solutions.