address review comments

Signed-off-by: JAYICE <1185430411@qq.com>

.github/workflows/build.yml

@@ -57,17 +57,15 @@ jobs:
             arch: amd64
           - os: openbsd
             arch: amd64
-          - os: netbsd
-            arch: amd64
           - os: windows
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
+        uses: actions/setup-go@v6
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/changelog-linter.yml

@@ -9,7 +9,7 @@ jobs:
   tip-lint:
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: 'actions/checkout@v6'
         with:
           # needed for proper diff
           fetch-depth: 0

.github/workflows/check-commit-signed.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0  # we need full history for commit verification
 

.github/workflows/check-licenses.yml

@@ -15,11 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@master
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
           cache: false
@@ -27,7 +27,7 @@ jobs:
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cache/go-build

.github/workflows/codeql-analysis-go.yml

@@ -29,18 +29,18 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
 
       - name: Set up Go
         id: go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
+        uses: actions/setup-go@v6
         with:
           cache: false
           go-version-file: 'go.mod'
       - run: go version
 
       - name: Cache Go artifacts
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cache/go-build
@@ -50,14 +50,14 @@ jobs:
           restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e  # v4.35.4
+        uses: github/codeql-action/init@v4.35.1
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@68bde559dea0fdcac2102bfdf6230c5f70eb485e  # v4.35.4
+        uses: github/codeql-action/autobuild@v4.35.1
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e  # v4.35.4
+        uses: github/codeql-action/analyze@v4.35.1
         with:
           category: 'language:go'

.github/workflows/docs.yaml

@@ -16,19 +16,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
           path: __vm-docs
 
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd  # v7.0.0
+        uses: crazy-max/ghaction-import-gpg@v7
         id: import-gpg
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}

.github/workflows/test.yml

@@ -32,11 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
+        uses: actions/setup-go@v6
         with:
           cache-dependency-path: |
             go.sum
@@ -47,7 +47,7 @@ jobs:
       - run: go version
 
       - name: Cache golangci-lint
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cache/golangci-lint
@@ -72,11 +72,11 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
+        uses: actions/setup-go@v6
         with:
           cache-dependency-path: |
             go.sum
@@ -94,11 +94,11 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
-        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c  # v6.4.0
+        uses: actions/setup-go@v6
         with:
           cache-dependency-path: |
             go.sum

.github/workflows/vmui.yml

@@ -32,11 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@v6
 
       - name: Cache node_modules
         id: cache
-        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
+        uses: actions/cache@v5
         with:
           path: app/vmui/packages/vmui/node_modules
           key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
@@ -69,7 +69,7 @@ jobs:
           VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
-        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9  # 3.0.0
+        uses: ataylorme/eslint-annotate-action@v3
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           report-json: app/vmui/packages/vmui/vmui-lint-report.json

Makefile

@@ -485,8 +485,8 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
 		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
 	); \
-	VMSINGLE_V1_132_0_PATH=$${DIR}/victoria-metrics-prod \
-	VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
+	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
+	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
@@ -535,15 +535,6 @@ remove-golangci-lint:
 govulncheck: install-govulncheck
 	govulncheck ./...
 
-govulncheck-docker:
-	docker run -w $(PWD) -v $(PWD):$(PWD) \
-		-v govulncheck-gomod-cache:/root/go/pkg/mod \
-		-v govulncheck-gobuild-cache:/root/.cache/go-build \
-		-v govulncheck-go-bin:/root/go/bin \
-		--env="GOCACHE=/root/.cache/go-build" \
-		--env="GOMODCACHE=/root/go/pkg/mod" \
-		"$(GO_BUILDER_IMAGE)" /bin/sh -c "which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./..."
-
 install-govulncheck:
 	which govulncheck || go install golang.org/x/vuln/cmd/govulncheck@latest
 

SECURITY.md

@@ -1,4 +1,42 @@
 # Security Policy
 
-You can find out about our security policy and VictoriaMetrics version support on the [security page](https://docs.victoriametrics.com/victoriametrics/#security) in the documentation.
+## Supported Versions
 
+The following versions of VictoriaMetrics receive regular security fixes:
+
+| Version                                                                        | Supported          |
+|--------------------------------------------------------------------------------|--------------------|
+| [Latest release](https://docs.victoriametrics.com/victoriametrics/changelog/)  | :white_check_mark: |
+| [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
+| other releases                                                                 | :x:                |
+
+See [this page](https://victoriametrics.com/security/) for more details.
+
+## Software Bill of Materials (SBOM)
+
+Every VictoriaMetrics container{{% available_from "#" %}} image published to
+[Docker Hub](https://hub.docker.com/u/victoriametrics)
+and [Quay.io](https://quay.io/organization/victoriametrics)
+includes an [SPDX](https://spdx.dev/) SBOM attestation
+generated automatically by BuildKit during
+`docker buildx build`.
+
+To inspect the SBOM for an image:
+
+```sh
+docker buildx imagetools inspect \
+  docker.io/victoriametrics/victoria-metrics:latest \
+  --format "{{ json .SBOM }}"
+```
+
+To scan an image using its SBOM attestation with
+[Trivy](https://github.com/aquasecurity/trivy):
+
+```sh
+trivy image --sbom-sources oci \
+  docker.io/victoriametrics/victoria-metrics:latest
+```
+
+## Reporting a Vulnerability
+
+Please report any security issues to <security@victoriametrics.com>

app/vmagent/main.go

@@ -83,9 +83,6 @@ var (
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 0, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
 	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
-
-	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", false, "Enables multitenancy via HTTP headers. "+
-		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy")
 )
 
 var (
@@ -118,7 +115,6 @@ func main() {
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
-	opentelemetry.Init()
 	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 
 	if promscrape.IsDryRun() {
@@ -220,7 +216,7 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 	}
 	return func(req *http.Request) error {
 		path := strings.ReplaceAll(req.URL.Path, "//", "/")
-		at, err := getAuthTokenFromPath(path, req.Header)
+		at, err := getAuthTokenFromPath(path)
 		if err != nil {
 			return fmt.Errorf("cannot obtain auth token from path %q: %w", path, err)
 		}
@@ -228,15 +224,8 @@ func getOpenTSDBHTTPInsertHandler() func(req *http.Request) error {
 	}
 }
 
-func parsePath(path string, header http.Header) (*httpserver.Path, error) {
-	if *enableMultitenancyViaHeaders {
-		return httpserver.ParsePathAndHeaders(path, header)
-	}
-	return httpserver.ParsePath(path)
-}
-
-func getAuthTokenFromPath(path string, header http.Header) (*auth.Token, error) {
-	p, err := parsePath(path, header)
+func getAuthTokenFromPath(path string) (*auth.Token, error) {
+	p, err := httpserver.ParsePath(path)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse multitenant path: %w", err)
 	}
@@ -570,15 +559,14 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path string) bool {
-	p, err := parsePath(path, r.Header)
+	p, err := httpserver.ParsePath(path)
 	if err != nil {
 		// Cannot parse multitenant path. Skip it - probably it will be parsed later.
 		return false
 	}
 	if p.Prefix != "insert" {
-		// processMultitenantRequest is called for all unmatched path variants,
-		// but we should try parsing only /insert prefixed to avoid catching all possible paths.
-		return false
+		httpserver.Errorf(w, r, `unsupported multitenant prefix: %q; expected "insert"`, p.Prefix)
+		return true
 	}
 	at, err := auth.NewTokenPossibleMultitenant(p.AuthToken)
 	if err != nil {

app/vmagent/opentelemetry/request_handler.go

@@ -25,11 +25,6 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
-// Init must be called after flag.Parse and before using the opentelemetry package.
-func Init() {
-	stream.InitDecodeOptions()
-}
-
 // InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {

app/vmagent/remotewrite/client.go

@@ -2,7 +2,6 @@ package remotewrite
 
 import (
 	"bytes"
-	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -59,9 +58,63 @@ var (
 		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
 		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
-	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
-	basicAuthUsernameFile = flagutil.NewArrayString("remoteWrite.basicAuth.usernameFile", "Optional path to basic auth username to use for the corresponding -remoteWrite.url. "+
+	mdxForcePromProto = flagutil.NewArrayBool("mdx.remoteWrite.forcePromProto", "Whether to force Prometheus remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
+	mdxForceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
+		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol")
+
+	mdxRateLimit = flagutil.NewArrayInt("mdx.remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
+		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
+		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
+	mdxSendTimeout      = flagutil.NewArrayDuration("mdx.remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	mdxRetryMinInterval = flagutil.NewArrayDuration("mdx.remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
+	// deprecated in the future. use -remoteWrite.retryMaxInterval instead
+	mdxRetryMaxTime     = flagutil.NewArrayDuration("mdx.remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. This flag is deprecated, use -remoteWrite.retryMaxInterval instead")
+	mdxRetryMaxInterval = flagutil.NewArrayDuration("mdx.remoteWrite.retryMaxInterval", time.Minute, "The maximum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url.  The delay doubles with each retry until this maximum is reached, after which it remains constant. See also -remoteWrite.retryMinInterval")
+	mdxProxyURL         = flagutil.NewArrayString("mdx.remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
+
+	mdxTlsHandshakeTimeout   = flagutil.NewArrayDuration("mdx.remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
+	mdxTlsInsecureSkipVerify = flagutil.NewArrayBool("mdx.remoteWrite.tlsInsecureSkipVerify", "Whether to skip tls verification when connecting to the corresponding -remoteWrite.url")
+	mdxTlsCertFile           = flagutil.NewArrayString("mdx.remoteWrite.tlsCertFile", "Optional path to client-side TLS certificate file to use when connecting "+
+		"to the corresponding -remoteWrite.url")
+	mdxTlsKeyFile = flagutil.NewArrayString("mdx.remoteWrite.tlsKeyFile", "Optional path to client-side TLS certificate key to use when connecting to the corresponding -remoteWrite.url")
+	mdxTlsCAFile  = flagutil.NewArrayString("mdx.remoteWrite.tlsCAFile", "Optional path to TLS CA file to use for verifying connections to the corresponding -remoteWrite.url. "+
+		"By default, system CA is used")
+	mdxTlsServerName = flagutil.NewArrayString("mdx.remoteWrite.tlsServerName", "Optional TLS server name to use for connections to the corresponding -remoteWrite.url. "+
+		"By default, the server name from -remoteWrite.url is used")
+
+	mdxHeaders = flagutil.NewArrayString("mdx.remoteWrite.headers", "Optional HTTP headers to send with each request to the corresponding -remoteWrite.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteWrite.url. "+
+		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
+	mdxBasicAuthUsername     = flagutil.NewArrayString("mdx.remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
+	mdxBasicAuthPassword     = flagutil.NewArrayString("mdx.remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
+	mdxBasicAuthPasswordFile = flagutil.NewArrayString("mdx.remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
 		"The file is re-read every second")
+	mdxBearerToken     = flagutil.NewArrayString("mdx.remoteWrite.bearerToken", "Optional bearer auth token to use for the corresponding -remoteWrite.url")
+	mdxBearerTokenFile = flagutil.NewArrayString("mdx.remoteWrite.bearerTokenFile", "Optional path to bearer token file to use for the corresponding -remoteWrite.url. "+
+		"The token is re-read from the file every second")
+
+	mdxOauth2ClientID         = flagutil.NewArrayString("mdx.remoteWrite.oauth2.clientID", "Optional OAuth2 clientID to use for the corresponding -remoteWrite.url")
+	mdxOauth2ClientSecret     = flagutil.NewArrayString("mdx.remoteWrite.oauth2.clientSecret", "Optional OAuth2 clientSecret to use for the corresponding -remoteWrite.url")
+	mdxOauth2ClientSecretFile = flagutil.NewArrayString("mdx.remoteWrite.oauth2.clientSecretFile", "Optional OAuth2 clientSecretFile to use for the corresponding -remoteWrite.url")
+	mdxOauth2EndpointParams   = flagutil.NewArrayString("mdx.remoteWrite.oauth2.endpointParams", "Optional OAuth2 endpoint parameters to use for the corresponding -remoteWrite.url . "+
+		`The endpoint parameters must be set in JSON format: {"param1":"value1",...,"paramN":"valueN"}`)
+	mdxOauth2TokenURL = flagutil.NewArrayString("mdx.remoteWrite.oauth2.tokenUrl", "Optional OAuth2 tokenURL to use for the corresponding -remoteWrite.url")
+	mdxOauth2Scopes   = flagutil.NewArrayString("mdx.remoteWrite.oauth2.scopes", "Optional OAuth2 scopes to use for the corresponding -remoteWrite.url. Scopes must be delimited by ';'")
+
+	mdxAwsUseSigv4 = flagutil.NewArrayBool("mdx.remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
+		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
+	mdxAwsEC2Endpoint = flagutil.NewArrayString("mdx.remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	mdxAwsSTSEndpoint = flagutil.NewArrayString("mdx.remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	mdxAwsRegion      = flagutil.NewArrayString("mdx.remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	mdxAwsRoleARN     = flagutil.NewArrayString("mdx.remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	mdxAwsAccessKey   = flagutil.NewArrayString("mdx.remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	mdxAwsService     = flagutil.NewArrayString("mdx.remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+		"Defaults to \"aps\"")
+	mdxAwsSecretKey = flagutil.NewArrayString("mdx.remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+
+	basicAuthUsername     = flagutil.NewArrayString("remoteWrite.basicAuth.username", "Optional basic auth username to use for the corresponding -remoteWrite.url")
 	basicAuthPassword     = flagutil.NewArrayString("remoteWrite.basicAuth.password", "Optional basic auth password to use for the corresponding -remoteWrite.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("remoteWrite.basicAuth.passwordFile", "Optional path to basic auth password to use for the corresponding -remoteWrite.url. "+
 		"The file is re-read every second")
@@ -123,24 +176,24 @@ type client struct {
 	stopCh chan struct{}
 }
 
-func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int) *client {
-	authCfg, err := getAuthConfig(argIdx)
+func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persistentqueue.FastQueue, concurrency int, flags *Flags) *client {
+	authCfg, err := getAuthConfig(argIdx, flags)
 	if err != nil {
 		logger.Fatalf("cannot initialize auth config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
-	awsCfg, err := getAWSAPIConfig(argIdx)
+	awsCfg, err := getAWSAPIConfig(argIdx, flags)
 	if err != nil {
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 
 	tr := httputil.NewTransport(false, "vmagent_remotewrite")
-	tr.TLSHandshakeTimeout = tlsHandshakeTimeout.GetOptionalArg(argIdx)
+	tr.TLSHandshakeTimeout = flags.tlsHandshakeTimeout.GetOptionalArg(argIdx)
 	tr.MaxConnsPerHost = 2 * concurrency
 	tr.MaxIdleConnsPerHost = 2 * concurrency
 	tr.IdleConnTimeout = time.Minute
 	tr.WriteBufferSize = 64 * 1024
 
-	pURL := proxyURL.GetOptionalArg(argIdx)
+	pURL := flags.proxyURL.GetOptionalArg(argIdx)
 	if len(pURL) > 0 {
 		if !strings.Contains(pURL, "://") {
 			logger.Fatalf("cannot parse -remoteWrite.proxyURL=%q: it must start with `http://`, `https://` or `socks5://`", pURL)
@@ -153,11 +206,11 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	}
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
-		Timeout:   sendTimeout.GetOptionalArg(argIdx),
+		Timeout:   flags.sendTimeout.GetOptionalArg(argIdx),
 	}
-	retryMaxIntervalFlag := retryMaxTime
+	retryMaxIntervalFlag := flags.retryMaxTime
 	if retryMaxInterval.String() != "" {
-		retryMaxIntervalFlag = retryMaxInterval
+		retryMaxIntervalFlag = flags.retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
@@ -166,14 +219,14 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		awsCfg:           awsCfg,
 		fq:               fq,
 		hc:               hc,
-		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
+		retryMinInterval: flags.retryMinInterval.GetOptionalArg(argIdx),
 		retryMaxInterval: retryMaxIntervalFlag.GetOptionalArg(argIdx),
 		stopCh:           make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
 
-	useVMProto := forceVMProto.GetOptionalArg(argIdx)
-	usePromProto := forcePromProto.GetOptionalArg(argIdx)
+	useVMProto := flags.forceVMProto.GetOptionalArg(argIdx)
+	usePromProto := flags.forcePromProto.GetOptionalArg(argIdx)
 	if useVMProto && usePromProto {
 		logger.Fatalf("-remoteWrite.useVMProto and -remoteWrite.usePromProto cannot be set simultaneously for -remoteWrite.url=%s", sanitizedURL)
 	}
@@ -187,16 +240,16 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	return c
 }
 
-func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
+func (c *client) init(argIdx, concurrency int, sanitizedURL string, flags *Flags) {
 	limitReached := metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rate_limit_reached_total{url=%q}`, c.sanitizedURL))
-	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
+	if bytesPerSec := flags.rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl = ratelimiter.New(int64(bytesPerSec), limitReached, c.stopCh)
 	}
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
 	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(rateLimit.GetOptionalArg(argIdx))
+		return float64(flags.rateLimit.GetOptionalArg(argIdx))
 	})
 	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
 	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
@@ -219,54 +272,52 @@ func (c *client) MustStop() {
 	logger.Infof("stopped client for -remoteWrite.url=%q", c.sanitizedURL)
 }
 
-func getAuthConfig(argIdx int) (*promauth.Config, error) {
-	headersValue := headers.GetOptionalArg(argIdx)
+func getAuthConfig(argIdx int, flags *Flags) (*promauth.Config, error) {
+	headersValue := flags.headers.GetOptionalArg(argIdx)
 	var hdrs []string
 	if headersValue != "" {
 		hdrs = strings.Split(headersValue, "^^")
 	}
-	username := basicAuthUsername.GetOptionalArg(argIdx)
-	usernameFile := basicAuthUsernameFile.GetOptionalArg(argIdx)
-	password := basicAuthPassword.GetOptionalArg(argIdx)
-	passwordFile := basicAuthPasswordFile.GetOptionalArg(argIdx)
+	username := flags.basicAuthUsername.GetOptionalArg(argIdx)
+	password := flags.basicAuthPassword.GetOptionalArg(argIdx)
+	passwordFile := flags.basicAuthPasswordFile.GetOptionalArg(argIdx)
 	var basicAuthCfg *promauth.BasicAuthConfig
-	if username != "" || usernameFile != "" || password != "" || passwordFile != "" {
+	if username != "" || password != "" || passwordFile != "" {
 		basicAuthCfg = &promauth.BasicAuthConfig{
 			Username:     username,
-			UsernameFile: usernameFile,
 			Password:     promauth.NewSecret(password),
 			PasswordFile: passwordFile,
 		}
 	}
 
-	token := bearerToken.GetOptionalArg(argIdx)
-	tokenFile := bearerTokenFile.GetOptionalArg(argIdx)
+	token := flags.bearerToken.GetOptionalArg(argIdx)
+	tokenFile := flags.bearerTokenFile.GetOptionalArg(argIdx)
 
 	var oauth2Cfg *promauth.OAuth2Config
-	clientSecret := oauth2ClientSecret.GetOptionalArg(argIdx)
-	clientSecretFile := oauth2ClientSecretFile.GetOptionalArg(argIdx)
+	clientSecret := flags.oauth2ClientSecret.GetOptionalArg(argIdx)
+	clientSecretFile := flags.oauth2ClientSecretFile.GetOptionalArg(argIdx)
 	if clientSecretFile != "" || clientSecret != "" {
-		endpointParamsJSON := oauth2EndpointParams.GetOptionalArg(argIdx)
+		endpointParamsJSON := flags.oauth2EndpointParams.GetOptionalArg(argIdx)
 		endpointParams, err := flagutil.ParseJSONMap(endpointParamsJSON)
 		if err != nil {
 			return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", endpointParamsJSON, err)
 		}
 		oauth2Cfg = &promauth.OAuth2Config{
-			ClientID:         oauth2ClientID.GetOptionalArg(argIdx),
+			ClientID:         flags.oauth2ClientID.GetOptionalArg(argIdx),
 			ClientSecret:     promauth.NewSecret(clientSecret),
 			ClientSecretFile: clientSecretFile,
 			EndpointParams:   endpointParams,
-			TokenURL:         oauth2TokenURL.GetOptionalArg(argIdx),
-			Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(argIdx), ";"),
+			TokenURL:         flags.oauth2TokenURL.GetOptionalArg(argIdx),
+			Scopes:           strings.Split(flags.oauth2Scopes.GetOptionalArg(argIdx), ";"),
 		}
 	}
 
 	tlsCfg := &promauth.TLSConfig{
-		CAFile:             tlsCAFile.GetOptionalArg(argIdx),
-		CertFile:           tlsCertFile.GetOptionalArg(argIdx),
-		KeyFile:            tlsKeyFile.GetOptionalArg(argIdx),
-		ServerName:         tlsServerName.GetOptionalArg(argIdx),
-		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
+		CAFile:             flags.tlsCAFile.GetOptionalArg(argIdx),
+		CertFile:           flags.tlsCertFile.GetOptionalArg(argIdx),
+		KeyFile:            flags.tlsKeyFile.GetOptionalArg(argIdx),
+		ServerName:         flags.tlsServerName.GetOptionalArg(argIdx),
+		InsecureSkipVerify: flags.tlsInsecureSkipVerify.GetOptionalArg(argIdx),
 	}
 
 	opts := &promauth.Options{
@@ -284,17 +335,17 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 	return authCfg, nil
 }
 
-func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
-	if !awsUseSigv4.GetOptionalArg(argIdx) {
+func getAWSAPIConfig(argIdx int, flags *Flags) (*awsapi.Config, error) {
+	if !flags.awsUseSigv4.GetOptionalArg(argIdx) {
 		return nil, nil
 	}
-	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
-	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
-	region := awsRegion.GetOptionalArg(argIdx)
-	roleARN := awsRoleARN.GetOptionalArg(argIdx)
-	accessKey := awsAccessKey.GetOptionalArg(argIdx)
-	secretKey := awsSecretKey.GetOptionalArg(argIdx)
-	service := awsService.GetOptionalArg(argIdx)
+	ec2Endpoint := flags.awsEC2Endpoint.GetOptionalArg(argIdx)
+	stsEndpoint := flags.awsSTSEndpoint.GetOptionalArg(argIdx)
+	region := flags.awsRegion.GetOptionalArg(argIdx)
+	roleARN := flags.awsRoleARN.GetOptionalArg(argIdx)
+	accessKey := flags.awsAccessKey.GetOptionalArg(argIdx)
+	secretKey := flags.awsSecretKey.GetOptionalArg(argIdx)
+	service := flags.awsService.GetOptionalArg(argIdx)
 	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service, "")
 	if err != nil {
 		return nil, err
@@ -311,6 +362,11 @@ func (c *client) runWorker() {
 		if !ok {
 			return
 		}
+		if len(block) == 0 {
+			// skip empty data blocks from sending
+			// see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6241
+			continue
+		}
 		go func() {
 			startTime := time.Now()
 			ch <- c.sendBlock(block)
@@ -326,20 +382,15 @@ func (c *client) runWorker() {
 			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			return
 		case <-c.stopCh:
-			// c must be stopped. Wait up to 5 seconds for the in-flight request to complete.
-			// If it succeeds, drain the remaining in-memory queue before returning.
-			stopCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-			defer cancel()
-
+			// c must be stopped. Wait for a while in the hope the block will be sent.
+			graceDuration := 5 * time.Second
 			select {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
 					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
-				} else {
-					c.drainInMemoryQueue(stopCtx, block[:0])
 				}
-			case <-stopCtx.Done():
+			case <-time.After(graceDuration):
 				// Return unsent block to the queue.
 				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			}
@@ -471,7 +522,7 @@ again:
 				goto again
 			}
 
-			logger.Warnf("failed to repack zstd block (%d bytes) to snappy: %s; The block will be rejected. "+
+			logger.Warnf("failed to repack zstd block (%s bytes) to snappy: %s; The block will be rejected. "+
 				"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
 				zstdBlockLen, err)
 		}
@@ -509,32 +560,6 @@ again:
 	goto again
 }
 
-func (c *client) drainInMemoryQueue(stopCtx context.Context, block []byte) {
-	var ok bool
-	for {
-		select {
-		case <-stopCtx.Done():
-			return
-		default:
-		}
-
-		block, ok = c.fq.MustReadInMemoryBlock(block[:0])
-		if !ok {
-			// The in memory queue has already been drained,
-			// or persisted queue is being used.
-			// In this case it is guaranteed that fq will be empty
-			return
-		}
-
-		// at this stage c.stopCh should be closed
-		// so sendBlock function should not perform retries
-		if ok := c.sendBlock(block); !ok {
-			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
-			return
-		}
-	}
-}
-
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 

app/vmagent/remotewrite/client_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 func TestParseRetryAfterHeader(t *testing.T) {
@@ -37,40 +36,6 @@ func TestParseRetryAfterHeader(t *testing.T) {
 	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
 }
 
-func TestInitSecretFlags(t *testing.T) {
-	showRemoteWriteURLOrig := *showRemoteWriteURL
-	defer func() {
-		*showRemoteWriteURL = showRemoteWriteURLOrig
-		flagutil.UnregisterAllSecretFlags()
-	}()
-
-	flagutil.UnregisterAllSecretFlags()
-	*showRemoteWriteURL = false
-	InitSecretFlags()
-	if !flagutil.IsSecretFlag("remotewrite.url") {
-		t.Fatalf("expecting remoteWrite.url to be secret")
-	}
-	if !flagutil.IsSecretFlag("remotewrite.headers") {
-		t.Fatalf("expecting remoteWrite.headers to be secret")
-	}
-	if !flagutil.IsSecretFlag("remotewrite.proxyurl") {
-		t.Fatalf("expecting remoteWrite.proxyURL to be secret")
-	}
-
-	flagutil.UnregisterAllSecretFlags()
-	*showRemoteWriteURL = true
-	InitSecretFlags()
-	if flagutil.IsSecretFlag("remotewrite.url") {
-		t.Fatalf("remoteWrite.url must remain visible when -remoteWrite.showURL is set")
-	}
-	if !flagutil.IsSecretFlag("remotewrite.headers") {
-		t.Fatalf("expecting remoteWrite.headers to remain secret")
-	}
-	if !flagutil.IsSecretFlag("remotewrite.proxyurl") {
-		t.Fatalf("expecting remoteWrite.proxyURL to remain secret")
-	}
-}
-
 func TestRepackBlockFromZstdToSnappy(t *testing.T) {
 	expectedPlainBlock := []byte(`foobar`)
 

app/vmagent/remotewrite/relabel.go

@@ -20,7 +20,8 @@ import (
 )
 
 var (
-	unparsedLabelsGlobal    = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to all -remoteWrite.url.")
+	unparsedLabels          = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to all -remoteWrite.url.")
+	mdxUnparsedLabels       = flagutil.NewArrayString("mdx.remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to all -remoteWrite.url.")
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
 		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
 		"The path can point either to local file or to http url. "+
@@ -28,6 +29,9 @@ var (
 	relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
 		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
 		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
+	mdxRelabelConfigPaths = flagutil.NewArrayString("mdx.remoteWrite.urlRelabelConfig", "Optional path to relabel configs for the corresponding -remoteWrite.url. "+
+		"See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. "+
+		"See https://docs.victoriametrics.com/victoriametrics/relabeling/")
 
 	usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+
 		"in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+
@@ -141,11 +145,15 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
+			len(*relabelConfigPaths), len(*remoteWriteURLs))
+	}
+	if len(*mdxRelabelConfigPaths) > len(*mdxRemoteWriteURLs) {
+		return nil, fmt.Errorf("too many -mdx.remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -mdx.remoteWrite.url args: %d",
+			len(*relabelConfigPaths), len(*mdxRemoteWriteURLs))
 	}
 
 	var urlRelabelCfgs []any
-	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
+	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs)+len(*mdxRemoteWriteURLs))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
 			urlRelabelCfgs = append(urlRelabelCfgs, nil)
@@ -167,6 +175,27 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 			urlRelabelCfgs = append(urlRelabelCfgs, nil)
 		}
 	}
+	for i, path := range *mdxRelabelConfigPaths {
+		if len(path) == 0 {
+			urlRelabelCfgs = append(urlRelabelCfgs, nil)
+			continue
+		}
+		prc, rawCfg, err := promrelabel.LoadRelabelConfigs(path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot load relabel configs from -mdx.remoteWrite.urlRelabelConfig=%q: %w", path, err)
+		}
+		rcs.perURL[len(*remoteWriteURLs)+i] = prc
+
+		var parsedCfg any
+		_ = yaml.Unmarshal(rawCfg, &parsedCfg)
+		urlRelabelCfgs = append(urlRelabelCfgs, parsedCfg)
+	}
+	if len(*mdxRemoteWriteURLs) > len(*mdxRelabelConfigPaths) {
+		// fill the urlRelabelCfgs with empty relabel configs if not set
+		for i := len(*mdxRelabelConfigPaths); i < len(*mdxRemoteWriteURLs); i++ {
+			urlRelabelCfgs = append(urlRelabelCfgs, nil)
+		}
+	}
 	remoteWriteURLRelabelConfigData.Store(&urlRelabelCfgs)
 	return &rcs, nil
 }
@@ -178,13 +207,16 @@ type relabelConfigs struct {
 
 // isSet indicates whether (global or per-URL) command-line flags is set
 func (rcs *relabelConfigs) isSet() bool {
-	return *relabelConfigPathGlobal != "" || len(*relabelConfigPaths) > 0
+	return *relabelConfigPathGlobal != "" || len(*relabelConfigPaths) > 0 || len(*mdxRelabelConfigPaths) > 0
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.
 func initLabelsGlobal() {
 	labelsGlobal = nil
-	for _, s := range *unparsedLabelsGlobal {
+	unparsedLabelsGlobal := make([]string, len(*unparsedLabels)+len(*mdxUnparsedLabels))
+	copy(unparsedLabelsGlobal, *unparsedLabels)
+	copy(unparsedLabelsGlobal[len(*unparsedLabels):], *mdxUnparsedLabels)
+	for _, s := range unparsedLabelsGlobal {
 		if len(s) == 0 {
 			continue
 		}

app/vmagent/remotewrite/remotewrite.go

@@ -12,6 +12,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mdx"
 	"github.com/cespare/xxhash/v2"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -102,6 +103,193 @@ var (
 		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
 	disableMetadataPerURL = flagutil.NewArrayBool("remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
 		"By default, metadata sending is controlled by the global -enableMetadata flag")
+
+	mdxRemoteWriteURLs = flagutil.NewArrayString("mdx.remoteWrite.url", "Remote storage URL to write data to. It must support either VictoriaMetrics remote write protocol "+
+		"or Prometheus remote_write protocol. Example url: http://<victoriametrics-host>:8428/api/v1/write . "+
+		"Pass multiple -remoteWrite.url options in order to replicate the collected data to multiple remote storage systems. "+
+		"The data can be sharded among the configured remote storage systems if -remoteWrite.shardByURL flag is set")
+	mdxQueues = flagutil.NewArrayInt("mdx.remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+		"isn't enough for sending high volume of collected data to remote storage. "+
+		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
+	mdxSignificantFigures = flagutil.NewArrayInt("mdx.remoteWrite.significantFigures", 0, "The number of significant figures to leave in metric values before writing them "+
+		"to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
+		"This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits")
+	mdxRoundDigits = flagutil.NewArrayInt("mdx.remoteWrite.roundDigits", 100, "Round metric values to this number of decimal digits after the point before "+
+		"writing them to remote storage. "+
+		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
+		"By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
+		"This option may be used for improving data compression for the stored metrics")
+	mdxDisableOnDiskQueue = flagutil.NewArrayBool("mdx.remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
+		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
+	mdxMaxPendingBytesPerURL = flagutil.NewArrayBytes("mdx.remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
+		"Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. "+
+		"Disk usage is unlimited if the value is set to 0")
+	mdxDisableMetadataPerURL = flagutil.NewArrayBool("mdx.remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
+		"By default, metadata sending is controlled by the global -enableMetadata flag")
+)
+
+// Flags contains all the array flags for remoteWrite and mdx.remoteWrite
+// They have similar functions and flags, but the flags are indexed differently. This structure is used to organize them.
+type Flags struct {
+	enableMdx                      bool
+	remoteWriteURLs                *flagutil.ArrayString
+	maxPendingBytesPerURL          *flagutil.ArrayBytes
+	queues                         *flagutil.ArrayInt
+	significantFigures             *flagutil.ArrayInt
+	roundDigits                    *flagutil.ArrayInt
+	disableOnDiskQueue             *flagutil.ArrayBool
+	disableMetadataPerURL          *flagutil.ArrayBool
+	streamAggrConfig               *flagutil.ArrayString
+	streamAggrDropInput            *flagutil.ArrayBool
+	streamAggrKeepInput            *flagutil.ArrayBool
+	streamAggrDedupInterval        *flagutil.ArrayDuration
+	streamAggrIgnoreOldSamples     *flagutil.ArrayBool
+	streamAggrIgnoreFirstIntervals *flagutil.ArrayInt
+	streamAggrDropInputLabels      *flagutil.ArrayString
+	streamAggrEnableWindows        *flagutil.ArrayBool
+	forcePromProto                 *flagutil.ArrayBool
+	forceVMProto                   *flagutil.ArrayBool
+	rateLimit                      *flagutil.ArrayInt
+	sendTimeout                    *flagutil.ArrayDuration
+	retryMaxTime                   *flagutil.ArrayDuration
+	retryMinInterval               *flagutil.ArrayDuration
+	retryMaxInterval               *flagutil.ArrayDuration
+	proxyURL                       *flagutil.ArrayString
+	tlsHandshakeTimeout            *flagutil.ArrayDuration
+	tlsInsecureSkipVerify          *flagutil.ArrayBool
+	tlsCertFile                    *flagutil.ArrayString
+	tlsKeyFile                     *flagutil.ArrayString
+	tlsCAFile                      *flagutil.ArrayString
+	tlsServerName                  *flagutil.ArrayString
+	headers                        *flagutil.ArrayString
+	basicAuthUsername              *flagutil.ArrayString
+	basicAuthPassword              *flagutil.ArrayString
+	basicAuthPasswordFile          *flagutil.ArrayString
+	bearerToken                    *flagutil.ArrayString
+	bearerTokenFile                *flagutil.ArrayString
+	oauth2ClientID                 *flagutil.ArrayString
+	oauth2ClientSecret             *flagutil.ArrayString
+	oauth2ClientSecretFile         *flagutil.ArrayString
+	oauth2EndpointParams           *flagutil.ArrayString
+	oauth2TokenURL                 *flagutil.ArrayString
+	oauth2Scopes                   *flagutil.ArrayString
+	awsUseSigv4                    *flagutil.ArrayBool
+	awsEC2Endpoint                 *flagutil.ArrayString
+	awsSTSEndpoint                 *flagutil.ArrayString
+	awsRegion                      *flagutil.ArrayString
+	awsRoleARN                     *flagutil.ArrayString
+	awsAccessKey                   *flagutil.ArrayString
+	awsService                     *flagutil.ArrayString
+	awsSecretKey                   *flagutil.ArrayString
+}
+
+var (
+	remoteWriteFlags = &Flags{
+		enableMdx:                      false,
+		remoteWriteURLs:                remoteWriteURLs,
+		queues:                         queues,
+		significantFigures:             significantFigures,
+		roundDigits:                    roundDigits,
+		disableMetadataPerURL:          disableMetadataPerURL,
+		disableOnDiskQueue:             disableOnDiskQueue,
+		maxPendingBytesPerURL:          maxPendingBytesPerURL,
+		streamAggrConfig:               streamAggrConfig,
+		streamAggrDropInput:            streamAggrDropInput,
+		streamAggrKeepInput:            streamAggrKeepInput,
+		streamAggrDedupInterval:        streamAggrDedupInterval,
+		streamAggrIgnoreOldSamples:     streamAggrIgnoreOldSamples,
+		streamAggrIgnoreFirstIntervals: streamAggrIgnoreFirstIntervals,
+		streamAggrDropInputLabels:      streamAggrDropInputLabels,
+		streamAggrEnableWindows:        streamAggrEnableWindows,
+		forcePromProto:                 forcePromProto,
+		forceVMProto:                   forceVMProto,
+		rateLimit:                      rateLimit,
+		sendTimeout:                    sendTimeout,
+		retryMinInterval:               retryMinInterval,
+		retryMaxInterval:               retryMaxInterval,
+		proxyURL:                       proxyURL,
+		tlsHandshakeTimeout:            tlsHandshakeTimeout,
+		tlsInsecureSkipVerify:          tlsInsecureSkipVerify,
+		tlsCertFile:                    tlsCertFile,
+		tlsKeyFile:                     tlsKeyFile,
+		tlsCAFile:                      tlsCAFile,
+		tlsServerName:                  tlsServerName,
+		headers:                        headers,
+		retryMaxTime:                   retryMaxTime,
+		basicAuthPassword:              basicAuthPassword,
+		basicAuthPasswordFile:          basicAuthPasswordFile,
+		basicAuthUsername:              basicAuthUsername,
+		bearerToken:                    bearerToken,
+		bearerTokenFile:                bearerTokenFile,
+		oauth2ClientID:                 oauth2ClientID,
+		oauth2ClientSecret:             oauth2ClientSecret,
+		oauth2ClientSecretFile:         oauth2ClientSecretFile,
+		oauth2EndpointParams:           oauth2EndpointParams,
+		oauth2TokenURL:                 oauth2TokenURL,
+		oauth2Scopes:                   oauth2Scopes,
+		awsUseSigv4:                    awsUseSigv4,
+		awsEC2Endpoint:                 awsEC2Endpoint,
+		awsSTSEndpoint:                 awsSTSEndpoint,
+		awsRegion:                      awsRegion,
+		awsRoleARN:                     awsRoleARN,
+		awsAccessKey:                   awsAccessKey,
+		awsService:                     awsService,
+		awsSecretKey:                   awsSecretKey,
+	}
+	mdxRemoteWriteFlags = &Flags{
+		enableMdx:                      true,
+		remoteWriteURLs:                mdxRemoteWriteURLs,
+		queues:                         mdxQueues,
+		significantFigures:             mdxSignificantFigures,
+		roundDigits:                    mdxRoundDigits,
+		disableMetadataPerURL:          mdxDisableMetadataPerURL,
+		disableOnDiskQueue:             mdxDisableOnDiskQueue,
+		maxPendingBytesPerURL:          mdxMaxPendingBytesPerURL,
+		streamAggrConfig:               mdxStreamAggrConfig,
+		streamAggrDropInput:            mdxStreamAggrDropInput,
+		streamAggrKeepInput:            mdxStreamAggrKeepInput,
+		streamAggrDedupInterval:        mdxStreamAggrDedupInterval,
+		streamAggrIgnoreOldSamples:     mdxStreamAggrIgnoreOldSamples,
+		streamAggrIgnoreFirstIntervals: mdxStreamAggrIgnoreFirstIntervals,
+		streamAggrDropInputLabels:      mdxStreamAggrDropInputLabels,
+		streamAggrEnableWindows:        mdxStreamAggrEnableWindows,
+		forcePromProto:                 mdxForcePromProto,
+		forceVMProto:                   mdxForceVMProto,
+		rateLimit:                      mdxRateLimit,
+		sendTimeout:                    mdxSendTimeout,
+		retryMinInterval:               mdxRetryMinInterval,
+		retryMaxInterval:               mdxRetryMaxInterval,
+		proxyURL:                       mdxProxyURL,
+		tlsHandshakeTimeout:            mdxTlsHandshakeTimeout,
+		tlsInsecureSkipVerify:          mdxTlsInsecureSkipVerify,
+		tlsCertFile:                    mdxTlsCertFile,
+		tlsKeyFile:                     mdxTlsKeyFile,
+		tlsCAFile:                      mdxTlsCAFile,
+		tlsServerName:                  mdxTlsServerName,
+		headers:                        mdxHeaders,
+		retryMaxTime:                   mdxRetryMaxTime,
+		basicAuthPassword:              mdxBasicAuthPassword,
+		basicAuthPasswordFile:          mdxBasicAuthPasswordFile,
+		basicAuthUsername:              mdxBasicAuthUsername,
+		bearerToken:                    mdxBearerToken,
+		bearerTokenFile:                mdxBearerTokenFile,
+		oauth2ClientID:                 mdxOauth2ClientID,
+		oauth2ClientSecret:             mdxOauth2ClientSecret,
+		oauth2ClientSecretFile:         mdxOauth2ClientSecretFile,
+		oauth2EndpointParams:           mdxOauth2EndpointParams,
+		oauth2TokenURL:                 mdxOauth2TokenURL,
+		oauth2Scopes:                   mdxOauth2Scopes,
+		awsUseSigv4:                    mdxAwsUseSigv4,
+		awsEC2Endpoint:                 mdxAwsEC2Endpoint,
+		awsSTSEndpoint:                 mdxAwsSTSEndpoint,
+		awsRegion:                      mdxAwsRegion,
+		awsRoleARN:                     mdxAwsRoleARN,
+		awsAccessKey:                   mdxAwsAccessKey,
+		awsService:                     mdxAwsService,
+		awsSecretKey:                   mdxAwsSecretKey,
+	}
 )
 
 var (
@@ -148,13 +336,10 @@ const persistentQueueDirname = "persistent-queue"
 // InitSecretFlags must be called after flag.Parse and before any logging.
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
-		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
+		// remoteWrite.url and mdx.remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
 		flagutil.RegisterSecretFlag("remoteWrite.url")
+		flagutil.RegisterSecretFlag("mdx.remoteWrite.url")
 	}
-	// remoteWrite.proxyURL can contain authentication codes.
-	flagutil.RegisterSecretFlag("remoteWrite.proxyURL")
-	// remoteWrite.headers can contain auth headers such as Authorization and API keys.
-	flagutil.RegisterSecretFlag("remoteWrite.headers")
 }
 
 var (
@@ -168,21 +353,9 @@ var (
 //
 // Stop must be called for graceful shutdown.
 func Init() {
-	if len(*remoteWriteURLs) == 0 {
-		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
+	if len(*remoteWriteURLs) == 0 && len(*mdxRemoteWriteURLs) == 0 {
+		logger.Fatalf("at least one `-remoteWrite.url` or `-mdx.remoteWrite.url` command-line flag must be set")
 	}
-	if *shardByURL && len(*disableOnDiskQueue) > 1 {
-		disableOnDiskQueues := *disableOnDiskQueue
-
-		firstValue := disableOnDiskQueues[0]
-		for _, v := range disableOnDiskQueues[1:] {
-			if firstValue != v {
-				logger.Fatalf("all -remoteWrite.url targets must have the same -remoteWrite.disableOnDiskQueue setting when -remoteWrite.shardByURL is enabled; " +
-					"either enable or disable -remoteWrite.disableOnDiskQueue for all targets")
-			}
-		}
-	}
-
 	if limit := getMaxHourlySeries(); limit > 0 {
 		hourlySeriesLimiter = bloomfilter.NewLimiter(limit, time.Hour)
 		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
@@ -220,7 +393,7 @@ func Init() {
 
 	initStreamAggrConfigGlobal()
 
-	initRemoteWriteCtxs(*remoteWriteURLs)
+	initRemoteWriteCtxs(*remoteWriteURLs, *mdxRemoteWriteURLs)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -229,7 +402,7 @@ func Init() {
 	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
 	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
 	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
-	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(*remoteWriteURLs) > 1
+	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(*remoteWriteURLs)+len(*mdxRemoteWriteURLs) > 1
 
 	dropDanglingQueues()
 
@@ -280,12 +453,15 @@ func dropDanglingQueues() {
 	}
 }
 
-func initRemoteWriteCtxs(urls []string) {
-	if len(urls) == 0 {
+func initRemoteWriteCtxs(urls []string, mdxUrls []string) {
+	if len(urls) == 0 && len(mdxUrls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
-	rwctxs := make([]*remoteWriteCtx, len(urls))
-	rwctxIdx := make([]int, len(urls))
+	if len(mdxUrls) != 0 && *shardByURL {
+		logger.Panicf("-remoteWrite.shardByURL should be false when -mdx.remoteWrite.url is set.")
+	}
+	rwctxs := make([]*remoteWriteCtx, len(urls)+len(mdxUrls))
+	rwctxIdx := make([]int, len(urls)+len(mdxUrls))
 	if retryMaxTime.String() != "" {
 		logger.Warnf("-remoteWrite.retryMaxTime is deprecated; use -remoteWrite.retryMaxInterval instead")
 	}
@@ -301,16 +477,33 @@ func initRemoteWriteCtxs(urls []string) {
 		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, sanitizedURL)
 		rwctxIdx[i] = i
 	}
-	fs.RegisterPathFsMetrics(*tmpDataPath)
+
+	normalRWLens := len(urls)
+	for i, remoteWriteURLRaw := range mdxUrls {
+		remoteWriteURL, err := url.Parse(remoteWriteURLRaw)
+		if err != nil {
+			logger.Fatalf("invalid -mdx.remoteWrite.url=%q: %s", remoteWriteURL, err)
+		}
+		sanitizedURL := fmt.Sprintf("%d:secret-url", normalRWLens+i+1)
+		if *showRemoteWriteURL {
+			sanitizedURL = fmt.Sprintf("%d:%s", normalRWLens+i+1, remoteWriteURL)
+		}
+		rwctxs[normalRWLens+i] = newRemoteWriteCtx(normalRWLens+i, remoteWriteURL, sanitizedURL)
+		rwctxIdx[normalRWLens+i] = normalRWLens + i
+	}
 
 	if *shardByURL {
-		consistentHashNodes := make([]string, 0, len(urls))
+		consistentHashNodes := make([]string, 0, normalRWLens)
 		for i, url := range urls {
 			consistentHashNodes = append(consistentHashNodes, fmt.Sprintf("%d:%s", i+1, url))
 		}
 		rwctxConsistentHashGlobal = consistenthash.NewConsistentHash(consistentHashNodes, 0)
 	}
 
+	if len(mdxUrls) != 0 {
+		mdx.InitGlobalFilter()
+	}
+
 	rwctxsGlobal = rwctxs
 	rwctxsGlobalIdx = rwctxIdx
 }
@@ -373,6 +566,7 @@ func Stop() {
 	if sl := dailySeriesLimiter; sl != nil {
 		sl.MustStop()
 	}
+	mdx.GlobalFilter.MustStop()
 }
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
@@ -515,9 +709,7 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 //
 // calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
 func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
-	// When -remoteWrite.shardByURL=true always use all configured remote writes to preserve stable metrics distribution across shards.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507
-	if !disableOnDiskQueueAny || *shardByURL {
+	if !disableOnDiskQueueAny {
 		return rwctxsGlobal, true
 	}
 
@@ -532,6 +724,12 @@ func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailu
 				return nil, false
 			}
 			rowsCount := getRowsCount(tss)
+			if *shardByURL {
+				// Todo: When shardByURL is enabled, the following metrics won't be 100% accurate. Because vmagent don't know
+				// which rwctx should data be pushed to yet. Let's consider the hashing algorithm fair and will distribute
+				// data to all rwctxs evenly.
+				rowsCount = rowsCount / len(rwctxsGlobal)
+			}
 			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 		}
 	}
@@ -711,7 +909,7 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
 			}
 			tmpLabels.Labels = hashLabels
 		}
-		h := getLabelsHashForShard(hashLabels)
+		h := getLabelsHash(hashLabels)
 
 		// Get the rwctxIdx through consistent hashing and then map it to the index in shards.
 		// The rwctxIdx is not always equal to the shardIdx, for example, when some rwctx are not available.
@@ -802,28 +1000,11 @@ var (
 	dailySeriesLimitRowsDropped  = metrics.NewCounter(`vmagent_daily_series_limit_rows_dropped_total`)
 )
 
-// getLabelsHashForShard is a separate function from getLabelsHash because
-// it omits the '=' separator between label name and value for backward compatibility.
-// Changing it would re-shard all series across remoteWrite targets.
-func getLabelsHashForShard(labels []prompb.Label) uint64 {
-	bb := labelsHashBufPool.Get()
-	b := bb.B[:0]
-	for _, label := range labels {
-		b = append(b, label.Name...)
-		b = append(b, label.Value...)
-	}
-	h := xxhash.Sum64(b)
-	bb.B = b
-	labelsHashBufPool.Put(bb)
-	return h
-}
-
 func getLabelsHash(labels []prompb.Label) uint64 {
 	bb := labelsHashBufPool.Get()
 	b := bb.B[:0]
 	for _, label := range labels {
 		b = append(b, label.Name...)
-		b = append(b, '=')
 		b = append(b, label.Value...)
 	}
 	h := xxhash.Sum64(b)
@@ -867,22 +1048,26 @@ type remoteWriteCtx struct {
 	// otherwise by the global -enableMetadata flag.
 	enableMetadata bool
 
+	enableMdx bool
+
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
 
 	rowsPushedAfterRelabel *metrics.Counter
 	rowsDroppedByRelabel   *metrics.Counter
+	rowsDroppedByMdx       *metrics.Counter
 
 	pushFailures                 *metrics.Counter
 	metadataDroppedOnPushFailure *metrics.Counter
 	rowsDroppedOnPushFailure     *metrics.Counter
+	flags                        *Flags
 }
 
 // isMetadataEnabledForURL returns true if metadata should be sent to the remote storage at argIdx.
 // It checks the per-URL -remoteWrite.disableMetadata flag first.
 // If not set, it falls back to the global -enableMetadata flag.
-func isMetadataEnabledForURL(argIdx int) bool {
-	if disableMetadataPerURL.GetOptionalArg(argIdx) {
+func isMetadataEnabledForURL(flags *Flags, argIdx int) bool {
+	if flags.disableMetadataPerURL.GetOptionalArg(argIdx) {
 		// Metadata is explicitly disabled for this URL
 		return false
 	}
@@ -891,28 +1076,36 @@ func isMetadataEnabledForURL(argIdx int) bool {
 }
 
 func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string) *remoteWriteCtx {
+	flags := remoteWriteFlags
+	flagPrefix := "remoteWrite"
+	if argIdx >= len(*remoteWriteURLs) {
+		// mdx remote write url
+		argIdx -= len(*remoteWriteURLs)
+		flags = mdxRemoteWriteFlags
+		flagPrefix = "mdx.remoteWrite"
+	}
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
 	pqURL.Fragment = ""
 	h := xxhash.Sum64([]byte(pqURL.String()))
 	queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
-	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArg(argIdx)
+	maxPendingBytes := flags.maxPendingBytesPerURL.GetOptionalArg(argIdx)
 	if maxPendingBytes != 0 && maxPendingBytes < persistentqueue.DefaultChunkFileSize {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195
-		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
+		logger.Warnf("rounding the -%s.maxDiskUsagePerURL=%d to the minimum supported value: %d", flagPrefix, maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
 
-	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
-	queuesSize := queues.GetOptionalArg(argIdx)
+	isPQDisabled := flags.disableOnDiskQueue.GetOptionalArg(argIdx)
+	queuesSize := flags.queues.GetOptionalArg(argIdx)
 	if queuesSize > maxQueues {
 		queuesSize = maxQueues
 	} else if queuesSize <= 0 {
 		queuesSize = 1
 	}
 
-	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / *maxRowsPerBlock / 100
+	maxInmemoryBlocks := memory.Allowed() / (len(*remoteWriteURLs) + len(*mdxRemoteWriteURLs)) / *maxRowsPerBlock / 100
 	if maxInmemoryBlocks/queuesSize > 100 {
 		// There is no much sense in keeping higher number of blocks in memory,
 		// since this means that the producer outperforms consumer and the queue
@@ -939,15 +1132,15 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
-		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, queuesSize)
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, queuesSize, flags)
 	default:
-		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
+		logger.Fatalf("unsupported scheme: %s for %sURL: %s, want `http`, `https`", flagPrefix, remoteWriteURL.Scheme, sanitizedURL)
 	}
-	c.init(argIdx, queuesSize, sanitizedURL)
+	c.init(argIdx, queuesSize, sanitizedURL, flags)
 
 	// Initialize pss
-	sf := significantFigures.GetOptionalArg(argIdx)
-	rd := roundDigits.GetOptionalArg(argIdx)
+	sf := flags.significantFigures.GetOptionalArg(argIdx)
+	rd := flags.roundDigits.GetOptionalArg(argIdx)
 	pssLen := queuesSize
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
@@ -958,16 +1151,22 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	for i := range pss {
 		pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
 	}
-
+	idx := argIdx
+	if flags.enableMdx {
+		idx += len(*remoteWriteURLs)
+	}
 	rwctx := &remoteWriteCtx{
-		idx:            argIdx,
+		idx:            idx,
 		fq:             fq,
 		c:              c,
 		pss:            pss,
-		enableMetadata: isMetadataEnabledForURL(argIdx),
+		enableMetadata: isMetadataEnabledForURL(flags, argIdx),
+		enableMdx:      flags.enableMdx,
+		flags:          flags,
 
 		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByMdx:       metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_mdx_rows_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 
 		pushFailures:                 metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 		metadataDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_metadata_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
@@ -1003,6 +1202,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 
 	rwctx.rowsPushedAfterRelabel = nil
 	rwctx.rowsDroppedByRelabel = nil
+	rwctx.rowsDroppedByMdx = nil
 }
 
 // TryPushTimeSeries sends tss series to the configured remote write endpoint
@@ -1020,17 +1220,30 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 		putRelabelCtx(rctx)
 	}()
 
+	if rwctx.enableMdx && mdx.GlobalFilter != nil {
+		// Make a copy of tss
+		rctx = getRelabelCtx()
+		rowsCountBeforeMdx := getRowsCount(tss)
+		v = tssPool.Get().(*[]prompb.TimeSeries)
+		tss = mdx.GlobalFilter.Filter(tss, *v)
+		rowsCountAfterMdx := getRowsCount(tss)
+		rwctx.rowsDroppedByMdx.Add(rowsCountBeforeMdx - rowsCountAfterMdx)
+	}
+
 	// Apply relabeling
 	rcs := allRelabelConfigs.Load()
 	pcs := rcs.perURL[rwctx.idx]
 	if pcs.Len() > 0 {
-		rctx = getRelabelCtx()
-		// Make a copy of tss before applying relabeling in order to prevent
-		// from affecting time series for other remoteWrite.url configs.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
-		// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
-		v = tssPool.Get().(*[]prompb.TimeSeries)
-		tss = append(*v, tss...)
+		if rctx == nil {
+			// Make a copy of tss before applying relabeling in order to prevent
+			// from affecting time series for other remoteWrite.url configs.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
+			// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
+			rctx = getRelabelCtx()
+			// Make a copy of tss before dropping aggregated series
+			v = tssPool.Get().(*[]prompb.TimeSeries)
+			tss = append(*v, tss...)
+		}
 		rowsCountBeforeRelabel := getRowsCount(tss)
 		tss = rctx.applyRelabeling(tss, pcs)
 		rowsCountAfterRelabel := getRowsCount(tss)

app/vmagent/remotewrite/remotewrite_test.go

@@ -25,7 +25,7 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		t.Helper()
 
 		// Distribute itemsCount hashes returned by getLabelsHash() across bucketsCount buckets.
-		itemsCount := 10_000 * bucketsCount
+		itemsCount := 1_000 * bucketsCount
 		m := make([]int, bucketsCount)
 		var labels []prompb.Label
 		for i := range itemsCount {
@@ -44,12 +44,10 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		}
 
 		// Verify that the distribution is even
-		expectedItemsPerBucket := float64(itemsCount / bucketsCount)
-		allowedDeviation := math.Round(float64(expectedItemsPerBucket) * 0.04)
+		expectedItemsPerBucket := itemsCount / bucketsCount
 		for _, n := range m {
-			if math.Abs(expectedItemsPerBucket-float64(n)) > allowedDeviation {
-				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want in range [%.0f, %.0f]",
-					bucketsCount, n, expectedItemsPerBucket-allowedDeviation, expectedItemsPerBucket+allowedDeviation)
+			if math.Abs(1-float64(n)/float64(expectedItemsPerBucket)) > 0.04 {
+				t.Fatalf("unexpected items in the bucket for %d buckets; got %d; want around %d", bucketsCount, n, expectedItemsPerBucket)
 			}
 		}
 	}

app/vmagent/remotewrite/streamaggr.go

@@ -65,6 +65,32 @@ var (
 	streamAggrEnableWindows = flagutil.NewArrayBool("remoteWrite.streamAggr.enableWindows", "Enables aggregation within fixed windows for all remote write's aggregators. "+
 		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
 		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
+
+	mdxStreamAggrConfig = flagutil.NewArrayString("mdx.remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
+		"See also -remoteWrite.streamAggr.keepInput, -mdx.remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
+	mdxStreamAggrDropInput = flagutil.NewArrayBool("mdx.remoteWrite.streamAggr.dropInput", "Whether to drop input samples that not matching any rule in "+
+		"the corresponding -remoteWrite.streamAggr.config. By default, only matched raw samples are dropped, while unmatched samples "+
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+	mdxStreamAggrKeepInput = flagutil.NewArrayBool("mdx.remoteWrite.streamAggr.keepInput", "Whether to keep input samples that match any rule in "+
+		"the corresponding -remoteWrite.streamAggr.config. By default, matched raw samples are aggregated and dropped, while unmatched samples "+
+		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+	mdxStreamAggrDedupInterval = flagutil.NewArrayDuration("mdx.remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
+		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
+	mdxStreamAggrIgnoreOldSamples = flagutil.NewArrayBool("mdx.remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
+		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples")
+	mdxStreamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("mdx.remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
+		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving buffered delayed data from clients pushing data into the vmagent. "+
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignore-aggregation-intervals-on-start")
+	mdxStreamAggrDropInputLabels = flagutil.NewArrayString("mdx.remoteWrite.streamAggr.dropInputLabels", "An optional list of labels to drop from samples "+
+		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
+		"Multiple labels per remoteWrite.url must be delimited by '^^': -mdx.remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels")
+	mdxStreamAggrEnableWindows = flagutil.NewArrayBool("mdx.remoteWrite.streamAggr.enableWindows", "Enables aggregation within fixed windows for all remote write's aggregators. "+
+		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
+		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows.")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
@@ -79,10 +105,20 @@ func CheckStreamAggrConfigs() error {
 	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
 		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
 	}
+	if len(*mdxStreamAggrConfig) > len(*mdxRemoteWriteURLs) {
+		return fmt.Errorf("too many -mdx.remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -mdx.remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
+	}
 
 	pushNoop := func(_ []prompb.TimeSeries) {}
 	for idx := range *streamAggrConfig {
-		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
+		sas, err := newStreamAggrConfigPerURL(idx, remoteWriteFlags, pushNoop)
+		if err != nil {
+			return err
+		}
+		sas.MustStop()
+	}
+	for idx := range *mdxStreamAggrConfig {
+		sas, err := newStreamAggrConfigPerURL(idx, mdxRemoteWriteFlags, pushNoop)
 		if err != nil {
 			return err
 		}
@@ -147,6 +183,9 @@ func initStreamAggrConfigGlobal() {
 
 func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
 	idx := rwctx.idx
+	if rwctx.enableMdx {
+		idx = idx - len(*remoteWriteURLs)
+	}
 
 	sas, err := rwctx.newStreamAggrConfig()
 	if err != nil {
@@ -155,24 +194,28 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
 	if sas != nil {
 		filePath := sas.FilePath()
 		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
+		rwctx.streamAggrKeepInput = rwctx.flags.streamAggrKeepInput.GetOptionalArg(idx)
+		rwctx.streamAggrDropInput = rwctx.flags.streamAggrDropInput.GetOptionalArg(idx)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
 	}
-	dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+	dedupInterval := rwctx.flags.streamAggrDedupInterval.GetOptionalArg(idx)
 	if dedupInterval > 0 {
 		alias := fmt.Sprintf("dedup-%d", idx+1)
 		var dropLabels []string
-		if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-			dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
+		if rwctx.flags.streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
+			dropLabels = strings.Split(rwctx.flags.streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
 		}
 		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, dropLabels, alias)
 	}
 }
 
 func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
-	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
+	idx := rwctx.idx
+	if rwctx.enableMdx {
+		idx = idx - len(*remoteWriteURLs)
+	}
+	path := rwctx.flags.streamAggrConfig.GetOptionalArg(idx)
 	if path == "" {
 		return
 	}
@@ -224,30 +267,34 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 }
 
 func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
-	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
+	idx := rwctx.idx
+	if rwctx.flags.enableMdx {
+		idx = idx - len(*remoteWriteURLs)
+	}
+	return newStreamAggrConfigPerURL(idx, rwctx.flags, rwctx.pushInternalTrackDropped)
 }
 
-func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path := streamAggrConfig.GetOptionalArg(idx)
+func newStreamAggrConfigPerURL(idx int, flags *Flags, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
+	path := flags.streamAggrConfig.GetOptionalArg(idx)
 	if path == "" {
 		return nil, nil
 	}
 
 	alias := fmt.Sprintf("%d:secret-url", idx+1)
 	if *showRemoteWriteURL {
-		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
+		alias = fmt.Sprintf("%d:%s", idx+1, flags.remoteWriteURLs.GetOptionalArg(idx))
 	}
 	var dropLabels []string
-	if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
-		dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
+	if flags.streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
+		dropLabels = strings.Split(flags.streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
 	}
 	opts := &streamaggr.Options{
-		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
+		DedupInterval:        flags.streamAggrDedupInterval.GetOptionalArg(idx),
 		DropInputLabels:      dropLabels,
-		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
-		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
-		EnableWindows:        streamAggrEnableWindows.GetOptionalArg(idx),
+		IgnoreOldSamples:     flags.streamAggrIgnoreOldSamples.GetOptionalArg(idx),
+		IgnoreFirstIntervals: flags.streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
+		KeepInput:            flags.streamAggrKeepInput.GetOptionalArg(idx),
+		EnableWindows:        flags.streamAggrEnableWindows.GetOptionalArg(idx),
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

app/vmalert-tool/unittest/unittest.go

@@ -61,7 +61,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	}
 	eu, err := url.Parse(externalURL)
 	if err != nil {
-		logger.Fatalf("failed to parse external URL: %s", err)
+		logger.Fatalf("failed to parse external URL: %w", err)
 	}
 	if err := templates.Load([]string{}, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)

app/vmalert/config/config.go

@@ -113,15 +113,15 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
 			if err := exprValidator(r.Expr); err != nil {
-				return fmt.Errorf("invalid expression for rule %q: %w", ruleName, err)
+				return fmt.Errorf("invalid expression for rule  %q: %w", ruleName, err)
 			}
 		}
 		if validateTplFn != nil {
 			if err := validateTplFn(r.Annotations); err != nil {
-				return fmt.Errorf("invalid annotations for rule %q: %w", ruleName, err)
+				return fmt.Errorf("invalid annotations for rule  %q: %w", ruleName, err)
 			}
 			if err := validateTplFn(r.Labels); err != nil {
-				return fmt.Errorf("invalid labels for rule %q: %w", ruleName, err)
+				return fmt.Errorf("invalid labels for rule  %q: %w", ruleName, err)
 			}
 		}
 	}

app/vmalert/config/config_test.go

@@ -121,7 +121,7 @@ func TestParse_Failure(t *testing.T) {
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
 	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/rules/rules1-bad.rules"}, "bad GraphiteQL expr")
+	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
 	f([]string{"testdata/rules/vlog-rules0-bad.rules"}, "bad LogsQL expr")
 	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
 	f([]string{"testdata/rules/rules-multi-doc-bad.rules"}, "unknown fields")
@@ -283,7 +283,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "up | 0",
 			},
 		},
-	}, true, "bad MetricsQL expr")
+	}, true, "bad prometheus expr")
 
 	f(&Group{
 		Name: "test graphite expr",
@@ -293,7 +293,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				"description": "some-description",
 			}},
 		},
-	}, true, "bad GraphiteQL expr")
+	}, true, "bad graphite expr")
 
 	f(&Group{
 		Name: "test vlogs expr",
@@ -327,7 +327,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				Expr:   "sum(up == 0 ) by (host)",
 			},
 		},
-	}, true, "bad GraphiteQL expr")
+	}, true, "bad graphite expr")
 
 	f(&Group{
 		Name: "test vlogs with prometheus exp",
@@ -351,7 +351,7 @@ func TestGroupValidate_Failure(t *testing.T) {
 				For:    promutil.NewDuration(10 * time.Millisecond),
 			},
 		},
-	}, true, "bad MetricsQL expr")
+	}, true, "bad prometheus expr")
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/config/types.go

@@ -66,11 +66,11 @@ func (t *Type) ValidateExpr(expr string) error {
 	switch t.String() {
 	case "graphite":
 		if _, err := graphiteql.Parse(expr); err != nil {
-			return fmt.Errorf("bad GraphiteQL expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad graphite expr: %q, err: %w", expr, err)
 		}
 	case "prometheus":
 		if _, err := metricsql.Parse(expr); err != nil {
-			return fmt.Errorf("bad MetricsQL expr: %q, err: %w", expr, err)
+			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
 	case "vlogs":
 		q, err := logstorage.ParseStatsQuery(expr, 0)

app/vmalert/datasource/client_test.go

@@ -772,7 +772,7 @@ func TestHeaders(t *testing.T) {
 
 	// basic auth
 	f(func() *Client {
-		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "", "bar", ""))
+		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
 		if err != nil {
 			t.Fatalf("Error get auth config: %s", err)
 		}
@@ -817,7 +817,7 @@ func TestHeaders(t *testing.T) {
 
 	// custom header overrides basic auth
 	f(func() *Client {
-		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "", "bar", ""))
+		cfg, err := vmalertutil.AuthConfig(vmalertutil.WithBasicAuth("foo", "bar", ""))
 		if err != nil {
 			t.Fatalf("Error get auth config: %s", err)
 		}

app/vmalert/datasource/init.go

@@ -27,7 +27,6 @@ var (
 		"Multiple headers must be delimited by '^^': -datasource.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
-	basicAuthUsernameFile = flag.String("datasource.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
 	basicAuthPasswordFile = flag.String("datasource.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -datasource.url")
 
@@ -64,7 +63,6 @@ func InitSecretFlags() {
 	if !*showDatasourceURL {
 		flagutil.RegisterSecretFlag("datasource.url")
 	}
-	flagutil.RegisterSecretFlag("datasource.headers")
 }
 
 // ShowDatasourceURL whether to show -datasource.url with sensitive information
@@ -107,7 +105,7 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -datasource.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/notifier/alertmanager.go

@@ -191,7 +191,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 
 	aCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(ba.Username, ba.UsernameFile, ba.Password.String(), ba.PasswordFile),
+		vmalertutil.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		vmalertutil.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
 		vmalertutil.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
 		vmalertutil.WithHeaders(strings.Join(authCfg.Headers, "^^")),

app/vmalert/notifier/config_watcher.go

@@ -105,7 +105,7 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, targetsFn
 			}
 			targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
 			for _, err := range errors {
-				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
+				logger.Errorf("failed to init notifier for %q: %w", typeK, err)
 			}
 			cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 		}
@@ -274,7 +274,7 @@ func (cw *configWatcher) updateTargets(key TargetType, targetMts map[string]targ
 	for addr, metadata := range targetMts {
 		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, metadata.alertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
-			logger.Errorf("failed to init %s notifier with addr %q: %s", key, addr, err)
+			logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
 			continue
 		}
 		updatedTargets = append(updatedTargets, Target{

app/vmalert/notifier/init.go

@@ -36,7 +36,6 @@ var (
 		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
 		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
-	basicAuthUsernameFile = flagutil.NewArrayString("notifier.basicAuth.usernameFile", "Optional path to basic auth username file for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
 
@@ -194,7 +193,6 @@ func InitSecretFlags() {
 	if !*showNotifierURL {
 		flagutil.RegisterSecretFlag("notifier.url")
 	}
-	flagutil.RegisterSecretFlag("notifier.headers")
 }
 
 func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
@@ -215,7 +213,6 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			},
 			BasicAuth: &promauth.BasicAuthConfig{
 				Username:     basicAuthUsername.GetOptionalArg(i),
-				UsernameFile: basicAuthUsernameFile.GetOptionalArg(i),
 				Password:     promauth.NewSecret(basicAuthPassword.GetOptionalArg(i)),
 				PasswordFile: basicAuthPasswordFile.GetOptionalArg(i),
 			},

app/vmalert/notifier/notifier.go

@@ -14,7 +14,7 @@ type Notifier interface {
 	Send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, notifierHeaders map[string]string) error
 	// Addr returns address where alerts are sent.
 	Addr() string
-	// LastError returns error, that occurred during last attempt to send data
+	// LastError returns error, that occured during last attempt to send data
 	LastError() string
 	// Close is a destructor for the Notifier
 	Close()

app/vmalert/remoteread/init.go

@@ -28,7 +28,6 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
-	basicAuthUsernameFile = flag.String("remoteRead.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
 	basicAuthPasswordFile = flag.String("remoteRead.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteRead.url")
 
@@ -59,7 +58,6 @@ func InitSecretFlags() {
 	if !*showRemoteReadURL {
 		flagutil.RegisterSecretFlag("remoteRead.url")
 	}
-	flagutil.RegisterSecretFlag("remoteRead.headers")
 }
 
 // Init creates a Querier from provided flag values.
@@ -82,7 +80,7 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/remotewrite/client.go

@@ -11,7 +11,6 @@ import (
 	"path"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/cespare/xxhash/v2"
@@ -19,8 +18,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
@@ -60,11 +57,6 @@ type Client struct {
 
 	wg     sync.WaitGroup
 	doneCh chan struct{}
-
-	// Whether to encode the write request with VictoriaMetrics remote write protocol.
-	// It is set to true by default, and will be switched to false if the client
-	// receives specific errors indicating that the remote storage doesn't support VictoriaMetrics remote write protocol.
-	isVMRemoteWrite atomic.Bool
 }
 
 // Config is config for remote write client.
@@ -124,7 +116,6 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 		doneCh:        make(chan struct{}),
 		input:         make(chan prompb.TimeSeries, cfg.MaxQueueSize),
 	}
-	c.isVMRemoteWrite.Store(true)
 
 	for i := 0; i < cc; i++ {
 		c.wg.Go(func() {
@@ -274,16 +265,8 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	defer wr.Reset()
 	defer bufferFlushDuration.UpdateDuration(time.Now())
 
-	bb := writeRequestBufPool.Get()
-	bb.B = wr.MarshalProtobuf(bb.B[:0])
-	zb := compressBufPool.Get()
-	defer compressBufPool.Put(zb)
-	if c.isVMRemoteWrite.Load() {
-		zb.B = zstd.CompressLevel(zb.B[:0], bb.B, 0)
-	} else {
-		zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B)
-	}
-	writeRequestBufPool.Put(bb)
+	data := wr.MarshalProtobuf(nil)
+	b := snappy.Encode(nil, data)
 
 	maxRetryInterval := *retryMaxTime
 	bt := timeutil.NewBackoffTimer(*retryMinInterval, maxRetryInterval)
@@ -295,17 +278,17 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	attempts := 0
 L:
 	for {
-		err := c.send(ctx, zb.B)
+		err := c.send(ctx, b)
 		if err != nil && (errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err)) {
 			// Something in the middle between client and destination might be closing
 			// the connection. So we do a one more attempt in hope request will succeed.
-			err = c.send(ctx, zb.B)
+			err = c.send(ctx, b)
 		}
 		if err == nil {
 			sentRows.Add(len(wr.Timeseries))
-			sentBytes.Add(len(zb.B))
+			sentBytes.Add(len(b))
 			flushedRows.Update(float64(len(wr.Timeseries)))
-			flushedBytes.Update(float64(len(zb.B)))
+			flushedBytes.Update(float64(len(b)))
 			return
 		}
 
@@ -357,16 +340,12 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
 
-	req.Header.Set("User-Agent", "vmalert")
+	// RFC standard compliant headers
+	req.Header.Set("Content-Encoding", "snappy")
 	req.Header.Set("Content-Type", "application/x-protobuf")
 
-	if encoding.IsZstd(data) {
-		req.Header.Set("Content-Encoding", "zstd")
-		req.Header.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
-	} else {
-		req.Header.Set("Content-Encoding", "snappy")
-		req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	}
+	// Prometheus compliant headers
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
 
 	if c.authCfg != nil {
 		err = c.authCfg.SetHeaders(req, true)
@@ -395,29 +374,6 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 		// respond with HTTP 2xx status code when write is successful.
 		return nil
 	case 4:
-		// - Remote Write v1 specification implicitly expects a `400 Bad Request` when the encoding is not supported.
-		// - Remote Write v2 specification explicitly specifies a `415 Unsupported Media Type` for unsupported encodings.
-		// - Real-world implementations of v1 use both 400 and 415 status codes.
-		// See more in research: https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462#issuecomment-2786918054
-		if resp.StatusCode == http.StatusUnsupportedMediaType || resp.StatusCode == http.StatusBadRequest {
-			if encoding.IsZstd(data) {
-				logger.Infof("received unsupported media type or bad request from remote storage at %q. Re-packing the block to Prometheus remote write and retrying."+
-					"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", req.URL.Redacted())
-				zstdBlockLen := len(data)
-				data, err = repackBlockFromZstdToSnappy(data)
-				if err == nil {
-					logger.Infof("received unsupported media type or bad request from remote storage at %q. Downgrading protocol from VictoriaMetrics to Prometheus remote write for all future requests. "+
-						"See https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol", req.URL.Redacted())
-					c.isVMRemoteWrite.Store(false)
-					return c.send(ctx, data)
-				}
-
-				logger.Warnf("failed to repack zstd block (%d bytes) to snappy: %s; The block will be rejected. "+
-					"Possible cause: ungraceful shutdown leading to persisted queue corruption.",
-					zstdBlockLen, err)
-			}
-		}
-
 		if resp.StatusCode != http.StatusTooManyRequests {
 			// MUST NOT retry write requests on HTTP 4xx responses other than 429
 			return &nonRetriableError{
@@ -438,19 +394,3 @@ type nonRetriableError struct {
 func (e *nonRetriableError) Error() string {
 	return e.err.Error()
 }
-
-var (
-	writeRequestBufPool bytesutil.ByteBufferPool
-	compressBufPool     bytesutil.ByteBufferPool
-)
-
-// repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
-func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
-	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
-	if err != nil {
-		return nil, err
-	}
-
-	return snappy.Encode(nil, plainBlock), nil
-}

app/vmalert/remotewrite/client_test.go

@@ -12,7 +12,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/golang/snappy"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
@@ -102,10 +103,7 @@ func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
 
 		// push time series to the client.
 		for range pushCnt {
-			if err = rwClient.Push(prompb.TimeSeries{
-				Labels:  []prompb.Label{{Name: "__name__", Value: "m"}},
-				Samples: []prompb.Sample{{Value: 1, Timestamp: 1000}},
-			}); err != nil {
+			if err = rwClient.Push(prompb.TimeSeries{}); err != nil {
 				t.Fatalf("cannot time series to the client: %s", err)
 			}
 		}
@@ -158,8 +156,8 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	h := r.Header.Get("Content-Encoding")
-	if h != "zstd" {
-		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not zstd (%q)", h))
+	if h != "snappy" {
+		rw.err(w, fmt.Errorf("header read error: Content-Encoding is not snappy (%q)", h))
 	}
 
 	h = r.Header.Get("Content-Type")
@@ -167,9 +165,9 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 		rw.err(w, fmt.Errorf("header read error: Content-Type is not x-protobuf (%q)", h))
 	}
 
-	h = r.Header.Get("X-VictoriaMetrics-Remote-Write-Version")
-	if h != "1" {
-		rw.err(w, fmt.Errorf("header read error: X-VictoriaMetrics-Remote-Write-Version is not 1 (%q)", h))
+	h = r.Header.Get("X-Prometheus-Remote-Write-Version")
+	if h != "0.1.0" {
+		rw.err(w, fmt.Errorf("header read error: X-Prometheus-Remote-Write-Version is not 0.1.0 (%q)", h))
 	}
 
 	data, err := io.ReadAll(r.Body)
@@ -179,7 +177,7 @@ func (rw *rwServer) handler(w http.ResponseWriter, r *http.Request) {
 	}
 	defer func() { _ = r.Body.Close() }()
 
-	b, err := zstd.Decompress(nil, data)
+	b, err := snappy.Decode(nil, data)
 	if err != nil {
 		rw.err(w, fmt.Errorf("decode err: %w", err))
 		return

app/vmalert/remotewrite/debug_client.go

@@ -9,7 +9,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
+	"github.com/golang/snappy"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -63,17 +64,19 @@ func (c *DebugClient) Close() error {
 }
 
 func (c *DebugClient) send(data []byte) error {
-	b := zstd.CompressLevel(nil, data, 0)
+	b := snappy.Encode(nil, data)
 	r := bytes.NewReader(b)
 	req, err := http.NewRequest(http.MethodPost, c.addr, r)
 	if err != nil {
 		return fmt.Errorf("failed to create new HTTP request: %w", err)
 	}
 
-	req.Header.Set("Content-Encoding", "zstd")
+	// RFC standard compliant headers
+	req.Header.Set("Content-Encoding", "snappy")
 	req.Header.Set("Content-Type", "application/x-protobuf")
 
-	req.Header.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
+	// Prometheus compliant headers
+	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
 
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")

app/vmalert/remotewrite/init.go

@@ -13,8 +13,8 @@ import (
 )
 
 var (
-	addr = flag.String("remoteWrite.url", "", "Optional URL to persist alerts state and recording rules results in form of timeseries. "+
-		"It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. "+
+	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
+		"and recording rules results in form of timeseries. "+
 		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
@@ -26,7 +26,6 @@ var (
 		"Multiple headers must be delimited by '^^': -remoteWrite.headers='header1:value1^^header2:value2'")
 
 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
-	basicAuthUsernameFile = flag.String("remoteWrite.basicAuth.usernameFile", "", "Optional path to basic auth username to use for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
 	basicAuthPasswordFile = flag.String("remoteWrite.basicAuth.passwordFile", "", "Optional path to basic auth password to use for -remoteWrite.url")
 
@@ -62,7 +61,6 @@ func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		flagutil.RegisterSecretFlag("remoteWrite.url")
 	}
-	flagutil.RegisterSecretFlag("remoteWrite.headers")
 }
 
 // Init creates Client object from given flags.
@@ -85,7 +83,7 @@ func Init(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteWrite.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
 	}
 	authCfg, err := vmalertutil.AuthConfig(
-		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthUsernameFile, *basicAuthPassword, *basicAuthPasswordFile),
+		vmalertutil.WithBasicAuth(*basicAuthUsername, *basicAuthPassword, *basicAuthPasswordFile),
 		vmalertutil.WithBearer(*bearerToken, *bearerTokenFile),
 		vmalertutil.WithOAuth(*oauth2ClientID, *oauth2ClientSecret, *oauth2ClientSecretFile, *oauth2TokenURL, *oauth2Scopes, endpointParams),
 		vmalertutil.WithHeaders(*headers))

app/vmalert/rule/group.go

@@ -8,7 +8,6 @@ import (
 	"hash/fnv"
 	"maps"
 	"net/url"
-	"path"
 	"sync"
 	"time"
 
@@ -43,9 +42,6 @@ var (
 		"For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 	maxStartDelay = flag.Duration("group.maxStartDelay", 5*time.Minute, "Defines the max delay before starting the group evaluation. Group's start is artificially delayed for random duration on interval"+
 		" [0..min(--group.maxStartDelay, group.interval)]. This helps smoothing out the load on the configured datasource, so evaluations aren't executed too close to each other.")
-	ruleStripFilePath = flag.Bool("rule.stripFilePath", false, "Whether to strip rule file paths in logs and all API responses, including /metrics. "+
-		"For example, file path '/path/to/tenant_id/rules.yml' will be stripped to 'groupHashID/rules.yml'. "+
-		"This flag may be useful for hiding sensitive information in file paths, such as S3 bucket details.")
 )
 
 // Group is an entity for grouping rules
@@ -151,12 +147,6 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 		g.EvalDelay = &cfg.EvalDelay.D
 	}
 	g.id = g.CreateID()
-	// strip file path from group.File after generated group ID when ruleStripFilePath is set,
-	// so it won't be exposed in logs and api responses
-	if *ruleStripFilePath {
-		_, filename := path.Split(g.File)
-		g.File = fmt.Sprintf("%d/%s", g.id, filename)
-	}
 	for _, h := range cfg.Headers {
 		g.Headers[h.Key] = h.Value
 	}

app/vmalert/rule/group_test.go

@@ -742,64 +742,3 @@ func parseTime(t *testing.T, s string) time.Time {
 	}
 	return tt
 }
-
-func TestRuleStripFilePath(t *testing.T) {
-	configG := config.Group{
-		Name:        "group",
-		File:        "/var/local/test/rules.yaml",
-		Type:        config.NewRawType("prometheus"),
-		Concurrency: 1,
-		Rules: []config.Rule{
-			{
-				ID:    0,
-				Alert: "alert",
-			},
-			{
-				ID:     1,
-				Record: "record",
-			},
-		}}
-	qb := &datasource.FakeQuerier{}
-	g := NewGroup(configG, qb, 1*time.Minute, nil)
-
-	gID := g.id
-	if g.File != "/var/local/test/rules.yaml" {
-		t.Fatalf("expected file path to be unchanged; got %q instead", g.File)
-	}
-
-	for _, r := range g.Rules {
-		if ar, ok := r.(*AlertingRule); ok {
-			if ar.File != "/var/local/test/rules.yaml" {
-				t.Fatalf("expected rule file path to be unchanged; got %q instead", ar.File)
-			}
-		}
-		if rr, ok := r.(*RecordingRule); ok {
-			if rr.File != "/var/local/test/rules.yaml" {
-				t.Fatalf("expected rule file path to be unchanged; got %q instead", rr.File)
-			}
-		}
-	}
-
-	oldRuleStripFilePath := *ruleStripFilePath
-	*ruleStripFilePath = true
-	defer func() {
-		*ruleStripFilePath = oldRuleStripFilePath
-	}()
-	g = NewGroup(configG, qb, 1*time.Minute, nil)
-
-	if g.File != fmt.Sprintf("%d/rules.yaml", gID) {
-		t.Fatalf("expected file path to be stripped to %q; got %q instead", fmt.Sprintf("%d/rules.yaml", gID), g.File)
-	}
-	for _, r := range g.Rules {
-		if ar, ok := r.(*AlertingRule); ok {
-			if ar.File != fmt.Sprintf("%d/rules.yaml", gID) {
-				t.Fatalf("expected rule file path to be unchanged; got %q instead", ar.File)
-			}
-		}
-		if rr, ok := r.(*RecordingRule); ok {
-			if rr.File != fmt.Sprintf("%d/rules.yaml", gID) {
-				t.Fatalf("expected rule file path to be unchanged; got %q instead", rr.File)
-			}
-		}
-	}
-}

app/vmalert/rule/web.go

@@ -252,9 +252,6 @@ func (r *ApiRule) ExtendState() {
 
 // ToAPI returns ApiGroup representation of g
 func (g *Group) ToAPI() *ApiGroup {
-	if g == nil {
-		return &ApiGroup{}
-	}
 	g.mu.RLock()
 	defer g.mu.RUnlock()
 	ag := ApiGroup{

app/vmalert/templates/template.go

@@ -402,20 +402,6 @@ func templateFuncs() textTpl.FuncMap {
 			return t, nil
 		},
 
-		// formatTime formats the given Unix timestamp with the provided layout.
-		// For example: {{ now | formatTime "2006-01-02T15:04:05Z07:00" }}
-		"formatTime": func(layout string, i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", fmt.Errorf("formatTime: %w", err)
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return "", fmt.Errorf("formatTime: cannot convert %v to time", v)
-			}
-			t := timeFromUnixTimestamp(v).Time().UTC()
-			return t.Format(layout), nil
-		},
-
 		/* URLs */
 
 		// externalURL returns value of `external.url` flag

app/vmalert/templates/template_test.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 	"testing"
 	textTpl "text/template"
-	"time"
 )
 
 func TestTemplateFuncs_StringConversion(t *testing.T) {
@@ -104,26 +103,6 @@ func TestTemplateFuncs_Formatting(t *testing.T) {
 	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
 }
 
-func TestTemplateFuncs_FormatTime(t *testing.T) {
-	funcs := templateFuncs()
-	formatTime := funcs["formatTime"].(func(layout string, i any) (string, error))
-
-	f := func(layout string, input any, expected string) {
-		t.Helper()
-		result, err := formatTime(layout, input)
-		if err != nil {
-			t.Fatalf("unexpected error for formatTime(%q, %v): %s", layout, input, err)
-		}
-		if result != expected {
-			t.Fatalf("unexpected result for formatTime(%q, %v); got\n%s\nwant\n%s", layout, input, result, expected)
-		}
-	}
-
-	f(time.RFC3339, float64(1679055557), "2023-03-17T12:19:17Z")
-	f("2006-01-02T15:04:05", int64(1679055557), "2023-03-17T12:19:17")
-	f(time.RFC822, int(1679055557), "17 Mar 23 12:19 UTC")
-}
-
 func mkTemplate(current, replacement any) textTemplate {
 	tmpl := textTemplate{}
 	if current != nil {

app/vmalert/vmalertutil/auth.go

@@ -20,12 +20,11 @@ func AuthConfig(filterOptions ...AuthConfigOptions) (*promauth.Config, error) {
 }
 
 // WithBasicAuth returns AuthConfigOptions and initialized promauth.BasicAuthConfig based on given params
-func WithBasicAuth(username, usernameFile, password, passwordFile string) AuthConfigOptions {
+func WithBasicAuth(username, password, passwordFile string) AuthConfigOptions {
 	return func(config *promauth.HTTPClientConfig) {
-		if username != "" || usernameFile != "" || password != "" || passwordFile != "" {
+		if username != "" || password != "" || passwordFile != "" {
 			config.BasicAuth = &promauth.BasicAuthConfig{
 				Username:     username,
-				UsernameFile: usernameFile,
 				Password:     promauth.NewSecret(password),
 				PasswordFile: passwordFile,
 			}

app/vmauth/auth_config.go

@@ -610,7 +610,6 @@ func areEqualBackendURLs(a, b []*backendURL) bool {
 }
 
 // getFirstAvailableBackendURL returns the first available backendURL, which isn't broken.
-// If all backendURLs are broken, then returns the first backendURL.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
@@ -629,22 +628,21 @@ func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 			return bu
 		}
 	}
-
-	// All backend urls are unavailable, then returning a first one, it could help increase the success rate of the requests。
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10837#issuecomment-4307050980.
-	bu.get()
-	return bu
+	return nil
 }
 
 // getLeastLoadedBackendURL returns a non-broken backendURL with the lowest number of concurrent requests.
-// If all backendURLs are broken, then returns the first backendURL.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
-	firstBu := bus[0]
 	if len(bus) == 1 {
-		firstBu.get()
-		return firstBu
+		// Fast path - return the only backend url.
+		bu := bus[0]
+		if bu.isBroken() {
+			return nil
+		}
+		bu.get()
+		return bu
 	}
 
 	// Slow path - select other backend urls.
@@ -682,10 +680,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	}
 	buMin := bus[buMinIdx]
 	if buMin.isBroken() {
-		// If all backendURLs are broken, then returns the first backendURL.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10837#issuecomment-4307050980.
-		firstBu.get()
-		return firstBu
+		return nil
 	}
 	buMin.get()
 	atomicCounter.CompareAndSwap(n+1, buMinIdx+1)

app/vmauth/auth_config_test.go

@@ -1031,33 +1031,6 @@ func TestLogRequest(t *testing.T) {
 	f("foo", 404, 10*time.Millisecond, `access_log request_host="localhost:8080" request_uri="" status_code=404 remote_addr="" user_agent="" referer="" duration_ms=10 username="foo"`)
 }
 
-func TestGetFirstAvailableBackend(t *testing.T) {
-	f := func(broken []bool, expectedIdx int) {
-		t.Helper()
-		bus := make([]*backendURL, len(broken))
-		for i := range broken {
-			bus[i] = &backendURL{
-				url: &url.URL{Host: fmt.Sprintf("server-%d", i)},
-			}
-			bus[i].broken.Store(broken[i])
-		}
-		bu := getFirstAvailableBackendURL(bus)
-		if bu == nil {
-			t.Fatalf("unexpected nil backend")
-		}
-		if bu.url.Host != fmt.Sprintf("server-%d", expectedIdx) {
-			t.Fatalf("unexpected backend, expected server-%d, got %s", expectedIdx, bu.url.Host)
-		}
-	}
-
-	f([]bool{false, false, false}, 0)
-	f([]bool{true, true, false}, 2)
-	// all backend are broken, then return the first one.
-	f([]bool{true, true, true}, 0)
-	f([]bool{true}, 0)
-
-}
-
 func getRegexs(paths []string) []*Regex {
 	var sps []*Regex
 	for _, path := range paths {

app/vmauth/example_config.yml

@@ -130,16 +130,6 @@ users:
   - "http://vmselect1:8481/select/{{.MetricsTenant}}/prometheus"
   - "http://vmselect2:8481/select/{{.MetricsTenant}}/prometheus"
 
-# JWT-based routing using header-based tenant identification (VictoriaMetrics cluster)
-# The AccountID and ProjectID from JWT vm_access claims are injected as HTTP headers.
-- name: jwt-header-tenant
-  jwt:
-    skip_verify: true
-  headers:
-  - "AccountID: {{.MetricsAccountID}}"
-  - "ProjectID: {{.MetricsProjectID}}"
-  url_prefix: "http://vminsert:8480/insert/prometheus"
-
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.

app/vmauth/jwt.go

@@ -17,8 +17,6 @@ import (
 
 const (
 	metricsTenantPlaceholder       = `{{.MetricsTenant}}`
-	metricsAccountIDPlaceholder    = `{{.MetricsAccountID}}`
-	metricsProjectIDPlaceholder    = `{{.MetricsProjectID}}`
 	metricsExtraLabelsPlaceholder  = `{{.MetricsExtraLabels}}`
 	metricsExtraFiltersPlaceholder = `{{.MetricsExtraFilters}}`
 
@@ -32,8 +30,6 @@ const (
 
 var allPlaceholders = []string{
 	metricsTenantPlaceholder,
-	metricsAccountIDPlaceholder,
-	metricsProjectIDPlaceholder,
 	metricsExtraLabelsPlaceholder,
 	metricsExtraFiltersPlaceholder,
 	logsAccountIDPlaceholder,
@@ -44,8 +40,6 @@ var allPlaceholders = []string{
 
 var urlPathPlaceHolders = []string{
 	metricsTenantPlaceholder,
-	metricsAccountIDPlaceholder,
-	metricsProjectIDPlaceholder,
 	logsAccountIDPlaceholder,
 	logsProjectIDPlaceholder,
 }
@@ -377,8 +371,6 @@ func jwtClaimsData(vma *jwt.VMAccessClaim) map[string][]string {
 	data := map[string][]string{
 		// TODO: optimize at parsing stage
 		metricsTenantPlaceholder:       {fmt.Sprintf("%d:%d", vma.MetricsAccountID, vma.MetricsProjectID)},
-		metricsAccountIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsAccountID)},
-		metricsProjectIDPlaceholder:    {fmt.Sprintf("%d", vma.MetricsProjectID)},
 		metricsExtraLabelsPlaceholder:  vma.MetricsExtraLabels,
 		metricsExtraFiltersPlaceholder: vma.MetricsExtraFilters,
 

app/vmauth/jwt_test.go

@@ -170,13 +170,13 @@ users:
   url_prefix: http://foo.bar
 `, "cannot parse public key from file \""+publicKeyFile+"\": failed to parse key \"invalidPEM\": failed to decode PEM block containing public key")
 
-	// unsupported placeholder in a URL path
+	// unsupported placeholder in a header
 	f(`
 users:
 - jwt: 
     skip_verify: true
   url_prefix: http://foo.bar/{{.UnsupportedPlaceholder}}/foo`,
-		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 	// unsupported placeholder in a header
 	f(`
@@ -187,7 +187,7 @@ users:
     - "AccountID: {{.UnsupportedPlaceholder}}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// spaces in templating not allowed
@@ -199,19 +199,7 @@ users:
     - "AccountID: {{ .LogsAccountID }}"
   url_prefix: http://foo.bar
 `,
-		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
-	)
-
-	// placeholder must match the entire header value
-	f(`
-users:
-- jwt: 
-    skip_verify: true
-  headers:
-    - "AccountID: foo {{.MetricsAccountID}}"
-  url_prefix: http://foo.bar
-`,
-		"request header: \"AccountID\" has unsupported placeholder: \"foo {{.MetricsAccountID}}\", supported values are: {{.MetricsTenant}}, {{.MetricsAccountID}}, {{.MetricsProjectID}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
 	)
 
 	// oidc is not an object
@@ -376,25 +364,10 @@ users:
   url_prefix: http://foo.bar
 `, validRSAPublicKey, validECDSAPublicKey))
 
-	// metrics header placeholders
 	f(`
 users:
 - jwt:
     skip_verify: true
-  headers:
-  - "MetricsAccountID: {{.MetricsAccountID}}"
-  - "MetricsProjectID: {{.MetricsProjectID}}"
-  url_prefix: http://foo.bar
-`)
-
-	// logs header placeholders
-	f(`
-users:
-- jwt:
-    skip_verify: true
-  headers:
-  - "LogsAccountID: {{.LogsAccountID}}"
-  - "LogsProjectID: {{.LogsProjectID}}"
   url_prefix: http://foo.bar
 `)
 

app/vmauth/main.go

@@ -51,7 +51,7 @@ var (
 		"This allows reducing the consumption of backend resources when processing requests from clients connected via slow networks. "+
 		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
 	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
-		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables retries. "+
+		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
 		"See also -requestBufferSize")
 
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process simultaneously. "+
@@ -850,18 +850,14 @@ func (bb *bufferedBody) Read(p []byte) (int, error) {
 }
 
 func (bb *bufferedBody) canRetry() bool {
-	if bb.r != nil {
-		return false
-	}
-	maxRetrySize := maxRequestBodySizeToRetry.IntN()
-	return len(bb.buf) == 0 || (maxRetrySize > 0 && len(bb.buf) <= maxRetrySize)
+	return bb.r == nil
 }
 
 // Close implements io.Closer interface.
 func (bb *bufferedBody) Close() error {
 	bb.resetReader()
-	bb.cannotRetry = !bb.canRetry()
 	if bb.r != nil {
+		bb.cannotRetry = true
 		return bb.r.Close()
 	}
 	return nil

app/vmauth/main_test.go

@@ -19,7 +19,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
@@ -851,30 +850,6 @@ users:
 		responseExpected,
 	)
 
-	// test header injection and URL templating with individual placeholders
-	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
-	request.Header.Set(`Authorization`, `Bearer `+fullToken)
-	responseExpected = `
-statusCode=200
-path: /select/123/234/api/v1/query
-query:
-headers:
-    AccountID=123
-    ProjectID=234`
-	f(fmt.Sprintf(
-		`
-users:
-- jwt:
-    public_keys:
-    - %q
-  url_prefix: {BACKEND}/select/{{.MetricsAccountID}}/{{.MetricsProjectID}}
-  headers:
-  - "AccountID: {{.MetricsAccountID}}"
-  - "ProjectID: {{.MetricsProjectID}}"`, string(publicKeyPEM)),
-		request,
-		responseExpected,
-	)
-
 	// extra_label and extra_filters from vm_access claim merged with statically defined
 	request = httptest.NewRequest(`GET`, "http://some-host.com/api/v1/query", nil)
 	request.Header.Set(`Authorization`, `Bearer `+fullToken)
@@ -1856,7 +1831,7 @@ func (r *mockBody) Read(p []byte) (n int, err error) {
 }
 
 func TestBufferedBody_RetrySuccess(t *testing.T) {
-	f := func(s string, maxSizeToRetry, bufferSize int) {
+	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		defaultRequestBufferSize := requestBufferSize.String()
@@ -1865,7 +1840,7 @@ func TestBufferedBody_RetrySuccess(t *testing.T) {
 				t.Fatalf("cannot reset requestBufferSize: %s", err)
 			}
 		}()
-		if err := requestBufferSize.Set(strconv.Itoa(bufferSize)); err != nil {
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
 			t.Fatalf("cannot set requestBufferSize: %s", err)
 		}
 
@@ -1875,7 +1850,7 @@ func TestBufferedBody_RetrySuccess(t *testing.T) {
 				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
 			}
 		}()
-		if err := maxRequestBodySizeToRetry.Set(strconv.Itoa(maxSizeToRetry)); err != nil {
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
 			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
 		}
 
@@ -1904,20 +1879,16 @@ func TestBufferedBody_RetrySuccess(t *testing.T) {
 		}
 	}
 
-	f("", 0, 2000)
-	f("", 0, 0)
-	f("", -1, 2000)
-	f("", 100, 2000)
-	f("foo", 100, 2000)
-	f("foobar", 100, 2000)
-	f("foobar", 100, 0)
-	f("foobar", 100, -1)
-	f(newTestString(1000), 1001, 2000)
-	f(newTestString(1000), 1001, 500)
+	f("", 0)
+	f("", -1)
+	f("", 100)
+	f("foo", 100)
+	f("foobar", 100)
+	f(newTestString(1000), 1001)
 }
 
 func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
-	f := func(s string, maxSizeToRetry, bufferSize int) {
+	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		// Check the case with partial read
@@ -1927,7 +1898,7 @@ func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 				t.Fatalf("cannot reset requestBufferSize: %s", err)
 			}
 		}()
-		if err := requestBufferSize.Set(strconv.Itoa(bufferSize)); err != nil {
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
 			t.Fatalf("cannot set requestBufferSize: %s", err)
 		}
 
@@ -1937,7 +1908,7 @@ func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
 			}
 		}()
-		if err := maxRequestBodySizeToRetry.Set(strconv.Itoa(maxSizeToRetry)); err != nil {
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
 			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
 		}
 
@@ -1981,20 +1952,16 @@ func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 		}
 	}
 
-	f("", 0, 2000)
-	f("", 0, 0)
-	f("", -1, 2000)
-	f("", 100, 2000)
-	f("foo", 100, 2000)
-	f("foobar", 100, 2000)
-	f("foobar", 100, 0)
-	f("foobar", 100, -1)
-	f(newTestString(1000), 1001, 2000)
-	f(newTestString(1000), 1001, 500)
+	f("", 0)
+	f("", -1)
+	f("", 100)
+	f("foo", 100)
+	f("foobar", 100)
+	f(newTestString(1000), 1001)
 }
 
 func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
-	f := func(s string, maxSizeToRetry, bufferSize int) {
+	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		defaultRequestBufferSize := requestBufferSize.String()
@@ -2003,7 +1970,7 @@ func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 				t.Fatalf("cannot reset requestBufferSize: %s", err)
 			}
 		}()
-		if err := requestBufferSize.Set(strconv.Itoa(bufferSize)); err != nil {
+		if err := requestBufferSize.Set("0"); err != nil {
 			t.Fatalf("cannot set requestBufferSize: %s", err)
 		}
 
@@ -2013,7 +1980,7 @@ func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
 			}
 		}()
-		if err := maxRequestBodySizeToRetry.Set(strconv.Itoa(maxSizeToRetry)); err != nil {
+		if err := maxRequestBodySizeToRetry.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
 			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
 		}
 
@@ -2058,17 +2025,12 @@ func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 	}
 
 	const maxBodySize = 1000
-	f(newTestString(maxBodySize+1), 0, 2*maxBodySize)
-	f(newTestString(maxBodySize+1), -1, 2*maxBodySize)
-	f(newTestString(maxBodySize+1), maxBodySize, 0)
-	f(newTestString(maxBodySize+1), maxBodySize, -1)
-	f(newTestString(maxBodySize+1), maxBodySize, maxBodySize)
-	f(newTestString(maxBodySize+1), maxBodySize, 2*maxBodySize)
-	f(newTestString(2*maxBodySize), maxBodySize, 0)
+	f(newTestString(maxBodySize+1), maxBodySize)
+	f(newTestString(2*maxBodySize), maxBodySize)
 }
 
-func TestBufferedBody_RetryDisabledByMaxRequestBodySizeToRetry(t *testing.T) {
-	f := func(s string, maxSizeToRetry, bufferSize int) {
+func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
+	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		defaultRequestBufferSize := requestBufferSize.String()
@@ -2077,20 +2039,10 @@ func TestBufferedBody_RetryDisabledByMaxRequestBodySizeToRetry(t *testing.T) {
 				t.Fatalf("cannot reset requestBufferSize: %s", err)
 			}
 		}()
-		if err := requestBufferSize.Set(strconv.Itoa(bufferSize)); err != nil {
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
 			t.Fatalf("cannot set requestBufferSize: %s", err)
 		}
 
-		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
-		defer func() {
-			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
-				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
-			}
-		}()
-		if err := maxRequestBodySizeToRetry.Set(strconv.Itoa(maxSizeToRetry)); err != nil {
-			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
-		}
-
 		ctx := context.Background()
 		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
 		if err != nil {
@@ -2099,8 +2051,8 @@ func TestBufferedBody_RetryDisabledByMaxRequestBodySizeToRetry(t *testing.T) {
 		bb, ok := rb.(*bufferedBody)
 		canRetry := !ok || bb.canRetry()
 
-		if canRetry {
-			t.Fatalf("canRetry() must return false before reading anything")
+		if !canRetry {
+			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		data, err := io.ReadAll(rb)
 		if err != nil {
@@ -2114,19 +2066,19 @@ func TestBufferedBody_RetryDisabledByMaxRequestBodySizeToRetry(t *testing.T) {
 		}
 
 		data, err = io.ReadAll(rb)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
+		if err != nil {
+			t.Fatalf("unexpected error in io.ReadAll: %s", err)
 		}
-		if len(data) != 0 {
-			t.Fatalf("unexpected non-empty data read: %q", data)
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
 	}
 
-	f("foobar", 0, 2048)
-	f(newTestString(1000), 0, 2048)
+	f("foobar", 0)
+	f(newTestString(1000), 0)
 
-	f("foobar", -1, 2048)
-	f(newTestString(1000), -1, 2048)
+	f("foobar", -1)
+	f(newTestString(1000), -1)
 }
 
 func newTestString(sLen int) string {

app/vmctl/vmctlutil/time_test.go

@@ -20,9 +20,6 @@ func TestGetTime_Failure(t *testing.T) {
 
 	// negative time
 	f("-292273086-05-16T16:47:06Z")
-
-	// relative duration that resolves to a timestamp before 1970
-	f("-9223372036.855")
 }
 
 func TestGetTime_Success(t *testing.T) {
@@ -80,6 +77,9 @@ func TestGetTime_Success(t *testing.T) {
 	// float timestamp representation",
 	f("1562529662.324", time.Date(2019, 7, 7, 20, 01, 02, 324e6, time.UTC))
 
+	// negative timestamp
+	f("-9223372036.855", time.Date(1970, 01, 01, 00, 00, 00, 00, time.UTC))
+
 	// big timestamp
 	f("1223372036855", time.Date(2008, 10, 7, 9, 33, 56, 855e6, time.UTC))
 

app/vminsert/main.go

@@ -89,7 +89,6 @@ var staticServer = http.FileServer(http.FS(staticFiles))
 func Init() {
 	relabel.Init()
 	common.InitStreamAggr()
-	opentelemetry.Init()
 	protoparserutil.StartUnmarshalWorkers()
 	if len(*graphiteListenAddr) > 0 {
 		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)

app/vminsert/opentelemetry/request_handler.go

@@ -20,11 +20,6 @@ var (
 	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="opentelemetry"}`)
 )
 
-// Init must be called after flag.Parse and before using the opentelemetry package.
-func Init() {
-	stream.InitDecodeOptions()
-}
-
 // InsertHandler processes opentelemetry metrics.
 func InsertHandler(req *http.Request) error {
 	extraLabels, err := protoparserutil.GetExtraLabels(req)

app/vmselect/main.go

@@ -60,17 +60,15 @@ func getDefaultMaxConcurrentRequests() int {
 
 // Init initializes vmselect
 func Init() {
-	tmpDirPath := vmstorage.DataPath() + "/tmp"
+	tmpDirPath := *vmstorage.DataPath + "/tmp"
 	fs.MustRemoveDirContents(tmpDirPath)
 	netstorage.InitTmpBlocksDir(tmpDirPath)
-	promql.InitRollupResultCache(vmstorage.DataPath() + "/cache/rollupResult")
+	promql.InitRollupResultCache(*vmstorage.DataPath + "/cache/rollupResult")
 	prometheus.InitMaxUniqueTimeseries(*maxConcurrentRequests)
 
 	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
 	initVMUIConfig()
 	initVMAlertProxy()
-
-	flagutil.RegisterSecretFlag("vmalert.proxyURL")
 }
 
 // Stop stops vmselect
@@ -264,7 +262,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/export":
 		exportRequests.Inc()
-		httpserver.EnableCORS(w, r)
 		if err := prometheus.ExportHandler(startTime, w, r); err != nil {
 			exportErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
@@ -273,7 +270,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/export/csv":
 		exportCSVRequests.Inc()
-		httpserver.EnableCORS(w, r)
 		if err := prometheus.ExportCSVHandler(startTime, w, r); err != nil {
 			exportCSVErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
@@ -282,7 +278,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/api/v1/export/native":
 		exportNativeRequests.Inc()
-		httpserver.EnableCORS(w, r)
 		if err := prometheus.ExportNativeHandler(startTime, w, r); err != nil {
 			exportNativeErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)

app/vmselect/netstorage/netstorage.go

@@ -990,6 +990,9 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 		return fmt.Errorf("timeout exceeded before starting data export: %s", deadline.String())
 	}
 	tr := sq.GetTimeRange()
+	if err := vmstorage.CheckTimeRange(tr); err != nil {
+		return err
+	}
 	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
 	if err != nil {
 		return err
@@ -1095,6 +1098,9 @@ func SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline
 
 	// Setup search.
 	tr := sq.GetTimeRange()
+	if err := vmstorage.CheckTimeRange(tr); err != nil {
+		return nil, err
+	}
 	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
 	if err != nil {
 		return nil, err
@@ -1121,6 +1127,9 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 
 	// Setup search.
 	tr := sq.GetTimeRange()
+	if err := vmstorage.CheckTimeRange(tr); err != nil {
+		return nil, err
+	}
 	tfss, err := setupTfss(qt, tr, sq.TagFilterss, sq.MaxMetrics, deadline)
 	if err != nil {
 		return nil, err

app/vmselect/searchutil/searchutil.go

@@ -132,20 +132,9 @@ func (d *Deadline) String() string {
 //
 //	{env="prod",team="devops",t1="v1",t2="v2"}
 //	{env=~"dev|staging",team!="devops",t1="v1",t2="v2"}
-//
-//	Query args from URL path have precedence over post form args.
 func GetExtraTagFilters(r *http.Request) ([][]storage.TagFilter, error) {
 	var tagFilters []storage.TagFilter
-	urlQueryValues := r.URL.Query()
-	getRequestParam := func(key string) []string {
-		// query request param must always take precedence over form values
-		// in order to simplify security enforcement policy for extra_label and extra_filters
-		if uv, ok := urlQueryValues[key]; ok {
-			return uv
-		}
-		return r.Form[key]
-	}
-	for _, match := range getRequestParam("extra_label") {
+	for _, match := range r.Form["extra_label"] {
 		tmp := strings.SplitN(match, "=", 2)
 		if len(tmp) != 2 {
 			return nil, fmt.Errorf("`extra_label` query arg must have the format `name=value`; got %q", match)
@@ -159,8 +148,8 @@ func GetExtraTagFilters(r *http.Request) ([][]storage.TagFilter, error) {
 			Value: []byte(tmp[1]),
 		})
 	}
-	extraFilters := append([]string{}, getRequestParam("extra_filters")...)
-	extraFilters = append(extraFilters, getRequestParam("extra_filters[]")...)
+	extraFilters := append([]string{}, r.Form["extra_filters"]...)
+	extraFilters = append(extraFilters, r.Form["extra_filters[]"]...)
 	if len(extraFilters) == 0 {
 		if len(tagFilters) == 0 {
 			return nil, nil

app/vmselect/searchutil/searchutil_test.go

@@ -20,7 +20,6 @@ func TestGetExtraTagFilters(t *testing.T) {
 		}
 		return &http.Request{
 			Form: q,
-			URL:  &url.URL{RawQuery: q.Encode()},
 		}
 	}
 	f := func(t *testing.T, r *http.Request, want []string, wantErr bool) {
@@ -80,24 +79,6 @@ func TestGetExtraTagFilters(t *testing.T) {
 		nil,
 		false,
 	)
-
-	formValues, err := url.ParseQuery(`extra_label=env=prod&extra_label=job=vmsingle&extra_label=tenant=prod&extra_filters[]={foo="bar"}&extra_filters[]={tenant="prod"}`)
-	if err != nil {
-		t.Fatalf("BUG: cannot parse query: %s", err)
-	}
-	urlValues, err := url.ParseQuery(`extra_label=job=vmagent&extra_label=env=dev&extra_filters[]={tenant="dev"}`)
-	if err != nil {
-		t.Fatalf("BUG: cannot parse query: %s", err)
-	}
-	httpReqWithBothFormAndURLParams := &http.Request{
-		Form: formValues,
-		URL: &url.URL{
-			RawQuery: urlValues.Encode(),
-		},
-	}
-	f(t, httpReqWithBothFormAndURLParams,
-		[]string{`{tenant="dev",job="vmagent",env="dev"}`},
-		false)
 }
 
 func TestParseMetricSelectorSuccess(t *testing.T) {

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-BjJ7fDL7.js"></script>
+  <script type="module" crossorigin src="./assets/index-fsxQMWD9.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
-  <link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
+  <link rel="modulepreload" crossorigin href="./assets/vendor-BF3F25aG.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BL7jEFBa.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-D2OEy8Ra.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -31,7 +31,6 @@ import (
 )
 
 var (
-	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
 	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
@@ -44,6 +43,9 @@ var (
 
 	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss")
 
+	// DataPath is a path to storage data.
+	DataPath = flag.String("storageDataPath", "victoria-metrics-data", "Path to storage data")
+
 	_ = flag.Duration("finalMergeDelay", 0, "Deprecated: this flag does nothing")
 	_ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing")
 	_ = flag.Int("smallMergeConcurrency", 0, "Deprecated: this flag does nothing")
@@ -54,8 +56,8 @@ var (
 
 	logNewSeries = flag.Bool("logNewSeries", false, "Whether to log new series. This option is for debug purposes only. It can lead to performance issues "+
 		"when big number of new series are ingested into VictoriaMetrics")
-	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod and -futureRetention. "+
-		"When set, then /api/v1/query_range will return an error for queries with 'from' value outside -retentionPeriod or 'to' value beyond -futureRetention. "+
+	denyQueriesOutsideRetention = flag.Bool("denyQueriesOutsideRetention", false, "Whether to deny queries outside the configured -retentionPeriod. "+
+		"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. "+
 		"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee")
 	maxHourlySeries = flag.Int64("storage.maxHourlySeries", 0, "The maximum number of unique series can be added to the storage during the last hour. "+
 		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter . "+
@@ -101,8 +103,19 @@ var (
 		"If set to 0 or a negative value, defaults to 1% of allowed memory.")
 )
 
-func DataPath() string {
-	return *storageDataPath
+// CheckTimeRange returns true if the given tr is denied for querying.
+func CheckTimeRange(tr storage.TimeRange) error {
+	if !*denyQueriesOutsideRetention {
+		return nil
+	}
+	minAllowedTimestamp := int64(fasttime.UnixTimestamp()*1000) - retentionPeriod.Milliseconds()
+	if tr.MinTimestamp > minAllowedTimestamp {
+		return nil
+	}
+	return &httpserver.ErrorWithStatusCode{
+		Err:        fmt.Errorf("the given time range %s is outside the allowed -retentionPeriod=%s according to -denyQueriesOutsideRetention", &tr, retentionPeriod),
+		StatusCode: http.StatusServiceUnavailable,
+	}
 }
 
 // Init initializes vmstorage.
@@ -134,21 +147,20 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	if *idbPrefillStart > 23*time.Hour {
 		logger.Panicf("-storage.idbPrefillStart cannot exceed 23 hours; got %s", idbPrefillStart)
 	}
-	logger.Infof("opening storage at %q with -retentionPeriod=%s", *storageDataPath, retentionPeriod)
+	logger.Infof("opening storage at %q with -retentionPeriod=%s", *DataPath, retentionPeriod)
 	startTime := time.Now()
 	WG = syncwg.WaitGroup{}
 	opts := storage.OpenOptions{
-		Retention:                   retentionPeriod.Duration(),
-		FutureRetention:             futureRetention.Duration(),
-		DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
-		MaxHourlySeries:             getMaxHourlySeries(),
-		MaxDailySeries:              getMaxDailySeries(),
-		DisablePerDayIndex:          *disablePerDayIndex,
-		TrackMetricNamesStats:       *trackMetricNamesStats,
-		IDBPrefillStart:             *idbPrefillStart,
-		LogNewSeries:                *logNewSeries,
+		Retention:             retentionPeriod.Duration(),
+		FutureRetention:       futureRetention.Duration(),
+		MaxHourlySeries:       getMaxHourlySeries(),
+		MaxDailySeries:        getMaxDailySeries(),
+		DisablePerDayIndex:    *disablePerDayIndex,
+		TrackMetricNamesStats: *trackMetricNamesStats,
+		IDBPrefillStart:       *idbPrefillStart,
+		LogNewSeries:          *logNewSeries,
 	}
-	strg := storage.MustOpenStorage(*storageDataPath, opts)
+	strg := storage.MustOpenStorage(*DataPath, opts)
 	Storage = strg
 	initStaleSnapshotsRemover(strg)
 
@@ -160,7 +172,7 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	rowsCount := tm.SmallRowsCount + tm.BigRowsCount
 	sizeBytes := tm.SmallSizeBytes + tm.BigSizeBytes
 	logger.Infof("successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
-		*storageDataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
+		*DataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
 
 	// register storage metrics
 	storageMetrics = metrics.NewSet()
@@ -168,7 +180,6 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 		writeStorageMetrics(w, strg)
 	})
 	metrics.RegisterSet(storageMetrics)
-	fs.RegisterPathFsMetrics(*storageDataPath)
 }
 
 var storageMetrics *metrics.Set
@@ -313,7 +324,7 @@ func Stop() {
 	metrics.UnregisterSet(storageMetrics, true)
 	storageMetrics = nil
 
-	logger.Infof("gracefully closing the storage at %s", *storageDataPath)
+	logger.Infof("gracefully closing the storage at %s", *DataPath)
 	startTime := time.Now()
 	WG.WaitAndBlock()
 	stopStaleSnapshotsRemover()
@@ -517,15 +528,15 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	tm := &m.TableMetrics
 	idbm := &m.TableMetrics.IndexDBMetrics
 
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *storageDataPath), uint64(minFreeDiskSpaceBytes.N))
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_total_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetTotalSpace(*storageDataPath))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *DataPath), fs.MustGetFreeSpace(*DataPath))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *DataPath), uint64(minFreeDiskSpaceBytes.N))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_total_disk_space_bytes{path=%q}`, *DataPath), fs.MustGetTotalSpace(*DataPath))
 
 	isReadOnly := 0
 	if strg.IsReadOnly() {
 		isReadOnly = 1
 	}
-	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_storage_is_read_only{path=%q}`, *storageDataPath), uint64(isReadOnly))
+	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_storage_is_read_only{path=%q}`, *DataPath), uint64(isReadOnly))
 
 	metrics.WriteGaugeUint64(w, `vm_active_merges{type="storage/inmemory"}`, tm.ActiveInmemoryMerges)
 	metrics.WriteGaugeUint64(w, `vm_active_merges{type="storage/small"}`, tm.ActiveSmallMerges)

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.26.3 AS build-web-stage
+FROM golang:1.26.2 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/package-lock.json

@@ -11,7 +11,7 @@
         "classnames": "^2.5.1",
         "dayjs": "^1.11.20",
         "lodash.debounce": "^4.0.8",
-        "marked": "^18.0.2",
+        "marked": "^18.0.0",
         "preact": "^10.29.1",
         "qs": "^6.15.1",
         "react-input-mask": "^2.0.4",
@@ -1394,6 +1394,9 @@
       "cpu": [
         "arm64"
       ],
+      "libc": [
+        "glibc"
+      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1410,6 +1413,9 @@
       "cpu": [
         "arm64"
       ],
+      "libc": [
+        "musl"
+      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1426,6 +1432,9 @@
       "cpu": [
         "ppc64"
       ],
+      "libc": [
+        "glibc"
+      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1442,6 +1451,9 @@
       "cpu": [
         "s390x"
       ],
+      "libc": [
+        "glibc"
+      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1458,6 +1470,9 @@
       "cpu": [
         "x64"
       ],
+      "libc": [
+        "glibc"
+      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -1474,6 +1489,9 @@
       "cpu": [
         "x64"
       ],
+      "libc": [
+        "musl"
+      ],
       "license": "MIT",
       "optional": true,
       "os": [
@@ -5135,9 +5153,9 @@
       }
     },
     "node_modules/marked": {
-      "version": "18.0.2",
-      "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.2.tgz",
-      "integrity": "sha512-NsmlUYBS/Zg57rgDWMYdnre6OTj4e+qq/JS2ot3KrYLSoHLw+sDu0Nm1ZGpRgYAq6c+b1ekaY5NzVchMCQnzcg==",
+      "version": "18.0.0",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.0.tgz",
+      "integrity": "sha512-2e7Qiv/HJSXj8rDEpgTvGKsP8yYtI9xXHKDnrftrmnrJPaFNM7VRb2YCzWaX4BP1iCJ/XPduzDJZMFoqTCcIMA==",
       "license": "MIT",
       "bin": {
         "marked": "bin/marked.js"
@@ -6197,6 +6215,7 @@
       "cpu": [
         "arm"
       ],
+      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6213,6 +6232,7 @@
       "cpu": [
         "arm64"
       ],
+      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6229,6 +6249,7 @@
       "cpu": [
         "arm"
       ],
+      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6245,6 +6266,7 @@
       "cpu": [
         "arm64"
       ],
+      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6261,6 +6283,7 @@
       "cpu": [
         "riscv64"
       ],
+      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6277,6 +6300,7 @@
       "cpu": [
         "x64"
       ],
+      "libc": "musl",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6293,6 +6317,7 @@
       "cpu": [
         "riscv64"
       ],
+      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [
@@ -6309,6 +6334,7 @@
       "cpu": [
         "x64"
       ],
+      "libc": "glibc",
       "license": "MIT",
       "optional": true,
       "os": [

app/vmui/packages/vmui/package.json

@@ -23,7 +23,7 @@
     "classnames": "^2.5.1",
     "dayjs": "^1.11.20",
     "lodash.debounce": "^4.0.8",
-    "marked": "^18.0.2",
+    "marked": "^18.0.0",
     "preact": "^10.29.1",
     "qs": "^6.15.1",
     "react-input-mask": "^2.0.4",

app/vmui/packages/vmui/src/api/types.ts

@@ -3,7 +3,6 @@ export interface MetricBase {
   metric: {
     [key: string]: string;
   };
-  nullTimestamps?: number[];
 }
 
 export interface MetricResult extends MetricBase {

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -16,7 +16,6 @@ export interface ChartTooltipProps {
   point: { top: number, left: number };
   unit?: string;
   statsFormatted?: SeriesItemStatsFormatted;
-  description?: ReactNode;
   isSticky?: boolean;
   info?: ReactNode;
   marker?: string;
@@ -35,7 +34,6 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
   unit = "",
   info,
   statsFormatted,
-  description,
   isSticky,
   marker,
   duplicateCount = 0,
@@ -175,7 +173,6 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
           ))}
         </table>
       )}
-      {description && <p className="vm-chart-tooltip__description">{description}</p>}
       {info && <p className="vm-chart-tooltip__info">{info}</p>}
     </div>
   ), u.root);

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -143,10 +143,4 @@ $chart-tooltip-y: -1 * ($padding-global + $chart-tooltip-half-icon);
     word-break: break-all;
     white-space: pre-wrap;
   }
-
-  &__description {
-    word-break: break-word;
-    white-space: pre-wrap;
-    opacity: 0.85;
-  }
 }

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -4,7 +4,7 @@ import { ChartTooltipProps } from "../../components/Chart/ChartTooltip/ChartTool
 import { SeriesItem } from "../../types";
 import dayjs from "dayjs";
 import { DATE_FULL_TIMEZONE_FORMAT } from "../../constants/date";
-import { getMetricName } from "../../utils/uplot";
+import { formatPrettyNumber, getMetricName } from "../../utils/uplot";
 import { MetricResult } from "../../api/types";
 import useEventListener from "../useEventListener";
 
@@ -15,65 +15,19 @@ interface LineTooltipHook {
   unit?: string;
 }
 
-// Pixel proximity for detecting hover over null-timestamp X markers drawn at chart bottom.
-const NULL_HOVER_PROX = 8;
-// Half the visual marker height in CSS px (BASE_POINT_SIZE * 1.4 / 2 from scatter.ts).
-// scatter.ts lifts the marker center by this amount above yMin so the icon sits inside
-// the plot area; the hover y-anchor must match that offset.
-const NULL_MARKER_HALF_CSS = 2.8;
-
-interface NullHover {
-  seriesIdx: number;
-  timestamp: number;
-}
-
-const findNullHover = (u: uPlot): NullHover | null => {
-  const cursorLeft = u.cursor.left ?? -1;
-  const cursorTop = u.cursor.top ?? -1;
-  if (cursorLeft < 0 || cursorTop < 0) return null;
-
-  const scaleY = u.scales["1"];
-  if (!scaleY || scaleY.min == null) return null;
-  const yPos = u.valToPos(scaleY.min, "1") - NULL_MARKER_HALF_CSS;
-  if (Math.abs(cursorTop - yPos) > NULL_HOVER_PROX) return null;
-
-  let best: { seriesIdx: number; timestamp: number; dist: number } | null = null;
-  for (let s = 1; s < u.series.length; s++) {
-    const seriesItem = u.series[s] as SeriesItem;
-    if (!seriesItem.show) continue;
-    const nullTs = seriesItem.nullTimestamps;
-    if (!nullTs || !nullTs.length) continue;
-
-    for (let i = 0; i < nullTs.length; i++) {
-      const t = nullTs[i];
-      const xPos = u.valToPos(t, "x");
-      const dist = Math.abs(cursorLeft - xPos);
-      if (dist < NULL_HOVER_PROX && (best === null || dist < best.dist)) {
-        best = { seriesIdx: s, timestamp: t, dist };
-      }
-    }
-  }
-  return best ? { seriesIdx: best.seriesIdx, timestamp: best.timestamp } : null;
-};
-
-const NULL_DESCRIPTION = "\"null\" can be a staleness marker or an actual NaN/null value produced by exporter.";
-
 const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   const [showTooltip, setShowTooltip] = useState(false);
   const [tooltipIdx, setTooltipIdx] = useState({ seriesIdx: -1, dataIdx: -1 });
-  const [nullTooltip, setNullTooltip] = useState<NullHover | null>(null);
   const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipProps[]>([]);
 
   const resetTooltips = () => {
     setStickyToolTips([]);
     setTooltipIdx({ seriesIdx: -1, dataIdx: -1 });
-    setNullTooltip(null);
   };
 
   const setCursor = (u: uPlot) => {
     const dataIdx = u.cursor.idx ?? -1;
     setTooltipIdx(prev => ({ ...prev, dataIdx }));
-    setNullTooltip(findNullHover(u));
   };
 
   const seriesFocus = (u: uPlot, sidx: (number | null)) => {
@@ -81,36 +35,7 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     setTooltipIdx(prev => ({ ...prev, seriesIdx }));
   };
 
-  const getNullTooltipProps = (hit: NullHover): ChartTooltipProps => {
-    const { seriesIdx, timestamp } = hit;
-    const metricItem = metrics[seriesIdx - 1];
-    const seriesItem = series[seriesIdx] as SeriesItem;
-
-    const groups = new Set(metrics.map(m => m.group));
-    const group = metricItem?.group || 0;
-
-    const yMin = u?.scales?.[1]?.min ?? 0;
-    const point = {
-      top: u ? u.valToPos(yMin, seriesItem?.scale || "1") - NULL_MARKER_HALF_CSS : 0,
-      left: u ? u.valToPos(timestamp, "x") : 0,
-    };
-
-    return {
-      u,
-      id: `null_${seriesIdx}_${timestamp}`,
-      title: groups.size > 1 ? `Query ${group}` : "",
-      dates: [dayjs(timestamp * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT)],
-      value: "null",
-      info: getMetricName(metricItem, seriesItem),
-      description: NULL_DESCRIPTION,
-      marker: `${seriesItem?.stroke}`,
-      point,
-    };
-  };
-
   const getTooltipProps = useCallback((): ChartTooltipProps => {
-    if (nullTooltip) return getNullTooltipProps(nullTooltip);
-
     const { seriesIdx, dataIdx } = tooltipIdx;
     const metricItem = metrics[seriesIdx - 1];
     const seriesItem = series[seriesIdx] as SeriesItem;
@@ -119,6 +44,8 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
     const group = metricItem?.group || 0;
 
     const value = u?.data?.[seriesIdx]?.[dataIdx] || 0;
+    const min = u?.scales?.[1]?.min || 0;
+    const max = u?.scales?.[1]?.max || 1;
     const date = u?.data?.[0]?.[dataIdx] || 0;
 
     let duplicateCount = 1;
@@ -153,13 +80,13 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
       id: `${seriesIdx}_${dataIdx}`,
       title: groups.size > 1 ? `Query ${group}` : "",
       dates: [date ? dayjs(date * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT) : "-"],
-      value: value.toLocaleString("en-US", { maximumFractionDigits: 20 }),
+      value: formatPrettyNumber(value, min, max),
       info: getMetricName(metricItem, seriesItem),
       statsFormatted: seriesItem?.statsFormatted,
       marker: `${seriesItem?.stroke}`,
       duplicateCount,
     };
-  }, [u, tooltipIdx, metrics, series, unit, nullTooltip]);
+  }, [u, tooltipIdx, metrics, series, unit]);
 
   const handleClick = useCallback(() => {
     if (!showTooltip) return;
@@ -174,9 +101,8 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   };
 
   useEffect(() => {
-    const normalHit = tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1;
-    setShowTooltip(normalHit || nullTooltip !== null);
-  }, [tooltipIdx, nullTooltip]);
+    setShowTooltip(tooltipIdx.dataIdx !== -1 && tooltipIdx.seriesIdx !== -1);
+  }, [tooltipIdx]);
 
   useEventListener("click", handleClick);
 

app/vmui/packages/vmui/src/pages/CustomPanel/utils.test.ts

@@ -20,7 +20,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123");
+    expect(result).toBe("header1,header2\n123,value2");
   });
 
   it("should return a valid CSV string for multiple metric entries with values", () => {
@@ -43,7 +43,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,value2,1623945600,123\n456,value4,1623949200,456");
+    expect(result).toBe("header1,header2\n123,value2\n456,value4");
   });
 
   it("should handle metric entries with multiple values field", () => {
@@ -58,7 +58,7 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2,__timestamp__,__value__\n123-456,values,-,-");
+    expect(result).toBe("header1,header2\n123-456,values");
   });
 
   it("should handle a combination of metric entries with value and values", () => {
@@ -81,19 +81,6 @@ describe("convertMetricsDataToCSV", () => {
       },
     ];
     const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("header1,header2,__timestamp__,__value__\n123,first,1623945600,123\n456-789,second,-,-");
+    expect(result).toBe("header1,header2\n123,first\n456-789,second");
   });
-
-  it("should return value and timestamp if metric field is empty", () => {
-    const data: InstantMetricResult[] = [
-      {
-        value: [1623945600, "123"],
-        group: 0,
-        metric: {}
-      },
-    ];
-    const result = convertMetricsDataToCSV(data);
-    expect(result).toBe("__timestamp__,__value__\n1623945600,123");
-  });
-
 });

app/vmui/packages/vmui/src/pages/CustomPanel/utils.ts

@@ -3,22 +3,16 @@ import { getColumns, MetricCategory } from "../../hooks/useSortedCategories";
 import { formatValueToCSV } from "../../utils/csv";
 
 const getHeaders = (data: InstantMetricResult[]): string => {
-  const metricHeaders = getColumns(data).map(({ key }) => key);
-  return [...metricHeaders, "__timestamp__", "__value__"].join(",");
+  return getColumns(data).map(({ key }) => key).join(",");
 };
 
 const getRows = (data: InstantMetricResult[], headers: MetricCategory[]) => {
-  return data?.map(d => {
-    const metricPart = headers.map(c => formatValueToCSV(d.metric[c.key] || "-"));
-    const timestamp = d.value ? formatValueToCSV(String(d.value[0])) : "-";
-    const value = d.value ? formatValueToCSV(d.value[1]) : "-";
-    return [...metricPart, timestamp, value].join(",");
-  });
+  return data?.map(d => headers.map(c => formatValueToCSV(d.metric[c.key] || "-")).join(","));
 };
 
 export const convertMetricsDataToCSV = (data: InstantMetricResult[]): string => {
-  if (!data.length) return "";
   const headers = getHeaders(data);
+  if (!headers.length) return "";
   const rows = getRows(data, getColumns(data));
   return [headers, ...rows].join("\n");
 };

app/vmui/packages/vmui/src/pages/RawQueryPage/hooks/useFetchExport.ts

@@ -149,21 +149,15 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
             const pointsToTake = shouldDownsample ? maxPointsPerSeries : totalPoints;
             const step = shouldDownsample ? totalPoints / maxPointsPerSeries : 1;
 
-            const values: [number, number][] = new Array(pointsToTake);
-            const nullTimestamps: number[] = [];
-            for (let i = 0; i < pointsToTake; i++) {
+            const values: [number, number][] = Array.from({ length: pointsToTake }, (_, i) => {
               const idx = shouldDownsample ? Math.floor(i * step) : i;
-              const ts = rawTimestamps[idx] / 1000;
-              const raw = rawValues[idx];
-              if (raw === null) nullTimestamps.push(ts);
-              values[i] = [ts, raw as number];
-            }
+              return [rawTimestamps[idx] / 1000, rawValues[idx]];
+            });
 
             tempData.push({
               group: counter,
               metric: jsonLine.metric,
               values,
-              nullTimestamps,
             } as MetricBase);
           }
 

app/vmui/packages/vmui/src/pages/TopQueries/TopQueryPanel/TopQueryPanel.tsx

@@ -1,4 +1,4 @@
-import { FC, ReactNode, useState } from "react";
+import { FC, useState } from "react";
 import { TopQuery } from "../../../types";
 import JsonView from "../../../components/Views/JsonView/JsonView";
 import { CodeIcon, TableIcon } from "../../../components/Main/Icons";
@@ -8,18 +8,10 @@ import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 
-export interface TopQueryColumn {
-  title?: string;
-  tooltip?: string;
-  key: keyof TopQuery;
-  sortBy?: keyof TopQuery;
-  format?: (row: TopQuery) => ReactNode;
-}
-
 export interface TopQueryPanelProps {
   rows: TopQuery[],
   title?: string,
-  columns: TopQueryColumn[],
+  columns: {title?: string, key: (keyof TopQuery), sortBy?: (keyof TopQuery)}[],
   defaultOrderBy?: keyof TopQuery,
 }
 const tabs = ["table", "JSON"].map((t, i) => ({

app/vmui/packages/vmui/src/pages/TopQueries/TopQueryTable/TopQueryTable.tsx

@@ -3,7 +3,7 @@ import { TopQuery } from "../../../types";
 import { getComparator, stableSort } from "../../../components/Table/helpers";
 import { TopQueryPanelProps } from "../TopQueryPanel/TopQueryPanel";
 import classNames from "classnames";
-import { ArrowDropDownIcon, CopyIcon, InfoOutlinedIcon, PlayCircleOutlineIcon } from "../../../components/Main/Icons";
+import { ArrowDropDownIcon, CopyIcon, PlayCircleOutlineIcon } from "../../../components/Main/Icons";
 import Button from "../../../components/Main/Button/Button";
 import Tooltip from "../../../components/Main/Tooltip/Tooltip";
 import { Link } from "react-router-dom";
@@ -35,40 +35,26 @@ const TopQueryTable:FC<TopQueryPanelProps> = ({ rows, columns, defaultOrderBy })
     <table className="vm-table">
       <thead className="vm-table-header">
         <tr className="vm-table__row vm-table__row_header">
-          {columns.map((col) => {
-            const sortKey = col.sortBy || col.key;
-
-            return (
-              <th
-                className="vm-table-cell vm-table-cell_header vm-table-cell_sort"
-                onClick={createSortHandler(sortKey)}
-                key={col.key}
-              >
-                <div className="vm-table-cell__content">
-                  {col.title || col.key}
-                  {col.tooltip && (
-                    <Tooltip
-                      placement="top-center"
-                      title={col.tooltip}
-                    >
-                      <span className="vm-top-queries-table__info-icon">
-                        <InfoOutlinedIcon/>
-                      </span>
-                    </Tooltip>
-                  )}
-                  <div
-                    className={classNames({
-                      "vm-table__sort-icon": true,
-                      "vm-table__sort-icon_active": orderBy === sortKey,
-                      "vm-table__sort-icon_desc": orderDir === "desc" && orderBy === sortKey
-                    })}
-                  >
-                    <ArrowDropDownIcon/>
-                  </div>
+          {columns.map((col) => (
+            <th
+              className="vm-table-cell vm-table-cell_header vm-table-cell_sort"
+              onClick={createSortHandler(col.sortBy || col.key)}
+              key={col.key}
+            >
+              <div className="vm-table-cell__content">
+                {col.title || col.key}
+                <div
+                  className={classNames({
+                    "vm-table__sort-icon": true,
+                    "vm-table__sort-icon_active": orderBy === col.key,
+                    "vm-table__sort-icon_desc": orderDir === "desc" && orderBy === col.key
+                  })}
+                >
+                  <ArrowDropDownIcon/>
                 </div>
-              </th>
-            );
-          })}
+              </div>
+            </th>
+          ))}
           <th className="vm-table-cell vm-table-cell_header"/> {/* empty cell for actions */}
         </tr>
       </thead>
@@ -83,7 +69,7 @@ const TopQueryTable:FC<TopQueryPanelProps> = ({ rows, columns, defaultOrderBy })
                 className="vm-table-cell"
                 key={col.key}
               >
-                {col.format?.(row) ?? row[col.key] ?? "-"}
+                {row[col.key] || "-"}
               </td>
             ))}
             <td className="vm-table-cell vm-table-cell_no-padding">

app/vmui/packages/vmui/src/pages/TopQueries/hooks/useFetchTopQueries.ts

@@ -34,7 +34,7 @@ const processResponse = (data: TopQueriesData) => {
     target.forEach(t => {
       const timeRange = getDurationFromMilliseconds(t.timeRangeSeconds*1000);
       t.url = getQueryUrl(t, timeRange);
-      t.timeRange = timeRange || "instant";
+      t.timeRange = timeRange;
     });
   });
 

app/vmui/packages/vmui/src/pages/TopQueries/hooks/useTopQueriesColumns.ts

@@ -1,77 +0,0 @@
-import { useMemo } from "react";
-import { TopQueryColumn } from "../TopQueryPanel/TopQueryPanel";
-import { humanizeSeconds } from "../../../utils/time";
-import { formatBytes } from "../../../utils/bytes";
-
-type UseTopQueriesColumns = {
-  maxLifetime: string;
-};
-
-export const useTopQueriesColumns = ({ maxLifetime }: UseTopQueriesColumns) => {
-  return useMemo(() => {
-    const queryCol: TopQueryColumn = {
-      key: "query"
-    };
-
-    const timeRangeCol: TopQueryColumn = {
-      key: "timeRange",
-      sortBy: "timeRangeSeconds",
-      title: "range",
-      tooltip: "The time range between start and end of the query request. 'instant' means the query was executed at a single point in time without a time range"
-    };
-
-    const countCol: TopQueryColumn = {
-      key: "count",
-      tooltip: `The number of times the query was executed over the last ${maxLifetime}`,
-    };
-
-    const topBySumDuration: TopQueryColumn[] = [
-      queryCol,
-      {
-        key: "sumDurationSeconds",
-        title: "duration",
-        tooltip: `Cumulative time spent executing the query across all its invocations over the last ${maxLifetime}`,
-        format: (row) => humanizeSeconds(row.sumDurationSeconds)
-      },
-      timeRangeCol,
-      countCol,
-    ];
-
-    const topByAvgDuration: TopQueryColumn[] = [
-      queryCol,
-      {
-        key: "avgDurationSeconds",
-        title: "duration",
-        tooltip: `Average time spent executing the query over the last ${maxLifetime}`,
-        format: (row) => humanizeSeconds(row.avgDurationSeconds)
-      },
-      timeRangeCol,
-      countCol,
-    ];
-
-    const topByCount: TopQueryColumn[] = [
-      queryCol,
-      timeRangeCol,
-      countCol,
-    ];
-
-    const topByAvgMemoryUsage: TopQueryColumn[] = [
-      queryCol,
-      {
-        key: "avgMemoryBytes",
-        title: "memory",
-        tooltip: `Average memory used during query execution over the last ${maxLifetime}`,
-        format: (row) => formatBytes(row.avgMemoryBytes)
-      },
-      timeRangeCol,
-      countCol,
-    ];
-
-    return {
-      topBySumDuration,
-      topByAvgDuration,
-      topByCount,
-      topByAvgMemoryUsage,
-    };
-  }, [maxLifetime]);
-};

app/vmui/packages/vmui/src/pages/TopQueries/index.tsx

@@ -15,7 +15,6 @@ import "./style.scss";
 import useDeviceDetect from "../../hooks/useDeviceDetect";
 import classNames from "classnames";
 import useStateSearchParams from "../../hooks/useStateSearchParams";
-import { useTopQueriesColumns } from "./hooks/useTopQueriesColumns";
 
 const exampleDuration = "30ms, 15s, 3d4h, 1y2w";
 
@@ -24,7 +23,6 @@ const TopQueries: FC = () => {
 
   const [topN, setTopN] = useStateSearchParams(10, "topN");
   const [maxLifetime, setMaxLifetime] = useStateSearchParams("10m", "maxLifetime");
-  const columns = useTopQueriesColumns({ maxLifetime });
 
   const { data, error, loading, fetch } = useFetchTopQueries({ topN, maxLifetime });
 
@@ -147,33 +145,52 @@ const TopQueries: FC = () => {
 
       {error && <Alert variant="error">{error}</Alert>}
 
-      {data && (
+      {data && (<>
         <div className="vm-top-queries-panels">
           <TopQueryPanel
-            title="Queries with most summary time to execute"
             rows={data.topBySumDuration}
-            columns={columns.topBySumDuration}
-            defaultOrderBy="sumDurationSeconds"
+            title={"Queries with most summary time to execute"}
+            columns={[
+              { key: "query" },
+              { key: "sumDurationSeconds", title: "sum duration, sec" },
+              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
+              { key: "count" }
+            ]}
+            defaultOrderBy={"sumDurationSeconds"}
           />
           <TopQueryPanel
-            title="Most heavy queries"
             rows={data.topByAvgDuration}
-            columns={columns.topByAvgDuration}
-            defaultOrderBy="avgDurationSeconds"
+            title={"Most heavy queries"}
+            columns={[
+              { key: "query" },
+              { key: "avgDurationSeconds", title: "avg duration, sec" },
+              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
+              { key: "count" }
+            ]}
+            defaultOrderBy={"avgDurationSeconds"}
           />
           <TopQueryPanel
-            title="Most frequently executed queries"
             rows={data.topByCount}
-            columns={columns.topByCount}
+            title={"Most frequently executed queries"}
+            columns={[
+              { key: "query" },
+              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
+              { key: "count" }
+            ]}
           />
           <TopQueryPanel
-            title="Queries with most memory to execute"
             rows={data.topByAvgMemoryUsage}
-            columns={columns.topByAvgMemoryUsage}
-            defaultOrderBy="avgMemoryBytes"
+            title={"Queries with most memory to execute"}
+            columns={[
+              { key: "query" },
+              { key: "avgMemoryBytes", title: "avg memory usage, bytes" },
+              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
+              { key: "count" }
+            ]}
+            defaultOrderBy={"avgMemoryBytes"}
           />
         </div>
-      )}
+      </>)}
     </div>
   );
 };

app/vmui/packages/vmui/src/pages/TopQueries/style.scss

@@ -1,19 +1,5 @@
 @use "src/styles/variables" as *;
 
-.vm-top-queries-table {
-  &__info-icon {
-    display: inline-flex;
-    align-items: center;
-    color: $color-text-secondary;
-    margin-left: 4px;
-
-    svg {
-      width: 14px;
-      height: 14px;
-    }
-  }
-}
-
 .vm-top-queries {
   display: grid;
   align-items: flex-start;

app/vmui/packages/vmui/src/types/uplot.ts

@@ -11,7 +11,6 @@ export interface SeriesItem extends Series {
     statsFormatted: SeriesItemStatsFormatted;
     median: number;
     hasAlias?: boolean;
-    nullTimestamps?: number[];
 }
 
 export interface HideSeriesArgs {

app/vmui/packages/vmui/src/utils/bytes.test.ts

@@ -1,47 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { formatBytes } from "./bytes";
-
-describe("formatBytes", () => {
-  it("returns null for invalid values", () => {
-    expect(formatBytes(-1)).toBeNull();
-    expect(formatBytes(Number.NaN)).toBeNull();
-    expect(formatBytes(Number.POSITIVE_INFINITY)).toBeNull();
-    expect(formatBytes(Number.NEGATIVE_INFINITY)).toBeNull();
-  });
-
-  it("formats zero bytes", () => {
-    expect(formatBytes(0)).toBe("0 B");
-  });
-
-  it("formats bytes", () => {
-    expect(formatBytes(0.5)).toBe("0.5 B");
-    expect(formatBytes(1)).toBe("1 B");
-    expect(formatBytes(512)).toBe("512 B");
-    expect(formatBytes(1023)).toBe("1023 B");
-  });
-
-  it("formats kilobytes", () => {
-    expect(formatBytes(1024)).toBe("1 KB");
-    expect(formatBytes(1536)).toBe("1.5 KB");
-  });
-
-  it("formats megabytes", () => {
-    expect(formatBytes(1024 ** 2)).toBe("1 MB");
-    expect(formatBytes(2.5 * 1024 ** 2)).toBe("2.5 MB");
-  });
-
-  it("formats gigabytes, terabytes and petabytes", () => {
-    expect(formatBytes(1024 ** 3)).toBe("1 GB");
-    expect(formatBytes(1024 ** 4)).toBe("1 TB");
-    expect(formatBytes(1024 ** 5)).toBe("1 PB");
-  });
-
-  it("caps values above PB to PB unit", () => {
-    expect(formatBytes(1024 ** 6)).toBe("1024 PB");
-  });
-
-  it("rounds to two decimals", () => {
-    expect(formatBytes(1234)).toBe("1.21 KB");
-    expect(formatBytes(1234567)).toBe("1.18 MB");
-  });
-});

app/vmui/packages/vmui/src/utils/bytes.ts

@@ -1,14 +0,0 @@
-const LOG_1024 = Math.log(1024);
-const UNITS = ["B", "KB", "MB", "GB", "TB", "PB"] as const;
-
-export const formatBytes = (bytes: number): string | null => {
-  if (!Number.isFinite(bytes) || bytes < 0) return null;
-  if (bytes === 0) return "0 B";
-
-  const unitIndex = Math.min(
-    Math.max(Math.floor(Math.log(bytes) / LOG_1024), 0),
-    UNITS.length - 1
-  );
-
-  return `${parseFloat((bytes / 1024 ** unitIndex).toFixed(2))} ${UNITS[unitIndex]}`;
-};

app/vmui/packages/vmui/src/utils/uplot/scatter.ts

@@ -103,28 +103,6 @@ export const drawPoints = (u: uPlot, seriesIdx: number) => {
     u.ctx.lineWidth = 1.4 * uPlot.pxRatio;
     u.ctx.strokeStyle = u.ctx.fillStyle;
     u.ctx.stroke(squaresPath);
-
-    const nullTs = (series as unknown as { nullTimestamps?: number[] }).nullTimestamps;
-    if (nullTs && nullTs.length) {
-      const xSize = BASE_POINT_SIZE * 1.4 * uPlot.pxRatio;
-      const xHalf = xSize / 2;
-      // Lift the marker by half its size so the entire icon sits inside the plot area
-      // (yMin maps to the plot's bottom edge, so centering on it would clip the lower half).
-      const cy = valToPosY(yMin, scaleY, yDim, yOff) - xHalf;
-      const xPath = new Path2D();
-      for (let i = 0; i < nullTs.length; i++) {
-        const t = nullTs[i];
-        if (t < xMin || t > xMax) continue;
-        const cx = valToPosX(t, scaleX, xDim, xOff);
-        xPath.moveTo(cx - xHalf, cy - xHalf);
-        xPath.lineTo(cx + xHalf, cy + xHalf);
-        xPath.moveTo(cx + xHalf, cy - xHalf);
-        xPath.lineTo(cx - xHalf, cy + xHalf);
-      }
-      u.ctx.lineWidth = 1.6 * uPlot.pxRatio;
-      u.ctx.strokeStyle = u.ctx.fillStyle;
-      u.ctx.stroke(xPath);
-    }
   };
 
   uPlot.orient(u, seriesIdx, orientCallback);

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -38,7 +38,6 @@ export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[],
       show: !includesHideSeries(label, hideSeries),
       scale: "1",
       paths: isRawQuery ? drawPoints : undefined,
-      nullTimestamps: d.nullTimestamps,
       ...getSeriesStatistics(d),
     };
   };

apptest/app.go

@@ -22,7 +22,6 @@ var (
 	vminsertAddrRE              = regexp.MustCompile(`accepting vminsert conns at (.*:\d{1,5})$`)
 	vminsertClusterNativeAddrRE = regexp.MustCompile(`started TCP clusternative server at "(.*:\d{1,5})"`)
 	vmselectAddrRE              = regexp.MustCompile(`accepting vmselect conns at (.*:\d{1,5})$`)
-	vmauthHttpListenAddrRE      = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
 )
 
 // app represents an instance of some VictoriaMetrics server (such as vmstorage,

apptest/client.go

@@ -2,8 +2,6 @@ package apptest
 
 import (
 	"bytes"
-	"encoding/json"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -14,10 +12,6 @@ import (
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
-	"github.com/golang/snappy"
 )
 
 // Client is used for interacting with the apps over the network.
@@ -111,35 +105,6 @@ func (c *Client) Write(t *testing.T, address string, data []string) {
 	}
 }
 
-// getClusterPath returns path in cluster's URL format.
-// Based on QueryOpts, it will either put tenant ID into URL
-// or will skip it if tenant is set via HTTP headers.
-func getClusterPath(addr, prefix, suffix string, o QueryOpts) string {
-	if o.Tenant != "" {
-		// QueryOpts.Tenant has priority over headers
-		return tenantViaURL(addr, prefix, o.Tenant, suffix)
-	}
-
-	h := o.getHeaders()
-	if h.Get("AccountID") != "" || h.Get("ProjectID") != "" {
-		return tenantViaHeaders(addr, prefix, suffix)
-	}
-
-	// tenant is missing in QueryOpts and in HTTP headers. Falling back to default 0:0 tenant in URL
-	return tenantViaURL(addr, prefix, "0:0", suffix)
-}
-
-// tenantViaURL returns path in cluster's URL format with tenant specified in URL
-func tenantViaURL(addr, prefix, tenant, suffix string) string {
-	return fmt.Sprintf("http://%s/%s/%s/%s", addr, prefix, tenant, suffix)
-}
-
-// tenantViaHeaders returns path in cluster's URL format where tenant is omitted in URL
-// Only supported if -enableMultitenancyViaHeaders is specified
-func tenantViaHeaders(addr, prefix, suffix string) string {
-	return fmt.Sprintf("http://%s/%s/%s", addr, prefix, suffix)
-}
-
 // readAllAndClose reads everything from the response body and then closes it.
 func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
 	t.Helper()
@@ -152,34 +117,27 @@ func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
 	return string(b)
 }
 
-// metricsClient is used to retrieve the app's metrics.
+// ServesMetrics is used to retrieve the app's metrics.
 //
 // This type is expected to be embedded by the apps that serve metrics.
-type metricsClient struct {
-	metricsCli *Client
-	url        string
-}
-
-func newMetricsClient(cli *Client, addr string) *metricsClient {
-	return &metricsClient{
-		metricsCli: cli,
-		url:        fmt.Sprintf("http://%s/metrics", addr),
-	}
+type ServesMetrics struct {
+	metricsURL string
+	cli        *Client
 }
 
 // GetIntMetric retrieves the value of a metric served by an app at /metrics URL.
 // The value is then converted to int.
-func (c *metricsClient) GetIntMetric(t *testing.T, metricName string) int {
+func (app *ServesMetrics) GetIntMetric(t *testing.T, metricName string) int {
 	t.Helper()
 
-	return int(c.GetMetric(t, metricName))
+	return int(app.GetMetric(t, metricName))
 }
 
 // GetMetric retrieves the value of a metric served by an app at /metrics URL.
-func (c *metricsClient) GetMetric(t *testing.T, metricName string) float64 {
+func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
 	t.Helper()
 
-	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
+	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -200,12 +158,12 @@ func (c *metricsClient) GetMetric(t *testing.T, metricName string) float64 {
 
 // GetMetricsByPrefix retrieves the values of all metrics that start with given
 // prefix.
-func (c *metricsClient) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
+func (app *ServesMetrics) GetMetricsByPrefix(t *testing.T, prefix string) []float64 {
 	t.Helper()
 
 	values := []float64{}
 
-	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
+	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -229,12 +187,12 @@ func (c *metricsClient) GetMetricsByPrefix(t *testing.T, prefix string) []float6
 	return values
 }
 
-func (c *metricsClient) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []float64 {
+func (app *ServesMetrics) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []float64 {
 	t.Helper()
 
 	values := []float64{}
 
-	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
+	metrics, statusCode := app.cli.Get(t, app.metricsURL, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -257,756 +215,3 @@ func (c *metricsClient) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []fl
 	}
 	return values
 }
-
-// rpcRowsSentTotal retrieves the values of all vminsert
-// `vm_rpc_rows_sent_total` metrics (there will be one for each vmstorage) and
-// returns their integer sum.
-func (c *metricsClient) rpcRowsSentTotal(t *testing.T) int {
-	total := 0.0
-	for _, v := range c.GetMetricsByPrefix(t, "vm_rpc_rows_sent_total") {
-		total += v
-	}
-	return int(total)
-}
-
-type vmselectClient struct {
-	vmselectCli              *Client
-	url                      func(op, path string, opts QueryOpts) string
-	metricNamesStatsResetURL string
-	tenantsURL               string
-}
-
-// PrometheusAPIV1Export is a test helper function that performs the export of
-// raw samples in JSON line format by sending a request to
-// /prometheus/api/v1/export endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
-func (c *vmselectClient) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/export", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1ExportNative is a test helper function that performs the export of
-// raw samples in native binary format by sending a request to
-// /prometheus/api/v1/export/native endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
-func (c *vmselectClient) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/export/native", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", query)
-	values.Add("format", "promapi")
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return []byte(res)
-}
-
-// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
-// instant query by sending a request to /prometheus/api/v1/query endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
-func (c *vmselectClient) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/query", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1QueryRange is a test helper function that performs
-// PromQL/MetricsQL range query by sending a request to
-// /prometheus/api/v1/query_range endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
-func (c *vmselectClient) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/query_range", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1QueryResponse(t, res)
-}
-
-// PrometheusAPIV1Series retrieves list of time series that match the query by
-// sending a request to /prometheus/api/v1/series endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (c *vmselectClient) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/series", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesResponse(t, res)
-}
-
-// PrometheusAPIV1SeriesCount retrieves the total number of time series by
-// sending a request to /prometheus/api/v1/series/count endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
-func (c *vmselectClient) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/series/count", opts)
-	values := opts.asURLValues()
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1SeriesCountResponse(t, res)
-}
-
-// PrometheusAPIV1Labels retrieves the label names for time series that match a
-// query by sending a request to /prometheus/api/v1/labels endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
-func (c *vmselectClient) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/labels", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1LabelsResponse(t, res)
-}
-
-// PrometheusAPIV1LabelValues retrieves the labels values for the metrics that
-// match the query by sending a request to /prometheus/api/v1/label/.../values
-// endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
-func (c *vmselectClient) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
-	t.Helper()
-	path := fmt.Sprintf("prometheus/api/v1/label/%s/values", labelName)
-	url := c.url("select", path, opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1LabelValuesResponse(t, res)
-}
-
-// PrometheusAPIV1Metadata retrieves metadata for the given metric by sending a
-// request to /prometheus/api/v1/metadata endpoint.
-func (c *vmselectClient) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/metadata", opts)
-	values := opts.asURLValues()
-	values.Add("metric", metric)
-	values.Add("limit", strconv.Itoa(limit))
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	return NewPrometheusAPIV1Metadata(t, res)
-}
-
-// PrometheusAPIV1AdminTSDBDeleteSeries deletes the series that match the query
-// by sending a request to /prometheus/api/v1/admin/tsdb/delete_series.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
-func (c *vmselectClient) PrometheusAPIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("delete", "prometheus/api/v1/admin/tsdb/delete_series", opts)
-	values := opts.asURLValues()
-	values.Add("match[]", matchQuery)
-	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// PrometheusAPIV1StatusMetricNamesStats sends a query to
-// /prometheus/api/v1/status/metric_names_stats endpoint and returns the metric
-// usage stats response for given params.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (c *vmselectClient) PrometheusAPIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/status/metric_names_stats", opts)
-	values := opts.asURLValues()
-	values.Add("limit", limit)
-	values.Add("le", le)
-	values.Add("match_pattern", matchPattern)
-	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-	var resp MetricNamesStatsResponse
-	if err := json.Unmarshal([]byte(res), &resp); err != nil {
-		t.Fatalf("could not unmarshal metric names stats response data:\n%s\n err: %v", res, err)
-	}
-	return resp
-}
-
-// PrometheusAPIV1StatusTSDB retrieves the TSDB status for the time series that
-// match the query on the given date by sending a request to
-// /prometheus/api/v1/status/tsdb endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
-func (c *vmselectClient) PrometheusAPIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
-	t.Helper()
-	url := c.url("select", "prometheus/api/v1/status/tsdb", opts)
-	values := opts.asURLValues()
-	addNonEmpty := func(name, value string) {
-		if len(value) == 0 {
-			return
-		}
-		values.Add(name, value)
-	}
-	addNonEmpty("match[]", matchQuery)
-	addNonEmpty("topN", topN)
-	addNonEmpty("date", date)
-	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var status TSDBStatusResponse
-	if err := json.Unmarshal([]byte(res), &status); err != nil {
-		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
-	}
-	status.Sort()
-	return status
-}
-
-// GraphiteMetricsIndex retrieves the list of all metrics by sending a request
-// to /graphite/metrics/index.json endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
-func (c *vmselectClient) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse {
-	t.Helper()
-
-	url := c.url("select", "graphite/metrics/index.json", opts)
-	res, statusCode := c.vmselectCli.Get(t, url, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	var index GraphiteMetricsIndexResponse
-	if err := json.Unmarshal([]byte(res), &index); err != nil {
-		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
-	}
-	return index
-}
-
-// GraphiteMetricsFind finds metrics under a given path by sending a request
-// to /metrics/find endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
-// and https://graphite.readthedocs.io/en/latest/metrics_api.html#metrics-find
-func (c *vmselectClient) GraphiteMetricsFind(t *testing.T, query string, opts QueryOpts) GraphiteMetricsFindResponse {
-	t.Helper()
-
-	url := c.url("select", "graphite/metrics/find", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
-	}
-
-	var res GraphiteMetricsFindResponse
-	if err := json.Unmarshal([]byte(resText), &res); err != nil {
-		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
-	}
-	return res
-}
-
-// GraphiteMetricsExpand expands the given query with matching paths by sending
-// a request to /graphite/metrics/expand endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
-// and https://graphite.readthedocs.io/en/latest/metrics_api.html#metrics-expand
-func (c *vmselectClient) GraphiteMetricsExpand(t *testing.T, query string, opts QueryOpts) GraphiteMetricsExpandResponse {
-	t.Helper()
-
-	url := c.url("select", "graphite/metrics/expand", opts)
-	values := opts.asURLValues()
-	values.Add("query", query)
-	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
-	}
-
-	var res GraphiteMetricsExpandResponse
-	if err := json.Unmarshal([]byte(resText), &res); err != nil {
-		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
-	}
-	return res
-}
-
-// GraphiteRender retrieves the raw metric data by sending a request to
-// /graphite/render endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#render-api
-// and https://graphite-api.readthedocs.io/en/latest/api.html#the-render-api-render
-func (c *vmselectClient) GraphiteRender(t *testing.T, target string, opts QueryOpts) GraphiteRenderResponse {
-	t.Helper()
-
-	url := c.url("select", "graphite/render", opts)
-	values := opts.asURLValues()
-	values.Add("format", "json")
-	values.Add("target", target)
-	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
-	}
-
-	var res GraphiteRenderResponse
-	if err := json.Unmarshal([]byte(resText), &res); err != nil {
-		t.Fatalf("could not unmarshal response data:\n%s\n err: %v", resText, err)
-	}
-	return res
-}
-
-// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
-// for a single time series by sending a request to /graphite/tags/tagSeries
-// endpoint.
-func (c *vmselectClient) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("select", "graphite/tags/tagSeries", opts)
-	values := opts.asURLValues()
-	values.Add("path", record)
-	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-// GraphiteTagsTagMultiSeries is a test helper function that registers Graphite
-// tags for a multiple time series by sending a request to
-// /graphite/tags/tagSeries endpoint.
-func (c *vmselectClient) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("select", "graphite/tags/tagMultiSeries", opts)
-	values := opts.asURLValues()
-	for _, rec := range records {
-		values.Add("path", rec)
-	}
-	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
-	if got, want := statusCode, http.StatusNotImplemented; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d", got, want)
-	}
-}
-
-// PrometheusAPIV1AdminStatusMetricNamesStatsReset resets the metric name usage
-// stats by sending a request to
-// /prometheus/api/v1/admin/status/metric_names_stats/reset endpoint
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
-func (c *vmselectClient) PrometheusAPIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts QueryOpts) {
-	t.Helper()
-	values := opts.asURLValues()
-	res, statusCode := c.vmselectCli.PostForm(t, c.metricNamesStatsResetURL, values, opts.Headers)
-	if statusCode != http.StatusNoContent {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
-	}
-}
-
-// APIV1AdminTenants retrieves the list of tenants by sending a request to
-// /admin/tenants endpoint.
-func (c *vmselectClient) APIV1AdminTenants(t *testing.T, opts QueryOpts) *AdminTenantsResponse {
-	t.Helper()
-	res, statusCode := c.vmselectCli.Get(t, c.tenantsURL, opts.Headers)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
-	}
-
-	tenants := &AdminTenantsResponse{}
-	if err := json.Unmarshal([]byte(res), tenants); err != nil {
-		t.Fatalf("could not unmarshal tenants response data:\n%s\n err: %v", res, err)
-	}
-
-	return tenants
-}
-
-type vminsertClient struct {
-	vminsertCli        *Client
-	url                func(op, path string, opts QueryOpts) string
-	openTSDBURL        func(op, path string, opts QueryOpts) string
-	graphiteListenAddr string
-	sendBlocking       func(t *testing.T, numRecordsToSend int, send func())
-}
-
-// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
-// of records in CSV format for the given tenant by sending an HTTP POST
-// request to prometheus/api/v1/import/csv vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (c *vminsertClient) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("insert", "prometheus/api/v1/import/csv", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	c.sendBlocking(t, len(records), func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
-// of records in Native format for the given tenant by sending an HTTP POST
-// request to prometheus/api/v1/import/native vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (c *vminsertClient) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("insert", "prometheus/api/v1/import/native", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	c.sendBlocking(t, 1, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1Write is a test helper function that inserts a
-// collection of records in Prometheus remote-write format by sending a HTTP
-// POST request to /prometheus/api/v1/write vminsert endpoint.
-func (c *vminsertClient) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("insert", "prometheus/api/v1/write", opts)
-	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
-	recordsCount := len(wr.Timeseries)
-	if prommetadata.IsEnabled() {
-		recordsCount += len(wr.Metadata)
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	c.sendBlocking(t, recordsCount, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
-// collection of records in Prometheus text exposition format for the given
-// tenant by sending a HTTP POST request to
-// /prometheus/api/v1/import/prometheus vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
-func (c *vminsertClient) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("insert", "prometheus/api/v1/import/prometheus", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	var recordsCount int
-	var metadataRecords int
-	uniqueMetadataMetricNames := make(map[string]struct{})
-	for _, record := range records {
-		// metric metadata has the following format:
-		//# HELP importprometheus_series
-		//# TYPE importprometheus_series
-		// it results into single metadata record
-		if strings.HasPrefix(record, "# ") {
-			metadataItems := strings.Split(record, " ")
-			if len(metadataItems) < 3 {
-				t.Fatalf("BUG: unexpected metadata format=%q", record)
-			}
-			metricName := metadataItems[2]
-			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
-				continue
-			}
-			uniqueMetadataMetricNames[metricName] = struct{}{}
-			metadataRecords++
-			continue
-		}
-		recordsCount++
-	}
-	if prommetadata.IsEnabled() {
-		recordsCount += metadataRecords
-	}
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	c.sendBlocking(t, recordsCount, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// InfluxWrite is a test helper function that inserts a collection of records in
-// Influx line format by sending a HTTP POST request to /influx/write endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
-func (c *vminsertClient) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("insert", "influx/write", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "text/plain")
-	c.sendBlocking(t, len(records), func() {
-		t.Helper()
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// OpentelemetryV1Metrics is a test helper function that inserts a
-// collection of records in Opentelemetry protocol format by sending a HTTP
-// POST request to /opentelemetry/v1/metrics vminsert endpoint.
-func (c *vminsertClient) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsData, opts QueryOpts) {
-	t.Helper()
-
-	var recordsCount int
-	for _, rss := range md.ResourceMetrics {
-		for _, sm := range rss.ScopeMetrics {
-			recordsCount += len(sm.Metrics)
-			for _, m := range sm.Metrics {
-				if prommetadata.IsEnabled() {
-					recordsCount += len(m.Metadata)
-				}
-			}
-		}
-	}
-	url := c.url("insert", "opentelemetry/v1/metrics", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := md.MarshalProtobuf(nil)
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/x-protobuf")
-	c.sendBlocking(t, recordsCount, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusOK {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-		}
-	})
-}
-
-// OpenTSDBAPIPut is a test helper function that inserts a collection of
-// records in OpenTSDB format for the given tenant by sending an HTTP POST
-// request to /opentsdb/api/put vminsert endpoint.
-//
-// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
-func (c *vminsertClient) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.openTSDBURL("insert", "opentsdb/api/put", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte("[" + strings.Join(records, ",") + "]")
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	c.sendBlocking(t, len(records), func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusNoContent {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
-		}
-	})
-}
-
-// ZabbixConnectorHistory is a test helper function that inserts a
-// collection of records in zabbixconnector  format by sending a HTTP
-// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
-func (c *vminsertClient) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
-	t.Helper()
-
-	url := c.url("insert", "zabbixconnector/api/v1/history", opts)
-	uv := opts.asURLValues()
-	uvs := uv.Encode()
-	if len(uvs) > 0 {
-		url += "?" + uvs
-	}
-	data := []byte(strings.Join(records, "\n"))
-	headers := opts.getHeaders()
-	headers.Set("Content-Type", "application/json")
-	c.sendBlocking(t, len(records), func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
-		if statusCode != http.StatusOK {
-			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-		}
-	})
-
-}
-
-// GraphiteWrite is a test helper function that sends a
-// collection of records to graphiteListenAddr port.
-//
-// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
-func (c *vminsertClient) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
-	t.Helper()
-	c.vminsertCli.Write(t, c.graphiteListenAddr, records)
-}
-
-type vmstorageClient struct {
-	vmstorageCli   *Client
-	httpListenAddr string
-}
-
-// ForceFlush is a test helper function that forces the flushing of inserted
-// data, so it becomes available for searching immediately.
-func (c *vmstorageClient) ForceFlush(t *testing.T) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/internal/force_flush", c.httpListenAddr)
-	_, statusCode := c.vmstorageCli.Get(t, url, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// ForceMerge is a test helper function that forces the merging of parts.
-func (c *vmstorageClient) ForceMerge(t *testing.T) {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/internal/force_merge", c.httpListenAddr)
-	_, statusCode := c.vmstorageCli.Get(t, url, nil)
-	if statusCode != http.StatusOK {
-		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
-	}
-}
-
-// SnapshotCreate creates a database snapshot by sending a query to the
-// /snapshot/create endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (c *vmstorageClient) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
-	t.Helper()
-
-	data, statusCode := c.vmstorageCli.Post(t, c.SnapshotCreateURL(), nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotCreateResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotCreateURL returns the URL for creating snapshots.
-func (c *vmstorageClient) SnapshotCreateURL() string {
-	return fmt.Sprintf("http://%s/snapshot/create", c.httpListenAddr)
-}
-
-// APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
-// /api/v1/admin/tsdb/snapshot endpoint.
-//
-// See https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot.
-func (c *vmstorageClient) APIV1AdminTSDBSnapshot(t *testing.T) *APIV1AdminTSDBSnapshotResponse {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/api/v1/admin/tsdb/snapshot", c.httpListenAddr)
-	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res APIV1AdminTSDBSnapshotResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal prometheus snapshot create response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotList lists existing database snapshots by sending a query to the
-// /snapshot/list endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (c *vmstorageClient) SnapshotList(t *testing.T) *SnapshotListResponse {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/snapshot/list", c.httpListenAddr)
-	data, statusCode := c.vmstorageCli.Get(t, url, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotListResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDelete deletes a snapshot by sending a query to the
-// /snapshot/delete endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (c *vmstorageClient) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", c.httpListenAddr, snapshotName)
-	data, statusCode := c.vmstorageCli.Delete(t, url)
-	wantStatusCodes := map[int]bool{
-		http.StatusOK:                  true,
-		http.StatusInternalServerError: true,
-	}
-	if !wantStatusCodes[statusCode] {
-		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
-	}
-
-	var res SnapshotDeleteResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}
-
-// SnapshotDeleteAll deletes all snapshots by sending a query to the
-// /snapshot/delete_all endpoint.
-//
-// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
-func (c *vmstorageClient) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
-	t.Helper()
-
-	url := fmt.Sprintf("http://%s/snapshot/delete_all", c.httpListenAddr)
-	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
-	if got, want := statusCode, http.StatusOK; got != want {
-		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
-	}
-
-	var res SnapshotDeleteAllResponse
-	if err := json.Unmarshal([]byte(data), &res); err != nil {
-		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
-	}
-
-	return &res
-}

apptest/model.go

@@ -27,16 +27,13 @@ type PrometheusQuerier interface {
 	PrometheusAPIV1LabelValues(t *testing.T, labelName, query string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse
 	PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
 	PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata
-	PrometheusAPIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse
-	PrometheusAPIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts)
+
+	APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts)
 
 	// TODO(@rtm0): Prometheus does not provide this API. Either move it to a
 	// separate interface or rename this interface to allow for multiple querier
 	// types.
 	GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse
-	GraphiteMetricsFind(t *testing.T, query string, opts QueryOpts) GraphiteMetricsFindResponse
-	GraphiteMetricsExpand(t *testing.T, query string, opts QueryOpts) GraphiteMetricsExpandResponse
-	GraphiteRender(t *testing.T, target string, opts QueryOpts) GraphiteRenderResponse
 	GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts)
 	GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts)
 }
@@ -94,9 +91,14 @@ type QueryOpts struct {
 	Format         string
 	NoCache        string
 	Headers        http.Header
-	From           string
-	Until          string
-	StorageStep    string
+}
+
+// getTenant returns tenant with optional default value
+func (qos *QueryOpts) getTenant() string {
+	if qos.Tenant == "" {
+		return "0"
+	}
+	return qos.Tenant
 }
 
 func (qos *QueryOpts) getHeaders() http.Header {
@@ -129,9 +131,6 @@ func (qos *QueryOpts) asURLValues() url.Values {
 	addNonEmpty("latency_offset", qos.LatencyOffset)
 	addNonEmpty("format", qos.Format)
 	addNonEmpty("nocache", qos.NoCache)
-	addNonEmpty("from", qos.From)
-	addNonEmpty("until", qos.Until)
-	addNonEmpty("storage_step", qos.StorageStep)
 
 	return uv
 }
@@ -489,6 +488,10 @@ type TSDBStatusResponse struct {
 	Data      TSDBStatusResponseData
 }
 
+// GraphiteMetricsIndexResponse is an in-memory representation of the json response
+// returned by the /graphite/metrics/index.json endpoint.
+type GraphiteMetricsIndexResponse = []string
+
 // AdminTenantsResponse is an in-memory representation of the json response
 // returned by the /api/v1/admin/tenants endpoint.
 type AdminTenantsResponse struct {
@@ -538,32 +541,3 @@ func sortTSDBStatusResponseEntries(entries []TSDBStatusResponseEntry) {
 		return left.Count < right.Count
 	})
 }
-
-// GraphiteMetricsIndexResponse is an in-memory representation of the json response
-// returned by the /graphite/metrics/index.json endpoint.
-type GraphiteMetricsIndexResponse = []string
-
-type GraphiteMetric struct {
-	Id            string
-	Text          string
-	AllowChildren int
-	Expandable    int
-	Leaf          int
-}
-
-// GraphiteMetricsIndexResponse is an in-memory representation of the json response
-// returned by the /graphite/metrics/find endpoint.
-type GraphiteMetricsFindResponse = []GraphiteMetric
-
-// GraphiteMetricsExpandResponse is an in-memory representation of the json response
-// returned by the /graphite/metrics/expand endpoint.
-type GraphiteMetricsExpandResponse = []string
-
-type GraphiteRenderedTarget struct {
-	Target     string
-	Datapoints [][2]float64
-}
-
-// GraphiteRenderResponse is an in-memory representation of the json response
-// returned by the /graphite/render endpoint.
-type GraphiteRenderResponse = []GraphiteRenderedTarget

apptest/testcase.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -88,11 +87,19 @@ func (tc *TestCase) MustStartDefaultVmsingle() *Vmsingle {
 }
 
 // MustStartVmsingle is a test helper function that starts an instance of
-// vmsingle (latest version) and fails the test if the app fails to start.
+// vmsingle located at ../../bin/victoria-metrics-race and fails the test if the app
+// fails to start.
 func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle {
 	tc.t.Helper()
+	return tc.MustStartVmsingleAt(instance, "../../bin/victoria-metrics-race", flags)
+}
 
-	app, err := StartVmsingle(instance, flags, tc.cli, tc.output)
+// MustStartVmsingleAt is a test helper function that starts an instance of
+// vmsingle and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmsingleAt(instance, binary string, flags []string) *Vmsingle {
+	tc.t.Helper()
+
+	app, err := StartVmsingleAt(instance, binary, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -101,11 +108,19 @@ func (tc *TestCase) MustStartVmsingle(instance string, flags []string) *Vmsingle
 }
 
 // MustStartVmstorage is a test helper function that starts an instance of
-// vmstorage (latest version) and fails the test if the app fails to start.
+// vmstorage located at ../../bin/vmstorage-race and fails the test if the app fails
+// to start.
 func (tc *TestCase) MustStartVmstorage(instance string, flags []string) *Vmstorage {
 	tc.t.Helper()
+	return tc.MustStartVmstorageAt(instance, "../../bin/vmstorage-race", flags)
+}
 
-	app, err := StartVmstorage(instance, flags, tc.cli, tc.output)
+// MustStartVmstorageAt is a test helper function that starts an instance of
+// vmstorage and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmstorageAt(instance string, binary string, flags []string) *Vmstorage {
+	tc.t.Helper()
+
+	app, err := StartVmstorageAt(instance, binary, flags, tc.cli, tc.output)
 	if err != nil {
 		tc.t.Fatalf("Could not start %s: %v", instance, err)
 	}
@@ -114,7 +129,7 @@ func (tc *TestCase) MustStartVmstorage(instance string, flags []string) *Vmstora
 }
 
 // MustStartVmselect is a test helper function that starts an instance of
-// vmselect (latest version) and fails the test if the app fails to start.
+// vmselect and fails the test if the app fails to start.
 func (tc *TestCase) MustStartVmselect(instance string, flags []string) *Vmselect {
 	tc.t.Helper()
 
@@ -154,18 +169,6 @@ func (tc *TestCase) MustStartVmagent(instance string, flags []string, promScrape
 	return app
 }
 
-// MustStartDefaultRWVmagent is a test helper function that starts an instance of
-// vmagent with defaults suitable for remote-write tests.
-func (tc *TestCase) MustStartDefaultRWVmagent(instance string, flags []string) *Vmagent {
-	tc.t.Helper()
-
-	defaultFlags := []string{
-		"-remoteWrite.flushInterval=50ms",
-	}
-	defaultFlags = slices.Concat(defaultFlags, flags)
-	return tc.MustStartVmagent(instance, defaultFlags, ``)
-}
-
 // Vmcluster represents a typical cluster setup: several vmstorage replicas, one
 // vminsert, and one vmselect.
 //
@@ -274,8 +277,10 @@ func (tc *TestCase) MustStartDefaultCluster() *Vmcluster {
 // tests usually come paired with corresponding vmsingle tests.
 type ClusterOptions struct {
 	Vmstorage1Instance string
+	Vmstorage1Binary   string
 	Vmstorage1Flags    []string
 	Vmstorage2Instance string
+	Vmstorage2Binary   string
 	Vmstorage2Flags    []string
 	VminsertInstance   string
 	VminsertFlags      []string
@@ -287,8 +292,15 @@ type ClusterOptions struct {
 func (tc *TestCase) MustStartCluster(opts *ClusterOptions) *Vmcluster {
 	tc.t.Helper()
 
-	vmstorage1 := tc.MustStartVmstorage(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
-	vmstorage2 := tc.MustStartVmstorage(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
+	if opts.Vmstorage1Binary == "" {
+		opts.Vmstorage1Binary = "../../bin/vmstorage-race"
+	}
+	vmstorage1 := tc.MustStartVmstorageAt(opts.Vmstorage1Instance, opts.Vmstorage1Binary, opts.Vmstorage1Flags)
+
+	if opts.Vmstorage2Binary == "" {
+		opts.Vmstorage2Binary = "../../bin/vmstorage-race"
+	}
+	vmstorage2 := tc.MustStartVmstorageAt(opts.Vmstorage2Instance, opts.Vmstorage2Binary, opts.Vmstorage2Flags)
 
 	opts.VminsertFlags = append(opts.VminsertFlags, []string{
 		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),

apptest/testcase_legacy.go

@@ -1,50 +0,0 @@
-package apptest
-
-// MustStartVmsingle_v1_132_0 is a test helper function that starts an instance
-// of vmsingle-v1.132.0 (last version that uses legacy index) and fails the test
-// if the app fails to start.
-func (tc *TestCase) MustStartVmsingle_v1_132_0(instance string, flags []string) *Vmsingle {
-	tc.t.Helper()
-
-	app, err := StartVmsingle_v1_132_0(instance, flags, tc.cli, tc.output)
-	if err != nil {
-		tc.t.Fatalf("Could not start %s: %v", instance, err)
-	}
-	tc.addApp(instance, app)
-	return app
-}
-
-// MustStartVmstorage_v1_132_0 is a test helper function that starts an instance
-// of vmstorage-v1.132.0 (last version that uses legacy index)  and fails the
-// test if the app fails to start.
-func (tc *TestCase) MustStartVmstorage_v1_132_0(instance string, flags []string) *Vmstorage {
-	tc.t.Helper()
-
-	app, err := StartVmstorage_v1_132_0(instance, flags, tc.cli, tc.output)
-	if err != nil {
-		tc.t.Fatalf("Could not start %s: %v", instance, err)
-	}
-	tc.addApp(instance, app)
-	return app
-}
-
-// MustStartCluster_v1_132_0 starts a cluster with vmstorage-v1.132.0 with
-// custom flags.
-func (tc *TestCase) MustStartCluster_v1_132_0(opts *ClusterOptions) *Vmcluster {
-	tc.t.Helper()
-
-	vmstorage1 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
-	vmstorage2 := tc.MustStartVmstorage_v1_132_0(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
-
-	opts.VminsertFlags = append(opts.VminsertFlags, []string{
-		"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
-	}...)
-	vminsert := tc.MustStartVminsert(opts.VminsertInstance, opts.VminsertFlags)
-
-	opts.VmselectFlags = append(opts.VmselectFlags, []string{
-		"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
-	}...)
-	vmselect := tc.MustStartVmselect(opts.VmselectInstance, opts.VmselectFlags)
-
-	return &Vmcluster{vminsert, vmselect, []*Vmstorage{vmstorage1, vmstorage2}}
-}

apptest/tests/ingestprotocols_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
-	otlppb "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
 )
 
 func TestSingleIngestionProtocols(t *testing.T) {
@@ -298,231 +297,6 @@ func TestSingleIngestionProtocols(t *testing.T) {
 		},
 	})
 
-	// opentelemetry metrics protocol
-	tsNano := uint64(1707123456700 * 1e6) // 2024-02-05T08:57:36.700Z
-	otlpData := otlppb.MetricsData{
-		ResourceMetrics: []*otlppb.ResourceMetrics{
-			{
-				Resource: &otlppb.Resource{
-					Attributes: []*otlppb.KeyValue{
-						{
-							Key:   "foo",
-							Value: &otlppb.AnyValue{StringValue: new("bar")},
-						},
-					},
-				},
-				ScopeMetrics: []*otlppb.ScopeMetrics{
-					{
-						Scope: &otlppb.InstrumentationScope{
-							Name:    new("otlp"),
-							Version: new("v1"),
-							Attributes: []*otlppb.KeyValue{
-								{
-									Key:   "scope_attribute",
-									Value: &otlppb.AnyValue{IntValue: new(int64(100))},
-								},
-							},
-						},
-						Metrics: []*otlppb.Metric{
-							{
-								Name: "otlp_series_gauge",
-								Gauge: &otlppb.Gauge{
-									DataPoints: []*otlppb.NumberDataPoint{
-										{IntValue: new(int64(10)), TimeUnixNano: tsNano},
-										{IntValue: new(int64(5)), TimeUnixNano: tsNano, Attributes: []*otlppb.KeyValue{{Key: "bar", Value: &otlppb.AnyValue{StringValue: new("foo")}}}},
-									},
-								},
-							},
-							{
-								Name: "otlp_series_counter",
-								Sum: &otlppb.Sum{
-									DataPoints: []*otlppb.NumberDataPoint{
-										{IntValue: new(int64(30)), TimeUnixNano: tsNano, Attributes: []*otlppb.KeyValue{{Key: "bar", Value: &otlppb.AnyValue{StringValue: new("foo")}}}},
-									},
-								},
-							},
-						},
-					},
-					{
-						Scope: &otlppb.InstrumentationScope{
-							Name:    new("otlp2"),
-							Version: new("v2"),
-						},
-						Metrics: []*otlppb.Metric{
-							{
-								Name: "otlp_series_histogram",
-								Histogram: &otlppb.Histogram{
-									DataPoints: []*otlppb.HistogramDataPoint{
-										{
-											Count:          15,
-											Sum:            new(float64(100)),
-											ExplicitBounds: []float64{0.1, 0.5, 1.0, 5.0},
-											BucketCounts:   []uint64{0, 5, 10, 0, 0},
-											TimeUnixNano:   tsNano,
-											Attributes: []*otlppb.KeyValue{
-												{Key: "baz", Value: &otlppb.AnyValue{ArrayValue: &otlppb.ArrayValue{Values: []*otlppb.AnyValue{
-													{StringValue: new("foo")},
-													{IntValue: new(int64(100))},
-												}}}},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			{
-				ScopeMetrics: []*otlppb.ScopeMetrics{
-					{
-						Metrics: []*otlppb.Metric{
-							{
-								Name: "otlp_series_summary",
-								Summary: &otlppb.Summary{
-									DataPoints: []*otlppb.SummaryDataPoint{
-										{
-											Attributes:   []*otlppb.KeyValue{},
-											TimeUnixNano: tsNano,
-											Sum:          17.5,
-											Count:        2,
-											QuantileValues: []*otlppb.ValueAtQuantile{
-												{
-													Quantile: 0.1,
-													Value:    7.5,
-												},
-												{
-													Quantile: 0.5,
-													Value:    10.0,
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-	sut.OpentelemetryV1Metrics(t, otlpData, apptest.QueryOpts{})
-	sut.ForceFlush(t)
-	f(sut, &opts{
-		query: `{__name__=~"otlp.+"}`,
-		wantMetrics: []map[string]string{
-			{
-				"__name__":                         "otlp_series_counter",
-				"foo":                              "bar",
-				"bar":                              "foo",
-				"scope.attributes.scope_attribute": "100",
-				"scope.name":                       "otlp",
-				"scope.version":                    "v1",
-			},
-			{
-				"__name__":                         "otlp_series_gauge",
-				"foo":                              "bar",
-				"bar":                              "foo",
-				"scope.attributes.scope_attribute": "100",
-				"scope.name":                       "otlp",
-				"scope.version":                    "v1",
-			},
-			{
-				"__name__":                         "otlp_series_gauge",
-				"foo":                              "bar",
-				"scope.attributes.scope_attribute": "100",
-				"scope.name":                       "otlp",
-				"scope.version":                    "v1",
-			},
-
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "+Inf",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "0.1",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "0.5",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "1",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "5",
-			},
-			{
-				"__name__":      "otlp_series_histogram_count",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-			},
-			{
-				"__name__":      "otlp_series_histogram_sum",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-			},
-			{
-				"__name__": "otlp_series_summary",
-				"quantile": "0.1",
-			},
-			{
-				"__name__": "otlp_series_summary",
-				"quantile": "0.5",
-			},
-			{
-				"__name__": "otlp_series_summary_count",
-			},
-			{
-				"__name__": "otlp_series_summary_sum",
-			},
-		},
-		wantSamples: []*apptest.Sample{
-			{Timestamp: 1707123456700, Value: 30},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 5},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 10},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 0},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 5},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 100},  // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 7.5},  // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 10},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 2},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 17.5}, // 2024-02-05T08:57:36.700Z
-
-		},
-	})
-
 }
 
 func TestSingleCardinalityLimiter(t *testing.T) {
@@ -944,231 +718,6 @@ func TestClusterIngestionProtocols(t *testing.T) {
 		},
 	})
 
-	// opentelemetry metrics protocol
-	tsNano := uint64(1707123456700 * 1e6) // 2024-02-05T08:57:36.700Z
-	otlpData := otlppb.MetricsData{
-		ResourceMetrics: []*otlppb.ResourceMetrics{
-			{
-				Resource: &otlppb.Resource{
-					Attributes: []*otlppb.KeyValue{
-						{
-							Key:   "foo",
-							Value: &otlppb.AnyValue{StringValue: new("bar")},
-						},
-					},
-				},
-				ScopeMetrics: []*otlppb.ScopeMetrics{
-					{
-						Scope: &otlppb.InstrumentationScope{
-							Name:    new("otlp"),
-							Version: new("v1"),
-							Attributes: []*otlppb.KeyValue{
-								{
-									Key:   "scope_attribute",
-									Value: &otlppb.AnyValue{IntValue: new(int64(100))},
-								},
-							},
-						},
-						Metrics: []*otlppb.Metric{
-							{
-								Name: "otlp_series_gauge",
-								Gauge: &otlppb.Gauge{
-									DataPoints: []*otlppb.NumberDataPoint{
-										{IntValue: new(int64(10)), TimeUnixNano: tsNano},
-										{IntValue: new(int64(5)), TimeUnixNano: tsNano, Attributes: []*otlppb.KeyValue{{Key: "bar", Value: &otlppb.AnyValue{StringValue: new("foo")}}}},
-									},
-								},
-							},
-							{
-								Name: "otlp_series_counter",
-								Sum: &otlppb.Sum{
-									DataPoints: []*otlppb.NumberDataPoint{
-										{IntValue: new(int64(30)), TimeUnixNano: tsNano, Attributes: []*otlppb.KeyValue{{Key: "bar", Value: &otlppb.AnyValue{StringValue: new("foo")}}}},
-									},
-								},
-							},
-						},
-					},
-					{
-						Scope: &otlppb.InstrumentationScope{
-							Name:    new("otlp2"),
-							Version: new("v2"),
-						},
-						Metrics: []*otlppb.Metric{
-							{
-								Name: "otlp_series_histogram",
-								Histogram: &otlppb.Histogram{
-									DataPoints: []*otlppb.HistogramDataPoint{
-										{
-											Count:          15,
-											Sum:            new(float64(100)),
-											ExplicitBounds: []float64{0.1, 0.5, 1.0, 5.0},
-											BucketCounts:   []uint64{0, 5, 10, 0, 0},
-											TimeUnixNano:   tsNano,
-											Attributes: []*otlppb.KeyValue{
-												{Key: "baz", Value: &otlppb.AnyValue{ArrayValue: &otlppb.ArrayValue{Values: []*otlppb.AnyValue{
-													{StringValue: new("foo")},
-													{IntValue: new(int64(100))},
-												}}}},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			{
-				ScopeMetrics: []*otlppb.ScopeMetrics{
-					{
-						Metrics: []*otlppb.Metric{
-							{
-								Name: "otlp_series_summary",
-								Summary: &otlppb.Summary{
-									DataPoints: []*otlppb.SummaryDataPoint{
-										{
-											Attributes:   []*otlppb.KeyValue{},
-											TimeUnixNano: tsNano,
-											Sum:          17.5,
-											Count:        2,
-											QuantileValues: []*otlppb.ValueAtQuantile{
-												{
-													Quantile: 0.1,
-													Value:    7.5,
-												},
-												{
-													Quantile: 0.5,
-													Value:    10.0,
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-	vminsert.OpentelemetryV1Metrics(t, otlpData, apptest.QueryOpts{})
-	vmstorage.ForceFlush(t)
-	f(&opts{
-		query: `{__name__=~"otlp.+"}`,
-		wantMetrics: []map[string]string{
-			{
-				"__name__":                         "otlp_series_counter",
-				"foo":                              "bar",
-				"bar":                              "foo",
-				"scope.attributes.scope_attribute": "100",
-				"scope.name":                       "otlp",
-				"scope.version":                    "v1",
-			},
-			{
-				"__name__":                         "otlp_series_gauge",
-				"foo":                              "bar",
-				"bar":                              "foo",
-				"scope.attributes.scope_attribute": "100",
-				"scope.name":                       "otlp",
-				"scope.version":                    "v1",
-			},
-			{
-				"__name__":                         "otlp_series_gauge",
-				"foo":                              "bar",
-				"scope.attributes.scope_attribute": "100",
-				"scope.name":                       "otlp",
-				"scope.version":                    "v1",
-			},
-
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "+Inf",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "0.1",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "0.5",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "1",
-			},
-			{
-				"__name__":      "otlp_series_histogram_bucket",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-				"le":            "5",
-			},
-			{
-				"__name__":      "otlp_series_histogram_count",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-			},
-			{
-				"__name__":      "otlp_series_histogram_sum",
-				"baz":           `["foo",100]`,
-				"foo":           "bar",
-				"scope.name":    "otlp2",
-				"scope.version": "v2",
-			},
-			{
-				"__name__": "otlp_series_summary",
-				"quantile": "0.1",
-			},
-			{
-				"__name__": "otlp_series_summary",
-				"quantile": "0.5",
-			},
-			{
-				"__name__": "otlp_series_summary_count",
-			},
-			{
-				"__name__": "otlp_series_summary_sum",
-			},
-		},
-		wantSamples: []*apptest.Sample{
-			{Timestamp: 1707123456700, Value: 30},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 5},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 10},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 0},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 5},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 15},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 100},  // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 7.5},  // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 10},   // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 2},    // 2024-02-05T08:57:36.700Z
-			{Timestamp: 1707123456700, Value: 17.5}, // 2024-02-05T08:57:36.700Z
-
-		},
-	})
-
 }
 
 func TestClusterCardinalityLimiter(t *testing.T) {

apptest/tests/legacy_indexdb_test.go

@@ -2,6 +2,7 @@ package tests
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"slices"
 	"testing"
@@ -10,6 +11,11 @@ import (
 	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
 )
 
+var (
+	legacyVmsinglePath  = os.Getenv("VM_LEGACY_VMSINGLE_PATH")
+	legacyVmstoragePath = os.Getenv("VM_LEGACY_VMSTORAGE_PATH")
+)
+
 type testLegacyDeleteSeriesOpts struct {
 	startLegacySUT func() at.PrometheusWriteQuerier
 	startNewSUT    func() at.PrometheusWriteQuerier
@@ -25,7 +31,7 @@ func TestLegacySingleDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.maxStalenessInterval=1m",
@@ -58,13 +64,15 @@ func TestLegacyClusterDeleteSeries(t *testing.T) {
 
 	opts := testLegacyDeleteSeriesOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
+			return tc.MustStartCluster(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -183,7 +191,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - start legacy vmsingle
 	// - insert data1
-	// - confirm that metric names and samples are searchable
+	// - confirm that metric names and samples are searcheable
 	// - stop legacy vmsingle
 	const step = 24 * 3600 * 1000 // 24h
 	start1 := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
@@ -196,17 +204,17 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 	opts.stopLegacySUT()
 
 	// - start new vmsingle
-	// - confirm that data1 metric names and samples are searchable
+	// - confirm that data1 metric names and samples are searcheable
 	// - delete data1
-	// - confirm that data1 metric names and samples are not searchable anymore
+	// - confirm that data1 metric names and samples are not searcheable anymore
 	// - insert data2 (same metric names, different dates)
-	// - confirm that metric names become searchable again
-	// - confirm that data1 samples are not searchable and data2 samples are searchable
+	// - confirm that metric names become searcheable again
+	// - confirm that data1 samples are not searchable and data2 samples are searcheable
 
 	newSUT := opts.startNewSUT()
 	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
 
-	newSUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	wantNoResults := &want{
 		series:       []map[string]string{},
 		queryResults: []*at.QueryResult{},
@@ -222,7 +230,7 @@ func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
 
 	// - restart new vmsingle
 	// - confirm that metric names still searchable, data1 samples are not
-	//   searchable, and data2 samples are searchable
+	//   searchable, and data2 samples are searcheable
 
 	opts.stopNewSUT()
 	newSUT = opts.startNewSUT()
@@ -247,7 +255,7 @@ func TestLegacySingleBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -290,13 +298,15 @@ func TestLegacyClusterBackupRestore(t *testing.T) {
 
 	opts := testLegacyBackupRestoreOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
+			return tc.MustStartCluster(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -573,7 +583,7 @@ func TestLegacySingleDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartVmsingle_v1_132_0("vmsingle-legacy", []string{
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
 				"-storageDataPath=" + storageDataPath,
 				"-retentionPeriod=100y",
 				"-search.disableCache=true",
@@ -608,13 +618,15 @@ func TestLegacyClusterDowngrade(t *testing.T) {
 
 	opts := testLegacyDowngradeOpts{
 		startLegacySUT: func() at.PrometheusWriteQuerier {
-			return tc.MustStartCluster_v1_132_0(&at.ClusterOptions{
+			return tc.MustStartCluster(&at.ClusterOptions{
 				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
 				Vmstorage1Flags: []string{
 					"-storageDataPath=" + storage1DataPath,
 					"-retentionPeriod=100y",
 				},
 				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
 				Vmstorage2Flags: []string{
 					"-storageDataPath=" + storage2DataPath,
 					"-retentionPeriod=100y",
@@ -865,7 +877,7 @@ func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
 	// Ingest legacy2 records, ensure the queries return only legacy2.
 	legacySUT = opts.startLegacySUT()
 	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1, numMetrics)
-	legacySUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	legacySUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	assertQueries(legacySUT, `{__name__=~".*"}`, wantEmpty, numMetrics)
 	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy2Data, at.QueryOpts{})
 	legacySUT.ForceFlush(t)
@@ -879,7 +891,7 @@ func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
 	newSUT = opts.startNewSUT()
 	// series count includes deleted metrics
 	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy2New1, 3*numMetrics)
-	newSUT.PrometheusAPIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
 	// series count includes deleted metrics
 	assertQueries(newSUT, `{__name__=~".*"}`, wantEmpty, 3*numMetrics)
 	opts.stopNewSUT()

apptest/tests/metric_names_stats_test.go

@@ -48,7 +48,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3"},
 		},
 	}
-	got := sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got := sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -63,7 +63,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -90,7 +90,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 		},
 	}
 	expectedStatsResponse.Sort()
-	gotStatus := sut.PrometheusAPIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{})
+	gotStatus := sut.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
 		t.Errorf("unexpected APIV1StatusTSDB response (-want, +got):\n%s", diff)
 	}
@@ -105,7 +105,7 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -118,17 +118,17 @@ func TestSingleMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_3", QueryRequestsCount: 1},
 		},
 	}
-	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "2", "", apptest.QueryOpts{})
+	got = sut.APIV1StatusMetricNamesStats(t, "", "2", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
 	// reset state and check empty request response
-	sut.PrometheusAPIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
+	sut.APIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
 	expected = apptest.MetricNamesStatsResponse{
 		Records: []apptest.MetricNamesStatsRecord{},
 	}
-	got = sut.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
+	got = sut.APIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{})
 	if diff := cmp.Diff(expected, got); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
@@ -158,7 +158,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 		fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VmselectAddr(), vmstorage2.VmselectAddr()),
 	})
 	// verify empty stats
-	resp := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "0:0"})
+	resp := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "0:0"})
 	if len(resp.Records) != 0 {
 		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Records), 0)
 	}
@@ -198,7 +198,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 				{MetricName: "metric_name_3"},
 			},
 		}
-		gotStats := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
+		gotStats := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expected, gotStats); diff != "" {
 			t.Errorf("unexpected response (-want, +got):\n%s", diff)
 		}
@@ -216,7 +216,7 @@ func TestClusterMetricNamesStats(t *testing.T) {
 				{MetricName: "metric_name_1", QueryRequestsCount: 3},
 			},
 		}
-		gotStats = vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
+		gotStats = vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expected, gotStats); diff != "" {
 			t.Errorf("unexpected response tenant: %s (-want, +got):\n%s", tenantID, diff)
 		}
@@ -243,9 +243,9 @@ func TestClusterMetricNamesStats(t *testing.T) {
 			},
 		}
 		expectedStatsResponse.Sort()
-		gotStatus := vmselect.PrometheusAPIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{Tenant: tenantID})
+		gotStatus := vmselect.APIV1StatusTSDB(t, "", date, "", apptest.QueryOpts{Tenant: tenantID})
 		if diff := cmp.Diff(expectedStatsResponse, gotStatus, tsdbMetricNameEntryCmpOpts); diff != "" {
-			t.Errorf("unexpected TSDB status for tenant %s (-want, +got):\n%s", tenantID, diff)
+			t.Errorf("unexpected APIV1StatusTSDB response tenant: %s (-want, +got):\n%s", tenantID, diff)
 		}
 	}
 
@@ -258,14 +258,14 @@ func TestClusterMetricNamesStats(t *testing.T) {
 			{MetricName: "metric_name_1", QueryRequestsCount: 9},
 		},
 	}
-	gotStats := vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
+	gotStats := vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
 	if diff := cmp.Diff(expected, gotStats); diff != "" {
 		t.Errorf("unexpected response (-want, +got):\n%s", diff)
 	}
 
 	// reset cache and check empty state
-	vmselect.PrometheusAPIV1AdminStatusMetricNamesStatsReset(t, apptest.QueryOpts{})
-	resp = vmselect.PrometheusAPIV1StatusMetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
+	vmselect.MetricNamesStatsReset(t, apptest.QueryOpts{})
+	resp = vmselect.MetricNamesStats(t, "", "", "", apptest.QueryOpts{Tenant: "multitenant"})
 	if len(resp.Records) != 0 {
 		t.Fatalf("want 0 records, got: %d", len(resp.Records))
 	}

apptest/tests/multitenancy_via_headers_test.go

@@ -1,313 +0,0 @@
-package tests
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
-)
-
-func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
-	fs.MustRemoveDir(t.Name())
-
-	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
-	cmpSROpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Status", "IsPartial")
-
-	tc := apptest.NewTestCase(t)
-	defer tc.Stop()
-	vmstorage := tc.MustStartVmstorage("vmstorage", []string{
-		"-storageDataPath=" + tc.Dir() + "/vmstorage",
-		"-retentionPeriod=100y",
-	})
-	vminsert := tc.MustStartVminsert("vminsert", []string{
-		"-storageNode=" + vmstorage.VminsertAddr(),
-		"-enableMultitenancyViaHeaders",
-	})
-	vmselect := tc.MustStartVmselect("vmselect", []string{
-		"-storageNode=" + vmstorage.VmselectAddr(),
-		"-search.tenantCacheExpireDuration=0",
-		"-enableMultitenancyViaHeaders",
-	})
-
-	multitenant := make(http.Header)
-	multitenant.Set("AccountID", "multitenant")
-
-	// test for empty tenants request
-	got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
-		Headers: multitenant,
-		Step:    "5m",
-		Time:    "2022-05-10T08:03:00.000Z",
-	})
-	want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
-	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// ingest per tenant data and verify it with search
-	samples := []string{
-		`foo_bar 1.00 1652169600000`, // 2022-05-10T08:00:00Z
-		`foo_bar 2.00 1652169660000`, // 2022-05-10T08:01:00Z
-		`foo_bar 3.00 1652169720000`, // 2022-05-10T08:02:00Z
-	}
-	tenantHeaders := []map[string]string{
-		{"AccountID": "1", "ProjectID": "1"},
-		{"AccountID": "1", "ProjectID": "15"},
-		{"AccountID": "2"},
-		{"ProjectID": "3"},
-	}
-	instantCT := "2022-05-10T08:05:00.000Z" // 1652169900 Unix seconds
-	for _, headers := range tenantHeaders {
-		h := make(http.Header)
-		for k, v := range headers {
-			h.Set(k, v)
-		}
-		vminsert.PrometheusAPIV1ImportPrometheus(t, samples, apptest.QueryOpts{Headers: h})
-		vmstorage.ForceFlush(t)
-
-		// verify tenants are searchable via tenantID in headers
-		got := vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
-			Headers: h, Time: instantCT,
-		})
-		want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}]}}`)
-		if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
-			t.Errorf("unexpected response (-want, +got):\n%s", diff)
-		}
-	}
-
-	// verify all tenants searchable with multitenant header
-
-	//  /api/v1/query
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
-		`{"data":
-       {"result":[
-          {"metric":{"__name__":"foo_bar","vm_account_id":"0","vm_project_id":"3"},"value":[1652169900,"3"]},
-          {"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id": "1"},"value":[1652169900,"3"]},
-          {"metric":{"__name__":"foo_bar","vm_account_id":"1","vm_project_id":"15"},"value":[1652169900,"3"]},
-          {"metric":{"__name__":"foo_bar","vm_account_id":"2","vm_project_id":"0"},"value":[1652169900,"3"]}
-                 ]
-       }
-     }`,
-	)
-
-	got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
-		Headers: multitenant,
-		Time:    instantCT,
-	})
-	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// /api/v1/query_range aggregated by tenant labels
-	query := "sum(foo_bar) by(vm_account_id,vm_project_id)"
-	got = vmselect.PrometheusAPIV1QueryRange(t, query, apptest.QueryOpts{
-		Headers: multitenant,
-		Start:   "2022-05-10T07:59:00.000Z",
-		End:     "2022-05-10T08:05:00.000Z",
-		Step:    "1m",
-	})
-
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
-		`{"data": 
-        {"result": [
-          {"metric": {"vm_account_id": "0","vm_project_id":"3"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
-          {"metric": {"vm_account_id": "1","vm_project_id":"1"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
-          {"metric": {"vm_account_id": "1","vm_project_id":"15"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]},
-          {"metric": {"vm_account_id": "2","vm_project_id":"0"}, "values": [[1652169600,"1"],[1652169660,"2"],[1652169720,"3"],[1652169780,"3"]]}
-                   ]
-        }
-     }`)
-	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// verify /api/v1/series response
-
-	wantSR := apptest.NewPrometheusAPIV1SeriesResponse(t,
-		`{"data": [
-        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
-        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"},
-        {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"},
-        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"}
-              ]
-     }`)
-	wantSR.Sort()
-
-	gotSR := vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
-		Headers: multitenant,
-		Start:   "2022-05-10T08:03:00.000Z",
-	})
-	gotSR.Sort()
-	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// test ingestion with multitenant header, tenants must be populated from labels
-	//
-	var tenantLabelsSamples = []string{
-		`foo_bar{vm_account_id="5"} 1.00 1652169720000`,                    // 2022-05-10T08:02:00Z'
-		`foo_bar{vm_project_id="10"} 2.00 1652169660000`,                   // 2022-05-10T08:01:00Z
-		`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
-	}
-
-	vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Headers: multitenant})
-	vmstorage.ForceFlush(t)
-
-	//  /api/v1/query with query filters
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
-		`{"data":
-        {"result":[
-	         {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"value":[1652169900,"1"]},
-	         {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id":"15"},"value":[1652169900,"3"]}
-	                ]
-        }
-    }`,
-	)
-	got = vmselect.PrometheusAPIV1Query(t, `foo_bar{vm_account_id="5"}`, apptest.QueryOpts{
-		Time:    instantCT,
-		Headers: multitenant,
-	})
-	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// /api/v1/series with extra_filters
-
-	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
-		`{"data": [
-	       {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"15"},
-	       {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"}
-	             ]
-	   }`)
-	wantSR.Sort()
-	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
-		Start:        "2022-05-10T08:00:00.000Z",
-		End:          "2022-05-10T08:30:00.000Z",
-		ExtraFilters: []string{`{vm_project_id="15"}`},
-		Headers:      multitenant,
-	})
-	gotSR.Sort()
-
-	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// /api/v1/label/../value with extra_filters
-
-	wantVR := apptest.NewPrometheusAPIV1LabelValuesResponse(t,
-		`{"data": [
-	       "5"
-	             ]
-	   }`)
-	// matchQuery is ignored for /api/v1/label/<labelName>/values lookups with multitenant token
-	gotVR := vmselect.PrometheusAPIV1LabelValues(t, "vm_account_id", "xxx", apptest.QueryOpts{
-		Start:        "2022-05-10T08:00:00.000Z",
-		End:          "2022-05-10T08:30:00.000Z",
-		ExtraFilters: []string{`{vm_account_id="5"}`},
-		Headers:      multitenant,
-	})
-	gotSR.Sort()
-
-	if diff := cmp.Diff(wantVR, gotVR, cmpopts.IgnoreFields(apptest.PrometheusAPIV1LabelValuesResponse{}, "Status", "IsPartial")); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// Delete series from specific tenant
-	tenantID := make(http.Header)
-	tenantID.Set("AccountID", "5")
-	tenantID.Set("ProjectID", "15")
-	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
-		Headers: tenantID,
-	})
-	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
-		`{"data": [
-        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"},
-        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
-        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"1"},
-        {"__name__":"foo_bar", "vm_account_id":"1", "vm_project_id":"15"},
-        {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"},
-        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
-              ]
-     }`)
-	wantSR.Sort()
-
-	gotSR = vmselect.PrometheusAPIV1Series(t, "foo_bar", apptest.QueryOpts{
-		Headers: multitenant,
-		Start:   "2022-05-10T08:03:00.000Z",
-	})
-	gotSR.Sort()
-	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	// Delete series for multitenant with tenant filter
-	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
-		Headers: multitenant,
-	})
-
-	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
-		`{"data": [
-        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"3"},
-        {"__name__":"foo_bar", "vm_account_id":"0", "vm_project_id":"10"},
-        {"__name__":"foo_bar", "vm_account_id":"2", "vm_project_id":"0"},
-        {"__name__":"foo_bar", "vm_account_id":"5", "vm_project_id":"0"}
-              ]
-     }`)
-	wantSR.Sort()
-
-	gotSR = vmselect.PrometheusAPIV1Series(t, `foo_bar`, apptest.QueryOpts{
-		Headers: multitenant,
-		Start:   "2022-05-10T08:03:00.000Z",
-	})
-	gotSR.Sort()
-	if diff := cmp.Diff(wantSR, gotSR, cmpSROpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-
-	if got := vmselect.GetIntMetric(t, `vm_cache_requests_total{type="multitenancy/tenants"}`); got != 0 {
-		t.Errorf("unexpected multitenancy tenants cache requests; got %d; want 0", got)
-	}
-
-	if got := vmselect.GetIntMetric(t, `vm_cache_misses_total{type="multitenancy/tenants"}`); got != 0 {
-		t.Errorf("unexpected multitenancy tenants cache misses; got %d; want 0", got)
-	}
-
-	if got := vmselect.GetIntMetric(t, `vm_cache_entries{type="multitenancy/tenants"}`); got != 0 {
-		t.Errorf("unexpected multitenancy tenants cache entries; got %d; want 0", got)
-	}
-
-	// verify that tenant in path has priority over tenant specified in headers
-
-	// /api/v1/import/prometheus
-
-	tenantInHeader := make(http.Header)
-	tenantInHeader.Set("AccountID", "42")
-	tenantInPath := "112"
-	vminsert.PrometheusAPIV1ImportPrometheus(t, samples, apptest.QueryOpts{
-		// tenants in header and path clash - path should have higher priority on ingestion
-		Headers: tenantInHeader,
-		Tenant:  "112",
-	})
-	vmstorage.ForceFlush(t)
-
-	want = apptest.NewPrometheusAPIV1QueryResponse(t,
-		`{"data":
-       {"result":[
-          {"metric":{"__name__":"foo_bar"},"value":[1652169900,"3"]}
-                 ]
-       }
-     }`,
-	)
-	got = vmselect.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{
-		// tenants in header and path clash - path should have higher priority on ingestion
-		Headers: multitenant,
-		Tenant:  tenantInPath,
-		Time:    instantCT,
-	})
-	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
-		t.Errorf("unexpected response (-want, +got):\n%s", diff)
-	}
-}

apptest/tests/multitenant_test.go

@@ -192,7 +192,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	}
 
 	// Delete series from specific tenant
-	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
+	vmselect.APIV1AdminTSDBDeleteSeries(t, "foo_bar", apptest.QueryOpts{
 		Tenant: "5:15",
 	})
 	wantSR = apptest.NewPrometheusAPIV1SeriesResponse(t,
@@ -215,7 +215,7 @@ func TestClusterMultiTenantSelect(t *testing.T) {
 	}
 
 	// Delete series for multitenant with tenant filter
-	vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
+	vmselect.APIV1AdminTSDBDeleteSeries(t, `foo_bar{vm_account_id="1"}`, apptest.QueryOpts{
 		Tenant: "multitenant",
 	})
 

apptest/tests/vmagent_remotewrite_test.go

@@ -30,12 +30,13 @@ func TestSingleVMAgentReloadConfigs(t *testing.T) {
 	relabelFilePath := fmt.Sprintf("%s/%s", t.TempDir(), "relabel_config.yaml")
 	fs.MustWriteSync(relabelFilePath, []byte(relabelingRules))
 
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent", []string{
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		`-remoteWrite.forcePromProto=true`,
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
 		fmt.Sprintf(`-remoteWrite.url=http://%s/api/v1/write`, vmsingle.HTTPAddr()),
 		fmt.Sprintf(`-remoteWrite.urlRelabelConfig=%s`, relabelFilePath),
-	})
+	}, ``)
 
 	checkResponse := func(query, expResponse string) {
 		t.Helper()
@@ -131,11 +132,12 @@ func testSingleVMAgentRemoteWrite(t *testing.T, forcePromProto bool) {
 
 	vmsingle := tc.MustStartDefaultVmsingle()
 
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent", []string{
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		fmt.Sprintf(`-remoteWrite.forcePromProto=%v`, forcePromProto),
 		fmt.Sprintf(`-remoteWrite.url=http://%s/api/v1/write`, vmsingle.HTTPAddr()),
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
-	})
+	}, ``)
 
 	vmagent.APIV1ImportPrometheus(t, []string{
 		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
@@ -179,11 +181,12 @@ func TestSingleVMAgentUnsupportedMediaTypeDropIfSnappy(t *testing.T) {
 	}))
 	defer remoteWriteSrv.Close()
 
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent", []string{
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		`-remoteWrite.forcePromProto=true`,
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
-	})
+	}, ``)
 
 	vmagent.APIV1ImportPrometheusNoWaitFlush(t, []string{
 		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
@@ -242,10 +245,11 @@ func TestSingleVMAgentDowngradeRemoteWriteProtocol(t *testing.T) {
 	}))
 	defer remoteWriteSrv.Close()
 
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent", []string{
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
-	})
+	}, ``)
 
 	// Send request encoded with `zstd`; it fails, gets repacked as `snappy`, and retries successfully.
 	vmagent.APIV1ImportPrometheus(t, []string{
@@ -290,7 +294,8 @@ func TestSingleVMAgentDropOnOverload(t *testing.T) {
 	}))
 	defer remoteWriteSrv2.Close()
 
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent", []string{
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv2.URL),
 		"-remoteWrite.disableOnDiskQueue=true",
@@ -306,7 +311,7 @@ func TestSingleVMAgentDropOnOverload(t *testing.T) {
 		// It improves the test stability on resource-constrained runners.
 		// Should be bigger than retries * period
 		"-remoteWrite.retryMinInterval=3s",
-	})
+	}, ``)
 
 	const (
 		retries = 20
@@ -392,12 +397,13 @@ func TestSingleVMAgentCardinalityLimiter(t *testing.T) {
 	defer remoteWriteSrv.Close()
 
 	// Verify hourly limit is applied
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent-hourly", []string{
+	vmagent := tc.MustStartVmagent("vmagent-hourly", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		"-remoteWrite.maxRowsPerBlock=1",
 		"-remoteWrite.maxHourlySeries=1",
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-hourly",
-	})
+	}, ``)
 
 	vmagent.APIV1ImportPrometheus(t, []string{
 		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
@@ -426,12 +432,13 @@ func TestSingleVMAgentCardinalityLimiter(t *testing.T) {
 	)
 
 	// Daily limits
-	vmagent2 := tc.MustStartDefaultRWVmagent("vmagent-daily", []string{
+	vmagent2 := tc.MustStartVmagent("vmagent-daily", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		"-remoteWrite.maxRowsPerBlock=1",
 		"-remoteWrite.maxDailySeries=1",
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-daily",
-	})
+	}, ``)
 
 	vmagent2.APIV1ImportPrometheus(t, []string{
 		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
@@ -460,13 +467,14 @@ func TestSingleVMAgentCardinalityLimiter(t *testing.T) {
 	)
 
 	// test running with unlimited tracker
-	vmagent3 := tc.MustStartDefaultRWVmagent("vmagent-unlimited", []string{
+	vmagent3 := tc.MustStartVmagent("vmagent-unlimited", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		"-remoteWrite.maxRowsPerBlock=10",
 		"-remoteWrite.maxDailySeries=-1",
 		"-remoteWrite.maxHourlySeries=-1",
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-unlimited",
-	})
+	}, ``)
 
 	metrics := make([]string, 0, 100)
 	for i := range 100 {
@@ -512,12 +520,13 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) {
 
 	sut := tc.MustStartDefaultCluster()
 
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent", []string{
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
 		`-remoteWrite.forcePromProto=true`,
 		`-enableMultitenantHandlers=true`,
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
 		fmt.Sprintf(`-remoteWrite.url=http://%s/insert/multitenant/prometheus/api/v1/write`, sut.Vminsert.HTTPAddr()),
-	})
+	}, ``)
 
 	prometheusRemoteWriteDataSet := prompb.WriteRequest{
 		Metadata: []prompb.MetricMetadata{
@@ -575,69 +584,3 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) {
 	})
 
 }
-
-// See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy
-func TestSingleVMAgentMultitenancy(t *testing.T) {
-	tc := apptest.NewTestCase(t)
-	defer tc.Stop()
-
-	remoteWriteSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusNoContent)
-	}))
-	defer remoteWriteSrv.Close()
-
-	vmagent := tc.MustStartDefaultRWVmagent("vmagent-multitenancy", []string{
-		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
-		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-multitenancy",
-		"-enableMultitenantHandlers",
-		"-enableMultitenancyViaHeaders",
-	})
-
-	vmagent.APIV1ImportPrometheus(t, []string{
-		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
-	}, apptest.QueryOpts{Tenant: "2"})
-	v := vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="2",projectID="0"}`)
-	if v != 1 {
-		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=2")
-	}
-
-	vmagent.APIV1ImportPrometheus(t, []string{
-		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
-	}, apptest.QueryOpts{Tenant: "2:2"})
-	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="2",projectID="2"}`)
-	if v != 1 {
-		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=2, projectID=2")
-	}
-
-	headers := make(http.Header)
-	headers.Set("AccountID", "3")
-	vmagent.APIV1ImportPrometheus(t, []string{
-		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
-	}, apptest.QueryOpts{Headers: headers})
-	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="3",projectID="0"}`)
-	if v != 1 {
-		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=3, projectID=0")
-	}
-
-	headers.Set("AccountID", "3")
-	headers.Set("ProjectID", "3")
-	vmagent.APIV1ImportPrometheus(t, []string{
-		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
-	}, apptest.QueryOpts{Headers: headers})
-	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="3",projectID="3"}`)
-	if v != 1 {
-		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=3, projectID=3")
-	}
-
-	// tenants in header and path clash - path should have higher priority on ingestion
-	opts := apptest.QueryOpts{Headers: make(http.Header)}
-	opts.Headers.Set("AccountID", "4")
-	opts.Tenant = "5"
-	vmagent.APIV1ImportPrometheus(t, []string{
-		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
-	}, opts)
-	v = vmagent.GetIntMetric(t, `vmagent_tenant_inserted_rows_total{type="prometheus",accountID="5",projectID="0"}`)
-	if v != 1 {
-		t.Fatalf("expected vmagent_tenant_inserted_rows_total to have value 1 for accountID=5, projectID=0")
-	}
-}

apptest/vmagent.go

@@ -16,63 +16,45 @@ import (
 	"github.com/golang/snappy"
 )
 
-// StartVmagent starts the latest version of vmagent.
-//
-// The path to the binary can be provided via VMAGENT_PATH environment
-// variable. If the variable is not set, ../../bin/vmagent-race will be
-// used.
+// Vmagent holds the state of a vmagent app and provides vmagent-specific functions
+type Vmagent struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr           string
+	apiV1ImportPrometheusURL string
+}
+
+// StartVmagent starts an instance of vmagent with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
 func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfigFilePath string, output io.Writer) (*Vmagent, error) {
-	binary := os.Getenv("VMAGENT_PATH")
-	if binary == "" {
-		binary = "../../bin/vmagent-race"
+	extractREs := []*regexp.Regexp{
+		httpListenAddrRE,
 	}
-	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
+
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmagent-race", flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-promscrape.config":       promScrapeConfigFilePath,
 			"-remoteWrite.tmpDataPath": fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
 		},
-		extractREs: []*regexp.Regexp{
-			httpListenAddrRE,
-		},
-		output: output,
+		extractREs: extractREs,
+		output:     output,
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	return newVmagent(app, cli, vmagentRuntimeValues{
-		httpListenAddr: stderrExtracts[0],
-	}), nil
-}
-
-type vmagentRuntimeValues struct {
-	httpListenAddr string
-}
-
-func newVmagent(app *app, cli *Client, rt vmagentRuntimeValues) *Vmagent {
 	return &Vmagent{
-		app:            app,
-		cli:            cli,
-		metricsClient:  newMetricsClient(cli, rt.httpListenAddr),
-		httpListenAddr: rt.httpListenAddr,
-	}
-}
-
-// Vmagent holds the state of a vmagent app and provides vmagent-specific
-// functions.
-type Vmagent struct {
-	*app
-	*metricsClient
-
-	cli            *Client
-	httpListenAddr string
-}
-
-// HTTPAddr returns the address at which the vmagent process is listening
-// for http connections.
-func (app *Vmagent) HTTPAddr() string {
-	return app.httpListenAddr
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
+		httpListenAddr:           stderrExtracts[0],
+		apiV1ImportPrometheusURL: fmt.Sprintf("http://%s/api/v1/import/prometheus", stderrExtracts[0]),
+	}, nil
 }
 
 // APIV1ImportPrometheus is a test helper function that inserts a
@@ -104,33 +86,12 @@ func (app *Vmagent) APIV1ImportPrometheusNoWaitFlush(t *testing.T, records []str
 	data := []byte(strings.Join(records, "\n"))
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "text/plain")
-	url := getVMAgentInsertPath(app.httpListenAddr, "prometheus/api/v1/import/prometheus", opts)
-	_, statusCode := app.cli.Post(t, url, data, headers)
+	_, statusCode := app.cli.Post(t, app.apiV1ImportPrometheusURL, data, headers)
 	if statusCode != http.StatusNoContent {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 	}
 }
 
-// getVMAgentInsertPath returns URL path for writes.
-// If tenant is set in QueryOpts, it will return cluster-like path for ingestion.
-// If tenant is empty, it will return single-node (no tenants) path.
-func getVMAgentInsertPath(addr, suffix string, o QueryOpts) string {
-	if o.Tenant != "" {
-		// QueryOpts.Tenant has priority over headers
-		return fmt.Sprintf("http://%s/insert/%s/%s", addr, o.Tenant, suffix)
-	}
-
-	h := o.getHeaders()
-	if h.Get("AccountID") != "" || h.Get("ProjectID") != "" {
-		// vmagent supports tenantID in HTTP headers only if -enableMultitenantHandlers and -enableMultitenancyViaHeaders are set
-		// see https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy
-		return fmt.Sprintf("http://%s/insert/%s", addr, suffix)
-	}
-
-	// tenant is missing in QueryOpts and in HTTP headers. Use single-node (no tenants) path
-	return fmt.Sprintf("http://%s/%s", addr, suffix)
-}
-
 // RemoteWriteRequestsRetriesCountTotal sums up the total retries for remote write requests.
 func (app *Vmagent) RemoteWriteRequestsRetriesCountTotal(t *testing.T) int {
 	total := 0.0
@@ -207,7 +168,10 @@ func (app *Vmagent) ReloadRelabelConfigs(t *testing.T) {
 func (app *Vmagent) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
 	t.Helper()
 
-	url := getVMAgentInsertPath(app.httpListenAddr, "prometheus/api/v1/write", opts)
+	url := fmt.Sprintf("http://%s/prometheus/api/v1/write", app.httpListenAddr)
+	if opts.Tenant != "" {
+		url = fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.Tenant)
+	}
 	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
 	recordsCount := len(wr.Timeseries)
 	if prommetadata.IsEnabled() {
@@ -223,6 +187,12 @@ func (app *Vmagent) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, o
 	})
 }
 
+// HTTPAddr returns the address at which the vmagent process is listening
+// for http connections.
+func (app *Vmagent) HTTPAddr() string {
+	return app.httpListenAddr
+}
+
 // sendBlocking sends the data to vmstorage by executing `send` function and
 // waits until the data is actually sent.
 //

apptest/vmauth.go

@@ -1,8 +1,8 @@
 package apptest
 
 import (
+	"fmt"
 	"io"
-	"os"
 	"regexp"
 	"syscall"
 	"testing"
@@ -11,63 +11,49 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 )
 
-// StartVmauth starts the latest version of vmauth.
-//
-// The path to the binary can be provided via VMAUTH_PATH environment
-// variable. If the variable is not set, ../../bin/vmauth-race will be
-// used.
-func StartVmauth(instance string, flags []string, cli *Client, configFilePath string, output io.Writer) (*Vmauth, error) {
-	binary := os.Getenv("VMAUTH_PATH")
-	if binary == "" {
-		binary = "../../bin/vmauth-race"
-	}
-	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
-		defaultFlags: map[string]string{
-			"-httpListenAddr": "127.0.0.1:0",
-			"-auth.config":    configFilePath,
-		},
-		extractREs: []*regexp.Regexp{
-			vmauthHttpListenAddrRE,
-		},
-		output: output,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return newVmauth(app, cli, configFilePath, vmauthRuntimeValues{
-		httpListenAddr: stderrExtracts[0],
-	}), nil
-}
-
-type vmauthRuntimeValues struct {
-	httpListenAddr string
-}
-
-func newVmauth(app *app, cli *Client, configFilePath string, rt vmauthRuntimeValues) *Vmauth {
-	return &Vmauth{
-		app:            app,
-		metricsClient:  newMetricsClient(cli, rt.httpListenAddr),
-		httpListenAddr: rt.httpListenAddr,
-		configFilePath: configFilePath,
-		cli:            cli,
-	}
-}
+var httpBuilitinListenAddrRE = regexp.MustCompile(`pprof handlers are exposed at http://(.*:\d{1,5})/debug/pprof/`)
 
 // Vmauth holds the state of a vmauth app and provides vmauth-specific
 // functions.
 type Vmauth struct {
 	*app
-	*metricsClient
+	*ServesMetrics
 
-	cli            *Client
 	httpListenAddr string
 	configFilePath string
+	cli            *Client
 }
 
-// GetHTTPListenAddr returns listen http addr
-func (app *Vmauth) GetHTTPListenAddr() string {
-	return app.httpListenAddr
+// StartVmauth starts an instance of vmauth with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
+func StartVmauth(instance string, flags []string, cli *Client, configFilePath string, output io.Writer) (*Vmauth, error) {
+	extractREs := []*regexp.Regexp{
+		httpBuilitinListenAddrRE,
+	}
+
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmauth-race", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr": "127.0.0.1:0",
+			"-auth.config":    configFilePath,
+		},
+		extractREs: extractREs,
+		output:     output,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vmauth{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
+		httpListenAddr: stderrExtracts[0],
+		configFilePath: configFilePath,
+		cli:            cli,
+	}, nil
 }
 
 // UpdateConfiguration updates the vmauth configuration file with the provided YAML content,
@@ -97,3 +83,8 @@ func (app *Vmauth) UpdateConfiguration(t *testing.T, configFileYAML string) {
 
 	t.Fatalf("config were not reloaded after SIGHUP signal; previous total: %d, current total: %d", prevTotal, currTotal)
 }
+
+// GetHTTPListenAddr returns listen http addr
+func (app *Vmauth) GetHTTPListenAddr() string {
+	return app.httpListenAddr
+}

apptest/vmbackup.go

@@ -1,26 +1,15 @@
 package apptest
 
-import (
-	"io"
-	"os"
-)
+import "io"
 
-// StartVmbackup starts the latest version of vmbackup with the given flags and
-// waits until it exits.
-//
-// The path to the binary can be provided via VMBACKUP_PATH environment
-// variable. If the variable is not set, ../../bin/vmbackup-race will be
-// used.
+// StartVmbackup starts an instance of vmbackup with the given flags and waits
+// until it exits.
 func StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string, output io.Writer) error {
-	binary := os.Getenv("VMBACKUP_PATH")
-	if binary == "" {
-		binary = "../../bin/vmbackup-race"
-	}
 	flags := []string{
 		"-storageDataPath=" + storageDataPath,
 		"-snapshot.createURL=" + snapshotCreateURL,
 		"-dst=" + dst,
 	}
-	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
+	_, _, err := startApp(instance, "../../bin/vmbackup-race", flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vmctl.go

@@ -1,23 +1,9 @@
 package apptest
 
-import (
-	"io"
-	"os"
-)
+import "io"
 
 // StartVmctl starts an instance of vmctl cli with the given flags
-
-// StartVmctl starts the latest version of vmctl with the given flags and
-// waits until it exits.
-//
-// The path to the binary can be provided via VMCTL_PATH environment
-// variable. If the variable is not set, ../../bin/vmctl-race will be
-// used.
 func StartVmctl(instance string, flags []string, output io.Writer) error {
-	binary := os.Getenv("VMCTL_PATH")
-	if binary == "" {
-		binary = "../../bin/vmctl-race"
-	}
-	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
+	_, _, err := startApp(instance, "../../bin/vmctl-race", flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vminsert.go

@@ -3,13 +3,32 @@ package apptest
 import (
 	"fmt"
 	"io"
-	"os"
+	"net/http"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
+// Vminsert holds the state of a vminsert app and provides vminsert-specific
+// functions.
+type Vminsert struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr          string
+	clusternativeListenAddr string
+	graphiteListenAddr      string
+	openTSDBListenAddr      string
+
+	cli *Client
+}
+
 // storageNodes returns the storage node addresses passed to vminsert via
 // -storageNode command line flag.
 func storageNodes(flags []string) []string {
@@ -21,11 +40,9 @@ func storageNodes(flags []string) []string {
 	return nil
 }
 
-// StartVminsert starts the latest version of vminsert.
-//
-// The path to the binary can be provided via VMINSERT_PATH environment
-// variable. If the variable is not set, ../../bin/vminsert-race will be
-// used.
+// StartVminsert starts an instance of vminsert with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
 func StartVminsert(instance string, flags []string, cli *Client, output io.Writer) (*Vminsert, error) {
 	extractREs := []*regexp.Regexp{
 		httpListenAddrRE,
@@ -40,15 +57,11 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		extractREs = append(extractREs, regexp.MustCompile(logRecord))
 	}
 
-	binary := os.Getenv("VMINSERT_PATH")
-	if binary == "" {
-		binary = "../../bin/vminsert-race"
-	}
-	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, "../../bin/vminsert-race", flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":                              "127.0.0.1:0",
 			"-clusternativeListenAddr":                     "127.0.0.1:0",
-			"-graphiteListenAddr":                          "127.0.0.1:0",
+			"-graphiteListenAddr":                          ":0",
 			"-opentsdbListenAddr":                          "127.0.0.1:0",
 			"-clusternative.vminsertConnsShutdownDuration": "1ms",
 		},
@@ -59,56 +72,18 @@ func StartVminsert(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
-	return newVminsert(app, cli, vminsertRuntimeValues{
+	return &Vminsert{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
 		graphiteListenAddr:      stderrExtracts[2],
 		openTSDBListenAddr:      stderrExtracts[3],
-	}), nil
-}
-
-type vminsertRuntimeValues struct {
-	httpListenAddr          string
-	clusternativeListenAddr string
-	graphiteListenAddr      string
-	openTSDBListenAddr      string
-}
-
-func newVminsert(app *app, cli *Client, rt vminsertRuntimeValues) *Vminsert {
-	metricsClient := newMetricsClient(cli, rt.httpListenAddr)
-	vminsertClient := &vminsertClient{
-		vminsertCli: cli,
-		url: func(op, path string, opts QueryOpts) string {
-			return getClusterPath(rt.httpListenAddr, op, path, opts)
-		},
-		openTSDBURL: func(op, path string, opts QueryOpts) string {
-			return getClusterPath(rt.openTSDBListenAddr, op, path, opts)
-		},
-		graphiteListenAddr: rt.graphiteListenAddr,
-		sendBlocking: func(t *testing.T, numRecordsToSend int, send func()) {
-			t.Helper()
-			sendBlocking(t, metricsClient, numRecordsToSend, send)
-		},
-	}
-
-	return &Vminsert{
-		app:                     app,
-		metricsClient:           metricsClient,
-		vminsertClient:          vminsertClient,
-		httpListenAddr:          rt.httpListenAddr,
-		clusternativeListenAddr: rt.clusternativeListenAddr,
-	}
-}
-
-// Vminsert holds the state of a vminsert app and provides vminsert-specific
-// functions.
-type Vminsert struct {
-	*app
-	*metricsClient
-	*vminsertClient
-
-	httpListenAddr          string
-	clusternativeListenAddr string
+		cli:                     cli,
+	}, nil
 }
 
 // ClusternativeListenAddr returns the address at which the vminsert process is
@@ -123,6 +98,213 @@ func (app *Vminsert) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
+// InfluxWrite is a test helper function that inserts a
+// collection of records in Influx line format by sending a HTTP
+// POST request to /influx/write vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
+func (app *Vminsert) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/influx/write", app.httpListenAddr, opts.getTenant())
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	app.sendBlocking(t, len(records), func() {
+		_, statusCode := app.cli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// GraphiteWrite is a test helper function that sends a
+// collection of records to graphiteListenAddr port.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
+func (app *Vminsert) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
+	t.Helper()
+	app.cli.Write(t, app.graphiteListenAddr, records)
+}
+
+// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
+// of records in CSV format for the given tenant by sending an HTTP POST
+// request to prometheus/api/v1/import/csv vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (app *Vminsert) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/csv", app.httpListenAddr, opts.getTenant())
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	app.sendBlocking(t, len(records), func() {
+		_, statusCode := app.cli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
+// of records in Native format for the given tenant by sending an HTTP POST
+// request to prometheus/api/v1/import/native vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (app *Vminsert) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/native", app.httpListenAddr, opts.getTenant())
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	app.sendBlocking(t, 1, func() {
+		_, statusCode := app.cli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// OpenTSDBAPIPut is a test helper function that inserts a collection of
+// records in OpenTSDB format for the given tenant by sending an HTTP POST
+// request to /opentsdb/api/put vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/cluster-victoriametrics/#url-format
+func (app *Vminsert) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/opentsdb/api/put", app.openTSDBListenAddr, opts.getTenant())
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte("[" + strings.Join(records, ",") + "]")
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/json")
+	app.sendBlocking(t, len(records), func() {
+		_, statusCode := app.cli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1Write is a test helper function that inserts a
+// collection of records in Prometheus remote-write format by sending a HTTP
+// POST request to /prometheus/api/v1/write vminsert endpoint.
+func (app *Vminsert) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/write", app.httpListenAddr, opts.getTenant())
+	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
+	recordsCount := len(wr.Timeseries)
+	if prommetadata.IsEnabled() {
+		recordsCount += len(wr.Metadata)
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/x-protobuf")
+	app.sendBlocking(t, recordsCount, func() {
+		_, statusCode := app.cli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// collection of records in Prometheus text exposition format for the given
+// tenant by sending a HTTP POST request to
+// /prometheus/api/v1/import/prometheus vminsert endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
+func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/prometheus/api/v1/import/prometheus", app.httpListenAddr, opts.getTenant())
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	var recordsCount int
+	var metadataRecords int
+	uniqueMetadataMetricNames := make(map[string]struct{})
+	for _, record := range records {
+		// metric metadata has the following format:
+		//# HELP importprometheus_series
+		//# TYPE importprometheus_series
+		// it results into single metadata record
+		if strings.HasPrefix(record, "# ") {
+			metadataItems := strings.Split(record, " ")
+			if len(metadataItems) < 2 {
+				t.Fatalf("BUG: unexpected metadata format=%q", record)
+			}
+			metricName := metadataItems[2]
+			if _, ok := uniqueMetadataMetricNames[metricName]; ok {
+				continue
+			}
+			uniqueMetadataMetricNames[metricName] = struct{}{}
+			metadataRecords++
+			continue
+		}
+		recordsCount++
+	}
+	if prommetadata.IsEnabled() {
+		recordsCount += metadataRecords
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	app.sendBlocking(t, recordsCount, func() {
+		_, statusCode := app.cli.Post(t, url, data, headers)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// ZabbixConnectorHistory is a test helper function that inserts a
+// collection of records in zabbixconnector  format by sending a HTTP
+// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
+func (app *Vminsert) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/zabbixconnector/api/v1/history", app.httpListenAddr, opts.getTenant())
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/json")
+	app.sendBlocking(t, len(records), func() {
+		_, statusCode := app.cli.Post(t, url, data, headers)
+		if statusCode != http.StatusOK {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+		}
+	})
+
+}
+
 // String returns the string representation of the vminsert app state.
 func (app *Vminsert) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)
@@ -138,10 +320,13 @@ func (app *Vminsert) String() string {
 // Waiting is implemented a retrieving the value of `vm_rpc_rows_sent_total`
 // metric and checking whether it is equal or greater than the wanted value.
 // If it is, then the data has been sent to vmstorage.
-func sendBlocking(t *testing.T, c *metricsClient, numRecordsToSend int, send func()) {
+//
+// Unreliable if the records are inserted concurrently.
+// TODO(rtm0): Put sending and waiting into a critical section to make reliable?
+func (app *Vminsert) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
 	t.Helper()
 
-	wantRowsSentCount := c.rpcRowsSentTotal(t) + numRecordsToSend
+	wantRowsSentCount := app.rpcRowsSentTotal(t) + numRecordsToSend
 
 	send()
 
@@ -150,7 +335,7 @@ func sendBlocking(t *testing.T, c *metricsClient, numRecordsToSend int, send fun
 		period  = 100 * time.Millisecond
 	)
 	for range retries {
-		d := c.rpcRowsSentTotal(t)
+		d := app.rpcRowsSentTotal(t)
 		if d >= wantRowsSentCount {
 			return
 		}
@@ -158,3 +343,14 @@ func sendBlocking(t *testing.T, c *metricsClient, numRecordsToSend int, send fun
 	}
 	t.Fatalf("timed out while waiting for inserted rows to be sent to vmstorage")
 }
+
+// rpcRowsSentTotal retrieves the values of all vminsert
+// `vm_rpc_rows_sent_total` metrics (there will be one for each vmstorage) and
+// returns their integer sum.
+func (app *Vminsert) rpcRowsSentTotal(t *testing.T) int {
+	total := 0.0
+	for _, v := range app.GetMetricsByPrefix(t, "vm_rpc_rows_sent_total") {
+		total += v
+	}
+	return int(total)
+}

apptest/vmrestore.go

@@ -1,25 +1,14 @@
 package apptest
 
-import (
-	"io"
-	"os"
-)
+import "io"
 
-// StartVmrestore starts the latest version of vmrestore with the given flags
-// and waits until it exits.
-//
-// The path to the binary can be provided via VMRESTORE_PATH environment
-// variable. If the variable is not set, ../../bin/vmrestore-race will be
-// used.
+// StartVmrestore starts an instance of vmrestore with the given flags and waits
+// until it exits.
 func StartVmrestore(instance, src, storageDataPath string, output io.Writer) error {
-	binary := os.Getenv("VMRESTORE_PATH")
-	if binary == "" {
-		binary = "../../bin/vmrestore-race"
-	}
 	flags := []string{
 		"-src=" + src,
 		"-storageDataPath=" + storageDataPath,
 	}
-	_, _, err := startApp(instance, binary, flags, &appOptions{wait: true, output: output})
+	_, _, err := startApp(instance, "../../bin/vmrestore-race", flags, &appOptions{wait: true, output: output})
 	return err
 }

apptest/vmselect.go

@@ -1,23 +1,31 @@
 package apptest
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
-	"os"
+	"net/http"
 	"regexp"
+	"strconv"
+	"testing"
 )
 
-// StartVmselect starts the latest version of vmselect.
-//
-// The path to the binary can be provided via VMSELECT_PATH environment
-// variable. If the variable is not set, ../../bin/vmselect-race will be
-// used.
+// Vmselect holds the state of a vmselect app and provides vmselect-specific
+// functions.
+type Vmselect struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr          string
+	clusternativeListenAddr string
+	cli                     *Client
+}
+
+// StartVmselect starts an instance of vmselect with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
 func StartVmselect(instance string, flags []string, cli *Client, output io.Writer) (*Vmselect, error) {
-	binary := os.Getenv("VMSELECT_PATH")
-	if binary == "" {
-		binary = "../../bin/vmselect-race"
-	}
-	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmselect-race", flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-httpListenAddr":          "127.0.0.1:0",
 			"-clusternativeListenAddr": "127.0.0.1:0",
@@ -32,43 +40,16 @@ func StartVmselect(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
-	return newVmselect(app, cli, vmselectRuntimeValues{
+	return &Vmselect{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
 		httpListenAddr:          stderrExtracts[0],
 		clusternativeListenAddr: stderrExtracts[1],
-	}), nil
-}
-
-type vmselectRuntimeValues struct {
-	httpListenAddr          string
-	clusternativeListenAddr string
-}
-
-func newVmselect(app *app, cli *Client, rt vmselectRuntimeValues) *Vmselect {
-	return &Vmselect{
-		app:           app,
-		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
-		vmselectClient: &vmselectClient{
-			vmselectCli: cli,
-			url: func(op, path string, opts QueryOpts) string {
-				return getClusterPath(rt.httpListenAddr, op, path, opts)
-			},
-			metricNamesStatsResetURL: fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", rt.httpListenAddr),
-			tenantsURL:               fmt.Sprintf("http://%s/admin/tenants", rt.httpListenAddr),
-		},
-		httpListenAddr:          rt.httpListenAddr,
-		clusternativeListenAddr: rt.clusternativeListenAddr,
-	}
-}
-
-// Vmselect holds the state of a vmselect app and provides vmselect-specific
-// functions.
-type Vmselect struct {
-	*app
-	*metricsClient
-	*vmselectClient
-
-	httpListenAddr          string
-	clusternativeListenAddr string
+		cli:                     cli,
+	}, nil
 }
 
 // ClusternativeListenAddr returns the address at which the vmselect process is
@@ -83,6 +64,298 @@ func (app *Vmselect) HTTPAddr() string {
 	return app.httpListenAddr
 }
 
+// PrometheusAPIV1Export is a test helper function that performs the export of
+// raw samples in JSON line format by sending a HTTP POST request to
+// /prometheus/api/v1/export vmselect endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
+func (app *Vmselect) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+	res, _ := app.cli.PostForm(t, exportURL, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1ExportNative is a test helper function that performs the export of
+// raw samples in native binary format by sending an HTTP POST request to
+// /prometheus/api/v1/export/native vmselect endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
+func (app *Vmselect) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
+	t.Helper()
+
+	exportURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/export/native", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+	res, _ := app.cli.PostForm(t, exportURL, values, opts.Headers)
+	return []byte(res)
+}
+
+// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
+// instant query by sending a HTTP POST request to /prometheus/api/v1/query
+// vmselect endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
+func (app *Vmselect) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("query", query)
+
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1QueryRange is a test helper function that performs
+// PromQL/MetricsQL range query by sending a HTTP POST request to
+// /prometheus/api/v1/query_range vmselect endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
+func (app *Vmselect) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/query_range", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("query", query)
+
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
+// and returns the list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (app *Vmselect) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesResponse(t, res)
+}
+
+// PrometheusAPIV1SeriesCount sends a query to a /prometheus/api/v1/series/count endpoint
+// and returns the total number of time series.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (app *Vmselect) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/series/count", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+
+	res, _ := app.cli.PostForm(t, seriesURL, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesCountResponse(t, res)
+}
+
+// PrometheusAPIV1Labels sends a query to a /prometheus/api/v1/labels endpoint
+// and returns the label names list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
+func (app *Vmselect) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/labels", app.httpListenAddr, opts.getTenant())
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1LabelsResponse(t, res)
+}
+
+// PrometheusAPIV1LabelValues sends a query to a /prometheus/api/v1/label/.../values endpoint
+// and returns the label names list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
+func (app *Vmselect) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/label/%s/values", app.httpListenAddr, opts.getTenant(), labelName)
+
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1LabelValuesResponse(t, res)
+}
+
+// PrometheusAPIV1Metadata sends a query to a /prometheus/api/v1/metadata endpoint
+// and returns the results.
+func (app *Vmselect) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("metric", metric)
+	values.Add("limit", strconv.Itoa(limit))
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/metadata", app.httpListenAddr, opts.getTenant())
+
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1Metadata(t, res)
+}
+
+// APIV1AdminTSDBDeleteSeries deletes the series that match the query by sending
+// a request to /api/v1/admin/tsdb/delete_series.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
+func (app *Vmselect) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/delete/%s/prometheus/api/v1/admin/tsdb/delete_series", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// MetricNamesStats sends a query to a /select/tenant/prometheus/api/v1/status/metric_names_stats endpoint
+// and returns the statistics response for given params.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (app *Vmselect) MetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("limit", limit)
+	values.Add("le", le)
+	values.Add("match_pattern", matchPattern)
+	queryURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/status/metric_names_stats", app.httpListenAddr, opts.getTenant())
+
+	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+	var resp MetricNamesStatsResponse
+	if err := json.Unmarshal([]byte(res), &resp); err != nil {
+		t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", res, err)
+	}
+	return resp
+}
+
+// MetricNamesStatsReset sends a query to a /admin/api/v1/status/metric_names_stats/reset endpoint
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (app *Vmselect) MetricNamesStatsReset(t *testing.T, opts QueryOpts) {
+	t.Helper()
+
+	values := opts.asURLValues()
+	queryURL := fmt.Sprintf("http://%s/admin/api/v1/admin/status/metric_names_stats/reset", app.httpListenAddr)
+
+	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
+// //
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
+func (app *Vmselect) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/select/%s/prometheus/api/v1/status/tsdb", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	addNonEmpty := func(name, value string) {
+		if len(value) == 0 {
+			return
+		}
+		values.Add(name, value)
+	}
+	addNonEmpty("match[]", matchQuery)
+	addNonEmpty("topN", topN)
+	addNonEmpty("date", date)
+
+	res, statusCode := app.cli.PostForm(t, seriesURL, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var status TSDBStatusResponse
+	if err := json.Unmarshal([]byte(res), &status); err != nil {
+		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
+	}
+	status.Sort()
+	return status
+}
+
+// GraphiteMetricsIndex sends a query to a /graphite/metrics/index.json
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+func (app *Vmselect) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) GraphiteMetricsIndexResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/select/%s/graphite/metrics/index.json", app.httpListenAddr, opts.getTenant())
+	res, statusCode := app.cli.Get(t, seriesURL, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var index GraphiteMetricsIndexResponse
+	if err := json.Unmarshal([]byte(res), &index); err != nil {
+		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
+	}
+	return index
+}
+
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a HTTP POST request to
+// /graphite/tags/tagSeries vmsingle endpoint.
+func (app *Vmselect) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagSeries", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	values.Add("path", record)
+
+	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+func (app *Vmselect) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/select/%s/graphite/tags/tagMultiSeries", app.httpListenAddr, opts.getTenant())
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+
+	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+// APIV1AdminTenants sends a query to a /admin/tenants endpoint
+func (app *Vmselect) APIV1AdminTenants(t *testing.T) *AdminTenantsResponse {
+	t.Helper()
+
+	tenantsURL := fmt.Sprintf("http://%s/admin/tenants", app.httpListenAddr)
+	res, statusCode := app.cli.Get(t, tenantsURL, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var tenants *AdminTenantsResponse
+	if err := json.Unmarshal([]byte(res), tenants); err != nil {
+		t.Fatalf("could not unmarshal tenants response data:\n%s\n err: %v", res, err)
+	}
+
+	return tenants
+}
+
 // String returns the string representation of the vmselect app state.
 func (app *Vmselect) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)

apptest/vmsingle.go

@@ -1,29 +1,59 @@
 package apptest
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"regexp"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
+
+	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
-// StartVmsingle starts the latest version of vmsingle.
-//
-// The path to the binary can be provided via VMSINGLE_PATH environment
-// variable. If the variable is not set, ../../bin/victoria-metrics-race will be
-// used.
-func StartVmsingle(instance string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
-	binary := os.Getenv("VMSINGLE_PATH")
-	if binary == "" {
-		binary = "../../bin/victoria-metrics-race"
-	}
+// Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
+// functions.
+type Vmsingle struct {
+	*app
+	*ServesMetrics
+
+	storageDataPath string
+	httpListenAddr  string
+
+	// vmstorage URLs.
+	forceFlushURL string
+	forceMergeURL string
+
+	// vminsert URLs.
+	influxLineWriteURL                 string
+	graphiteWriteAddr                  string
+	openTSDBHTTPURL                    string
+	prometheusAPIV1ImportPrometheusURL string
+	prometheusAPIV1WriteURL            string
+
+	// vmselect URLs.
+	prometheusAPIV1ExportURL       string
+	prometheusAPIV1ExportNativeURL string
+	prometheusAPIV1QueryURL        string
+	prometheusAPIV1QueryRangeURL   string
+	prometheusAPIV1SeriesURL       string
+}
+
+// StartVmsingleAt starts an instance of vmsingle with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr).
+func StartVmsingleAt(instance, binary string, flags []string, cli *Client, output io.Writer) (*Vmsingle, error) {
 	app, stderrExtracts, err := startApp(instance, binary, flags, &appOptions{
 		defaultFlags: map[string]string{
 			"-storageDataPath":    fmt.Sprintf("%s/%s-%d", os.TempDir(), instance, time.Now().UnixNano()),
 			"-httpListenAddr":     "127.0.0.1:0",
-			"-graphiteListenAddr": "127.0.0.1:0",
+			"-graphiteListenAddr": ":0",
 			"-opentsdbListenAddr": "127.0.0.1:0",
 		},
 		extractREs: []*regexp.Regexp{
@@ -38,67 +68,595 @@ func StartVmsingle(instance string, flags []string, cli *Client, output io.Write
 		return nil, err
 	}
 
-	return newVmsingle(app, cli, vmsingleRuntimeValues{
-		storageDataPath:    stderrExtracts[0],
-		httpListenAddr:     stderrExtracts[1],
-		graphiteListenAddr: stderrExtracts[2],
-		openTSDBListenAddr: stderrExtracts[3],
-	}), nil
-}
-
-type vmsingleRuntimeValues struct {
-	storageDataPath    string
-	httpListenAddr     string
-	graphiteListenAddr string
-	openTSDBListenAddr string
-}
-
-func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
 	return &Vmsingle{
-		app:           app,
-		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
-		vmstorageClient: &vmstorageClient{
-			vmstorageCli:   cli,
-			httpListenAddr: rt.httpListenAddr,
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[1]),
+			cli:        cli,
 		},
-		vmselectClient: &vmselectClient{
-			vmselectCli: cli,
-			url: func(op, path string, opts QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
-			},
-			metricNamesStatsResetURL: fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", rt.httpListenAddr),
-			tenantsURL:               "vmsingle-does-not-serve-tenants",
-		},
-		vminsertClient: &vminsertClient{
-			vminsertCli: cli,
-			url: func(_, path string, _ QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
-			},
-			openTSDBURL: func(_, path string, _ QueryOpts) string {
-				return fmt.Sprintf("http://%s/%s", rt.openTSDBListenAddr, path)
-			},
-			graphiteListenAddr: rt.graphiteListenAddr,
-			sendBlocking: func(t *testing.T, _ int, send func()) {
-				t.Helper()
-				send()
-			},
-		},
-		storageDataPath: rt.storageDataPath,
-		httpListenAddr:  rt.httpListenAddr,
+		storageDataPath: stderrExtracts[0],
+		httpListenAddr:  stderrExtracts[1],
+
+		forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
+		forceMergeURL: fmt.Sprintf("http://%s/internal/force_merge", stderrExtracts[1]),
+
+		influxLineWriteURL:                 fmt.Sprintf("http://%s/influx/write", stderrExtracts[1]),
+		graphiteWriteAddr:                  stderrExtracts[2],
+		openTSDBHTTPURL:                    fmt.Sprintf("http://%s", stderrExtracts[3]),
+		prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
+		prometheusAPIV1WriteURL:            fmt.Sprintf("http://%s/prometheus/api/v1/write", stderrExtracts[1]),
+		prometheusAPIV1ExportURL:           fmt.Sprintf("http://%s/prometheus/api/v1/export", stderrExtracts[1]),
+		prometheusAPIV1ExportNativeURL:     fmt.Sprintf("http://%s/prometheus/api/v1/export/native", stderrExtracts[1]),
+		prometheusAPIV1QueryURL:            fmt.Sprintf("http://%s/prometheus/api/v1/query", stderrExtracts[1]),
+		prometheusAPIV1QueryRangeURL:       fmt.Sprintf("http://%s/prometheus/api/v1/query_range", stderrExtracts[1]),
+		prometheusAPIV1SeriesURL:           fmt.Sprintf("http://%s/prometheus/api/v1/series", stderrExtracts[1]),
+	}, nil
+}
+
+// ForceFlush is a test helper function that forces the flushing of inserted
+// data, so it becomes available for searching immediately.
+func (app *Vmsingle) ForceFlush(t *testing.T) {
+	t.Helper()
+
+	_, statusCode := app.cli.Get(t, app.forceFlushURL, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
 }
 
-// Vmsingle holds the state of a vmsingle app and provides vmsingle-specific
-// functions.
-type Vmsingle struct {
-	*app
-	*metricsClient
-	*vmstorageClient
-	*vmselectClient
-	*vminsertClient
+// ForceMerge is a test helper function that forces the merging of parts.
+func (app *Vmsingle) ForceMerge(t *testing.T) {
+	t.Helper()
 
-	storageDataPath string
-	httpListenAddr  string
+	_, statusCode := app.cli.Get(t, app.forceMergeURL, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// InfluxWrite is a test helper function that inserts a
+// collection of records in Influx line format by sending a HTTP
+// POST request to /influx/write vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#influxwrite
+func (app *Vmsingle) InfluxWrite(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	data := []byte(strings.Join(records, "\n"))
+
+	url := app.influxLineWriteURL
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	_, statusCode := app.cli.Post(t, url, data, headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+	}
+}
+
+// GraphiteWrite is a test helper function that sends a collection of records
+// to graphiteListenAddr port.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
+func (app *Vmsingle) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
+	t.Helper()
+	app.cli.Write(t, app.graphiteWriteAddr, records)
+}
+
+// PrometheusAPIV1ImportCSV is a test helper function that inserts a collection
+// of records in CSV format for the given tenant by sending an HTTP POST
+// request to /api/v1/import/csv vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-import-csv-data
+func (app *Vmsingle) PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/api/v1/import/csv", app.httpListenAddr)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	_, statusCode := app.cli.Post(t, url, data, headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+	}
+}
+
+// PrometheusAPIV1ImportNative is a test helper function that inserts a collection
+// of records in native format for the given tenant by sending an HTTP POST
+// request to /api/v1/import/native vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-native-format
+func (app *Vmsingle) PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/api/v1/import/native", app.httpListenAddr)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	_, statusCode := app.cli.Post(t, url, data, headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+	}
+}
+
+// OpenTSDBAPIPut is a test helper function that inserts a collection of
+// records in OpenTSDB format for the given tenant by sending an HTTP POST
+// request to /api/put vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http
+func (app *Vmsingle) OpenTSDBAPIPut(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	// add extra label
+	url := app.openTSDBHTTPURL + "/api/put"
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte("[" + strings.Join(records, ",") + "]")
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	_, statusCode := app.cli.Post(t, url, data, headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+	}
+}
+
+// PrometheusAPIV1Write is a test helper function that inserts a
+// collection of records in Prometheus remote-write format by sending a HTTP
+// POST request to /prometheus/api/v1/write vmsingle endpoint.
+func (app *Vmsingle) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts) {
+	t.Helper()
+
+	data := snappy.Encode(nil, wr.MarshalProtobuf(nil))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/x-protobuf")
+	_, statusCode := app.cli.Post(t, app.prometheusAPIV1WriteURL, data, headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+	}
+}
+
+// PrometheusAPIV1ImportPrometheus is a test helper function that inserts a
+// collection of records in Prometheus text exposition format by sending a HTTP
+// POST request to /prometheus/api/v1/import/prometheus vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1importprometheus
+func (app *Vmsingle) PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	// add extra label
+	url := app.prometheusAPIV1ImportPrometheusURL
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "text/plain")
+	data := []byte(strings.Join(records, "\n"))
+	_, statusCode := app.cli.Post(t, url, data, headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+	}
+}
+
+// PrometheusAPIV1Export is a test helper function that performs the export of
+// raw samples in JSON line format by sending a HTTP POST request to
+// /prometheus/api/v1/export vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1export
+func (app *Vmsingle) PrometheusAPIV1Export(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+
+	res, _ := app.cli.PostForm(t, app.prometheusAPIV1ExportURL, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1ExportNative is a test helper function that performs the export of
+// raw samples in native binary format by sending an HTTP POST request to
+// /prometheus/api/v1/export/native vmselect endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1exportnative
+func (app *Vmsingle) PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte {
+	t.Helper()
+
+	t.Helper()
+	values := opts.asURLValues()
+	values.Add("match[]", query)
+	values.Add("format", "promapi")
+
+	res, _ := app.cli.PostForm(t, app.prometheusAPIV1ExportNativeURL, values, opts.Headers)
+	return []byte(res)
+}
+
+// PrometheusAPIV1Query is a test helper function that performs PromQL/MetricsQL
+// instant query by sending a HTTP POST request to /prometheus/api/v1/query
+// vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query
+func (app *Vmsingle) PrometheusAPIV1Query(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("query", query)
+	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryURL, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1QueryRange is a test helper function that performs
+// PromQL/MetricsQL range query by sending a HTTP POST request to
+// /prometheus/api/v1/query_range vmsingle endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1query_range
+func (app *Vmsingle) PrometheusAPIV1QueryRange(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1QueryResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("query", query)
+
+	res, _ := app.cli.PostForm(t, app.prometheusAPIV1QueryRangeURL, values, opts.Headers)
+	return NewPrometheusAPIV1QueryResponse(t, res)
+}
+
+// PrometheusAPIV1Series sends a query to a /prometheus/api/v1/series endpoint
+// and returns the list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (app *Vmsingle) PrometheusAPIV1Series(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1SeriesResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	res, _ := app.cli.PostForm(t, app.prometheusAPIV1SeriesURL, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesResponse(t, res)
+}
+
+// PrometheusAPIV1SeriesCount sends a query to a /prometheus/api/v1/series/count endpoint
+// and returns the total number of time series.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series
+func (app *Vmsingle) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts) *PrometheusAPIV1SeriesCountResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+
+	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/series/count", app.httpListenAddr)
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1SeriesCountResponse(t, res)
+}
+
+// PrometheusAPIV1Labels sends a query to a /prometheus/api/v1/labels endpoint
+// and returns the label names list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels
+func (app *Vmsingle) PrometheusAPIV1Labels(t *testing.T, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelsResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/labels", app.httpListenAddr)
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1LabelsResponse(t, res)
+}
+
+// PrometheusAPIV1LabelValues sends a query to a /prometheus/api/v1/label/.../values endpoint
+// and returns the label names list of time series that match the query.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues
+func (app *Vmsingle) PrometheusAPIV1LabelValues(t *testing.T, labelName, matchQuery string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/label/%s/values", app.httpListenAddr, labelName)
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1LabelValuesResponse(t, res)
+}
+
+// PrometheusAPIV1Metadata sends a query to a /prometheus/api/v1/metadata endpoint
+// and returns the results.
+func (app *Vmsingle) PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("metric", metric)
+	values.Add("limit", strconv.Itoa(limit))
+	queryURL := fmt.Sprintf("http://%s/prometheus/api/v1/metadata", app.httpListenAddr)
+
+	res, _ := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	return NewPrometheusAPIV1Metadata(t, res)
+}
+
+// APIV1AdminTSDBDeleteSeries deletes the series that match the query by sending
+// a request to /api/v1/admin/tsdb/delete_series.
+//
+// See https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series
+func (app *Vmsingle) APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts) {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/api/v1/admin/tsdb/delete_series", app.httpListenAddr)
+	values := opts.asURLValues()
+	values.Add("match[]", matchQuery)
+
+	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// GraphiteMetricsIndex sends a query to a /metrics/index.json
+//
+// See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#metrics-api
+func (app *Vmsingle) GraphiteMetricsIndex(t *testing.T, _ QueryOpts) GraphiteMetricsIndexResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/metrics/index.json", app.httpListenAddr)
+	res, statusCode := app.cli.Get(t, seriesURL, nil)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var index GraphiteMetricsIndexResponse
+	if err := json.Unmarshal([]byte(res), &index); err != nil {
+		t.Fatalf("could not unmarshal metrics index response data:\n%s\n err: %v", res, err)
+	}
+	return index
+}
+
+// GraphiteTagsTagSeries is a test helper function that registers Graphite tags
+// for a single time series by sending a HTTP POST request to
+// /graphite/tags/tagSeries vmsingle endpoint.
+func (app *Vmsingle) GraphiteTagsTagSeries(t *testing.T, record string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/graphite/tags/tagSeries", app.httpListenAddr)
+	values := opts.asURLValues()
+	values.Add("path", record)
+
+	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+func (app *Vmsingle) GraphiteTagsTagMultiSeries(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/graphite/tags/tagMultiSeries", app.httpListenAddr)
+	values := opts.asURLValues()
+	for _, rec := range records {
+		values.Add("path", rec)
+	}
+
+	_, statusCode := app.cli.PostForm(t, url, values, opts.Headers)
+	if got, want := statusCode, http.StatusNotImplemented; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d", got, want)
+	}
+}
+
+// APIV1StatusMetricNamesStats sends a query to a /api/v1/status/metric_names_stats endpoint
+// and returns the statistics response for given params.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (app *Vmsingle) APIV1StatusMetricNamesStats(t *testing.T, limit, le, matchPattern string, opts QueryOpts) MetricNamesStatsResponse {
+	t.Helper()
+
+	values := opts.asURLValues()
+	values.Add("limit", limit)
+	values.Add("le", le)
+	values.Add("match_pattern", matchPattern)
+	queryURL := fmt.Sprintf("http://%s/api/v1/status/metric_names_stats", app.httpListenAddr)
+
+	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+	var resp MetricNamesStatsResponse
+	if err := json.Unmarshal([]byte(res), &resp); err != nil {
+		t.Fatalf("could not unmarshal metric names stats response data:\n%s\n err: %v", res, err)
+	}
+	return resp
+}
+
+// APIV1AdminStatusMetricNamesStatsReset sends a query to a /api/v1/admin/status/metric_names_stats/reset endpoint
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#track-ingested-metrics-usage
+func (app *Vmsingle) APIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts QueryOpts) {
+	t.Helper()
+
+	values := opts.asURLValues()
+	queryURL := fmt.Sprintf("http://%s/api/v1/admin/status/metric_names_stats/reset", app.httpListenAddr)
+
+	res, statusCode := app.cli.PostForm(t, queryURL, values, opts.Headers)
+	if statusCode != http.StatusNoContent {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
+	}
+}
+
+// SnapshotCreate creates a database snapshot by sending a query to the
+// /snapshot/create endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
+	t.Helper()
+
+	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotCreateResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot create response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotCreateURL returns the URL for creating snapshots.
+func (app *Vmsingle) SnapshotCreateURL() string {
+	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
+}
+
+// APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
+// /api/v1/admin/tsdb/snapshot endpoint.
+//
+// See https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot.
+func (app *Vmsingle) APIV1AdminTSDBSnapshot(t *testing.T) *APIV1AdminTSDBSnapshotResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/api/v1/admin/tsdb/snapshot", app.httpListenAddr)
+	data, statusCode := app.cli.Post(t, queryURL, nil, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res APIV1AdminTSDBSnapshotResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal prometheus snapshot create response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotList lists existing database snapshots by sending a query to the
+// /snapshot/list endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (app *Vmsingle) SnapshotList(t *testing.T) *SnapshotListResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/snapshot/list", app.httpListenAddr)
+	data, statusCode := app.cli.Get(t, queryURL, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotListResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot list response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotDelete deletes a snapshot by sending a query to the
+// /snapshot/delete endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (app *Vmsingle) SnapshotDelete(t *testing.T, snapshotName string) *SnapshotDeleteResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", app.httpListenAddr, snapshotName)
+	data, statusCode := app.cli.Delete(t, queryURL)
+	wantStatusCodes := map[int]bool{
+		http.StatusOK:                  true,
+		http.StatusInternalServerError: true,
+	}
+	if !wantStatusCodes[statusCode] {
+		t.Fatalf("unexpected status code: got %d, want %v, resp text=%q", statusCode, wantStatusCodes, data)
+	}
+
+	var res SnapshotDeleteResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot delete response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// SnapshotDeleteAll deletes all snapshots by sending a query to the
+// /snapshot/delete_all endpoint.
+//
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots
+func (app *Vmsingle) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse {
+	t.Helper()
+
+	queryURL := fmt.Sprintf("http://%s/snapshot/delete_all", app.httpListenAddr)
+	data, statusCode := app.cli.Get(t, queryURL, nil)
+	if got, want := statusCode, http.StatusOK; got != want {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
+	}
+
+	var res SnapshotDeleteAllResponse
+	if err := json.Unmarshal([]byte(data), &res); err != nil {
+		t.Fatalf("could not unmarshal snapshot delete all response: data=%q, err: %v", data, err)
+	}
+
+	return &res
+}
+
+// APIV1StatusTSDB sends a query to a /prometheus/api/v1/status/tsdb
+// //
+// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats
+func (app *Vmsingle) APIV1StatusTSDB(t *testing.T, matchQuery string, date string, topN string, opts QueryOpts) TSDBStatusResponse {
+	t.Helper()
+
+	seriesURL := fmt.Sprintf("http://%s/prometheus/api/v1/status/tsdb", app.httpListenAddr)
+	values := opts.asURLValues()
+	addNonEmpty := func(name, value string) {
+		if len(value) == 0 {
+			return
+		}
+		values.Add(name, value)
+	}
+	addNonEmpty("match[]", matchQuery)
+	addNonEmpty("topN", topN)
+	addNonEmpty("date", date)
+
+	res, statusCode := app.cli.PostForm(t, seriesURL, values, opts.Headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
+	}
+
+	var status TSDBStatusResponse
+	if err := json.Unmarshal([]byte(res), &status); err != nil {
+		t.Fatalf("could not unmarshal tsdb status response data:\n%s\n err: %v", res, err)
+	}
+	status.Sort()
+	return status
+}
+
+// ZabbixConnectorHistory is a test helper function that inserts a
+// collection of records in zabbixconnector  format by sending a HTTP
+// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
+func (app *Vmsingle) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/zabbixconnector/api/v1/history", app.httpListenAddr)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	headers := opts.getHeaders()
+	headers.Set("Content-Type", "application/json")
+	_, statusCode := app.cli.Post(t, url, data, headers)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
 }
 
 // HTTPAddr returns the address at which the vminsert process is