lib/{fs,filestream}: use single ParallelExecutor for fs and filestream tasks

### Describe Your Changes

implement
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10221

### Checklist

The following checks are **mandatory**:

- [x] My change adheres to [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
- [x] My change adheres to [VictoriaMetrics development
goals](https://docs.victoriametrics.com/victoriametrics/goals/).

.github/ISSUE_TEMPLATE/question.yml

@@ -5,7 +5,7 @@ body:
   - type: textarea
     id: describe-the-component
     attributes:
-      label: Is your question request related to a specific component?
+      label: Is your question related to a specific component?
       placeholder: |
         VictoriaMetrics, vmagent, vmalert, vmui, etc...
     validations:

.github/scripts/lint-changelog-tip.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env sh
+
+set -e
+
+CHANGELOG_FILE="docs/victoriametrics/changelog/CHANGELOG.md"
+
+GITHUB_BASE_REF=${GITHUB_BASE_REF:-"master"}
+GIT_REMOTE=${GIT_REMOTE:-"origin"}
+
+git diff "${GIT_REMOTE}/${GITHUB_BASE_REF}"...HEAD -- $CHANGELOG_FILE > diff.txt
+if ! grep -q "^+" diff.txt; then
+  echo "No additions in CHANGELOG.md"
+  exit 0
+fi
+
+ADDED_LINES=$(grep "^+\S" diff.txt | sed 's/^+//')
+
+START_TIP=$(grep -n "^## tip" "$CHANGELOG_FILE" | head -1 | cut -d: -f1)
+if [ -z "$START_TIP" ]; then
+  echo "ERROR: ${CHANGELOG_FILE} does not contain a ## tip section"
+  exit 1
+fi
+
+END_TIP=$(awk "NR>$START_TIP && /^## / {print NR; exit}" "${CHANGELOG_FILE}")
+if [ -z "$END_TIP" ]; then
+  END_TIP=$(wc -l < "$CHANGELOG_FILE")
+fi
+
+BAD=0
+while IFS= read -r line; do
+  # Grep exact line inside the file and get line numbers
+  MATCHES=$(grep -n -F "$line" "$CHANGELOG_FILE" | cut -d: -f1)
+  for m in $MATCHES; do
+    if [ "$m" -lt "$START_TIP" ] || [ "$m" -gt "$END_TIP" ]; then
+      echo "'$line' on line ${m} is outside ## tip section (lines ${START_TIP}-${END_TIP})"
+      BAD=1
+    fi
+  done
+done << EOF
+$ADDED_LINES
+EOF
+
+if [ "$BAD" -ne 0 ]; then
+  echo "CHANGELOG modifications must be placed inside the ## tip section."
+  exit 1
+fi
+
+echo "CHANGELOG modifications are valid."

.github/workflows/build.yml

@@ -61,7 +61,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go

.github/workflows/changelog-linter.yml

@@ -0,0 +1,19 @@
+name: 'changelog-linter'
+
+on:
+  pull_request:
+    paths:
+      - "docs/victoriametrics/changelog/CHANGELOG.md"
+
+jobs:
+  tip-lint:
+    runs-on: 'ubuntu-latest'
+    steps:
+      - uses: 'actions/checkout@v6'
+        with:
+          # needed for proper diff
+          fetch-depth: 0
+
+      - name: 'Validate that changelog changes are under ## tip'
+        run: |
+          GITHUB_BASE_REF=${{ github.base_ref }} ./.github/scripts/lint-changelog-tip.sh

.github/workflows/check-commit-signed.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0  # we need full history for commit verification
 

.github/workflows/codeql-analysis-go.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Set up Go
         id: go

.github/workflows/docs.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
@@ -71,7 +71,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
@@ -97,7 +97,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go

.github/workflows/vmui.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Node
         uses: actions/setup-node@v6

Makefile

@@ -17,7 +17,7 @@ EXTRA_GO_BUILD_TAGS ?=
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.4.0
+GOLANGCI_LINT_VERSION := 2.7.2
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -435,7 +435,7 @@ release-vmutils-windows-goarch: \
 		vmctl-windows-$(GOARCH)-prod.exe
 
 pprof-cpu:
-	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics $(PPROF_FILE)
 
 fmt:
 	gofmt -l -w -s ./lib
@@ -471,7 +471,23 @@ integration-test:
 
 apptest:
 	$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
-	go test ./apptest/... -skip="^TestCluster.*"
+	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+
+integration-test-legacy: victoria-metrics vmbackup vmrestore
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.132.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
+	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
+	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
 	GOEXPERIMENT=synctest go test -bench=. ./lib/...
@@ -500,7 +516,8 @@ app-local-windows-goarch:
 	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
-	qtc
+	qtc -dir=lib
+	qtc -dir=app
 
 install-qtc:
 	which qtc || go install github.com/valyala/quicktemplate/qtc@latest

app/victoria-metrics/self_scraper.go

@@ -10,9 +10,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
@@ -48,6 +50,7 @@ func selfScraper(scrapeInterval time.Duration) {
 
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
+	var metadataRows prometheus.MetadataRows
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
@@ -57,8 +60,12 @@ func selfScraper(scrapeInterval time.Duration) {
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
-		// VictoriaMetrics components don't expose metadata yet, only need to parse samples
-		rows.UnmarshalWithErrLogger(s, nil)
+		// Parse metrics and optionally metadata when enabled
+		if prommetadata.IsEnabled() {
+			rows, metadataRows = prometheus.UnmarshalWithMetadata(rows, metadataRows, s, nil)
+		} else {
+			rows.UnmarshalWithErrLogger(s, nil)
+		}
 		mrs = mrs[:0]
 		for i := range rows.Rows {
 			r := &rows.Rows[i]
@@ -91,6 +98,19 @@ func selfScraper(scrapeInterval time.Duration) {
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
+		if len(metadataRows.Rows) > 0 {
+			mms := make([]metricsmetadata.Row, 0, len(metadataRows.Rows))
+			for _, mm := range metadataRows.Rows {
+				mms = append(mms, metricsmetadata.Row{
+					MetricFamilyName: bytesutil.ToUnsafeBytes(mm.Metric),
+					Help:             bytesutil.ToUnsafeBytes(mm.Help),
+					Type:             mm.Type,
+				})
+			}
+			if err := vmstorage.AddMetadataRows(mms); err != nil {
+				logger.Errorf("cannot store self-scraped metrics metadata: %s", err)
+			}
+		}
 	}
 	for {
 		select {

app/vmagent/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -350,6 +351,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "/zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(nil, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -644,6 +656,17 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(at, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -765,6 +788,9 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 

app/vmagent/newrelic/request_handler.go

@@ -78,7 +78,7 @@ func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label)
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
-	rowsInserted.Add(len(rows))
+	rowsInserted.Add(samplesCount)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(samplesCount)
 	}

app/vmagent/opentelemetry/request_handler.go

@@ -25,7 +25,7 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
-// InsertHandler processes metrics from given reader.
+// InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
 		return insertRows(at, tss, mms, nil)

app/vmagent/remotewrite/client.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -554,9 +553,9 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 // For more details, see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9417
 func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
 	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
+	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
 	if err != nil {
-		return nil, fmt.Errorf("zstd: decompress: %s", err)
+		return nil, err
 	}
 
 	return snappy.Encode(nil, plainBlock), nil

app/vmagent/zabbixconnector/request_handler.go

@@ -0,0 +1,80 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="zabbixconnector"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="zabbixconnector"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := len(rows)
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompb.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		labels = append(labels, extraLabels...)
+
+		samplesLen := len(samples)
+		samples = append(samples, prompb.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+
+		tssDst = append(tssDst, prompb.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/config/config_test.go

@@ -116,7 +116,7 @@ func TestParse_Failure(t *testing.T) {
 
 	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
 	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "invalid annotations")
 	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
@@ -343,7 +343,6 @@ func TestGroupValidate_Failure(t *testing.T) {
 			},
 		},
 	}, true, "bad prometheus expr")
-
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/manager.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -45,13 +46,15 @@ func (m *manager) ruleAPI(gID, rID uint64) (rule.ApiRule, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
-	g, ok := m.groups[gID]
+	group, ok := m.groups[gID]
 	if !ok {
 		return rule.ApiRule{}, fmt.Errorf("can't find group with id %d", gID)
 	}
+	g := group.ToAPI()
+	ruleID := strconv.FormatUint(rID, 10)
 	for _, r := range g.Rules {
-		if r.ID() == rID {
-			return r.ToAPI(), nil
+		if r.ID == ruleID {
+			return r, nil
 		}
 	}
 	return rule.ApiRule{}, fmt.Errorf("can't find rule with id %d in group %q", rID, g.Name)
@@ -62,17 +65,20 @@ func (m *manager) alertAPI(gID, aID uint64) (*rule.ApiAlert, error) {
 	m.groupsMu.RLock()
 	defer m.groupsMu.RUnlock()
 
-	g, ok := m.groups[gID]
+	group, ok := m.groups[gID]
 	if !ok {
 		return nil, fmt.Errorf("can't find group with id %d", gID)
 	}
+	g := group.ToAPI()
 	for _, r := range g.Rules {
-		ar, ok := r.(*rule.AlertingRule)
-		if !ok {
+		if r.Type != rule.TypeAlerting {
 			continue
 		}
-		if apiAlert := ar.AlertToAPI(aID); apiAlert != nil {
-			return apiAlert, nil
+		alertID := strconv.FormatUint(aID, 10)
+		for _, a := range r.Alerts {
+			if a.ID == alertID {
+				return a, nil
+			}
 		}
 	}
 	return nil, fmt.Errorf("can't find alert with id %d in group %q", aID, g.Name)

app/vmalert/notifier/alert.go

@@ -166,8 +166,8 @@ func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl
 		ctmpl, _ := tmpl.Clone()
 		ctmpl = ctmpl.Option("missingkey=zero")
 		if err := templateAnnotation(&buf, builder.String(), tData, ctmpl, execute); err != nil {
-			r[key] = text
-			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
+			r[key] = err.Error()
+			eg.Add(fmt.Errorf("(key: %q, value: %q): %w", key, text, err))
 			continue
 		}
 		r[key] = buf.String()
@@ -184,13 +184,13 @@ type tplData struct {
 func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
 	tpl, err := tpl.Parse(text)
 	if err != nil {
-		return fmt.Errorf("error parsing annotation template: %w", err)
+		return fmt.Errorf("error parsing template: %w", err)
 	}
 	if !execute {
 		return nil
 	}
 	if err = tpl.Execute(dst, data); err != nil {
-		return fmt.Errorf("error evaluating annotation template: %w", err)
+		return fmt.Errorf("error evaluating template: %w", err)
 	}
 	return nil
 }

app/vmalert/notifier/alertmanager.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -86,6 +87,11 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert, alertLabels []
 	err := am.send(ctx, alerts, alertLabels, headers)
 	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
+		// the context can be cancelled on graceful shutdown
+		// or on group update. So no need to handle the error as usual.
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
 		am.metrics.alertsSendErrors.Add(len(alerts))
 		am.lastError = err.Error()
 	} else {

app/vmalert/rule/alerting.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"math"
@@ -246,16 +247,6 @@ func (ar *AlertingRule) GetAlerts() []*notifier.Alert {
 	return alerts
 }
 
-// GetAlert returns alert if id exists
-func (ar *AlertingRule) GetAlert(id uint64) *notifier.Alert {
-	ar.alertsMu.RLock()
-	defer ar.alertsMu.RUnlock()
-	if ar.alerts == nil {
-		return nil
-	}
-	return ar.alerts[id]
-}
-
 func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...any) {
 	if !ar.Debug {
 		return
@@ -321,6 +312,11 @@ type labelSet struct {
 // On k conflicts in origin set, the original value is preferred and copied
 // to processed with `exported_%k` key. The copy happens only if passed v isn't equal to origin[k] value.
 func (ls *labelSet) add(k, v string) {
+	// do not add label with empty value, since it has no meaning.
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9984
+	if v == "" {
+		return
+	}
 	ls.processed[k] = v
 	ov, ok := ls.origin[k]
 	if !ok {
@@ -355,9 +351,6 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		Value:  m.Values[0],
 		Expr:   ar.Expr,
 	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to expand labels: %w", err)
-	}
 	for k, v := range extraLabels {
 		ls.add(k, v)
 	}
@@ -368,7 +361,7 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 	if !*disableAlertGroupLabel && ar.GroupName != "" {
 		ls.add(alertGroupNameLabel, ar.GroupName)
 	}
-	return ls, nil
+	return ls, err
 }
 
 // execRange executes alerting rule on the given time range similarly to exec.
@@ -461,7 +454,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 
 	defer func() {
 		ar.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			ar.metrics.errors.Inc()
 		}
 	}()
@@ -484,8 +477,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 	for i, m := range res.Data {
 		ls, err := ar.expandLabelTemplates(m, qFn)
 		if err != nil {
+			// only set error in current state, but do not break alert processing
 			curState.Err = err
-			return nil, curState.Err
+			logger.Errorf("got templating error in rule %s: %q", ar.Name, err)
 		}
 		at := ts
 		alertID := hash(ls.processed)
@@ -497,8 +491,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		}
 		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
 		if err != nil {
+			// only set error in current state, but do not break alert processing
 			curState.Err = err
-			return nil, curState.Err
+			logger.Errorf("got templating error in rule %s: %q", ar.Name, err)
 		}
 		expandedLabels[i] = ls
 		expandedAnnotations[i] = as
@@ -607,7 +602,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
 	ls, err := ar.toLabels(m, qFn)
 	if err != nil {
-		return nil, fmt.Errorf("failed to expand label templates: %s", err)
+		return ls, fmt.Errorf("failed to expand label templates: %s", err)
 	}
 	return ls, nil
 }
@@ -625,7 +620,7 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {
-		return nil, fmt.Errorf("failed to expand annotation templates: %s", err)
+		return as, fmt.Errorf("failed to expand annotation templates: %s", err)
 	}
 	return as, nil
 }

app/vmalert/rule/alerting_test.go

@@ -1370,8 +1370,10 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 
 	ar := &AlertingRule{
 		Labels: map[string]string{
-			"instance": "override", // this should override instance with new value
-			"group":    "vmalert",  // this shouldn't have effect since value in metric is equal
+			"instance":      "override", // this should override instance with new value
+			"group":         "vmalert",  // this shouldn't have effect since value in metric is equal
+			"invalid_label": "{{ .Values.mustRuntimeFail }}",
+			"empty_label":   "", // this should be dropped
 		},
 		Expr:      "sum(vmalert_alerting_rules_error) by(instance, group, alertname) > 0",
 		Name:      "AlertingRulesError",
@@ -1379,10 +1381,11 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 	}
 
 	expectedOriginLabels := map[string]string{
-		"instance":   "0.0.0.0:8800",
-		"group":      "vmalert",
-		"alertname":  "ConfigurationReloadFailure",
-		"alertgroup": "vmalert",
+		"instance":      "0.0.0.0:8800",
+		"group":         "vmalert",
+		"alertname":     "ConfigurationReloadFailure",
+		"alertgroup":    "vmalert",
+		"invalid_label": `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1392,11 +1395,12 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
+		"invalid_label":      `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+	if err == nil || !strings.Contains(err.Error(), "error evaluating template") {
+		t.Fatalf("unexpected error %q", err.Error())
 	}
 
 	if !reflect.DeepEqual(ls.origin, expectedOriginLabels) {

app/vmalert/rule/recording.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -197,7 +198,7 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 
 	defer func() {
 		rr.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			rr.metrics.errors.Inc()
 		}
 	}()
@@ -236,7 +237,8 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 			Labels: stringToLabels(k),
 			Samples: []prompb.Sample{
 				{Value: decimal.StaleNaN, Timestamp: ts.UnixNano() / 1e6},
-			}})
+			},
+		})
 	}
 	rr.lastEvaluation = curEvaluation
 	return tss, nil
@@ -291,6 +293,11 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompb.TimeSeries {
 	}
 	// add extra labels configured by user
 	for k := range rr.Labels {
+		// do not add label with empty value, since it has no meaning.
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9984
+		if rr.Labels[k] == "" {
+			continue
+		}
 		existingLabel := promrelabel.GetLabelByName(m.Labels, k)
 		if existingLabel != nil { // there is a conflict between extra and existing label
 			if existingLabel.Value == rr.Labels[k] {

app/vmalert/rule/web.go

@@ -209,15 +209,6 @@ func (ar *AlertingRule) AlertsToAPI() []*ApiAlert {
 	return alerts
 }
 
-// AlertToAPI generates apiAlert object from alert by its id(hash)
-func (ar *AlertingRule) AlertToAPI(id uint64) *ApiAlert {
-	a := ar.GetAlert(id)
-	if a == nil {
-		return nil
-	}
-	return NewAlertAPI(ar, a)
-}
-
 // NewAlertAPI creates apiAlert for notifier.Alert
 func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 	aa := &ApiAlert{

app/vmalert/web.go

@@ -412,18 +412,18 @@ func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	defer rh.m.groupsMu.RUnlock()
 
 	var gAlerts []rule.GroupAlerts
-	for _, g := range rh.m.groups {
+	for _, group := range rh.m.groups {
 		var alerts []*rule.ApiAlert
+		g := group.ToAPI()
 		for _, r := range g.Rules {
-			a, ok := r.(*rule.AlertingRule)
-			if !ok {
+			if r.Type != rule.TypeAlerting {
 				continue
 			}
-			alerts = append(alerts, a.AlertsToAPI()...)
+			alerts = append(alerts, r.Alerts...)
 		}
 		if len(alerts) > 0 {
 			gAlerts = append(gAlerts, rule.GroupAlerts{
-				Group:  g.ToAPI(),
+				Group:  g,
 				Alerts: alerts,
 			})
 		}
@@ -444,12 +444,12 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
 		if !rf.matchesGroup(group) {
 			continue
 		}
-		for _, r := range group.Rules {
-			a, ok := r.(*rule.AlertingRule)
-			if !ok {
+		g := group.ToAPI()
+		for _, r := range g.Rules {
+			if r.Type != rule.TypeAlerting {
 				continue
 			}
-			lr.Data.Alerts = append(lr.Data.Alerts, a.AlertsToAPI()...)
+			lr.Data.Alerts = append(lr.Data.Alerts, r.Alerts...)
 		}
 	}
 

app/vmalert/web.qtpl

@@ -602,11 +602,11 @@
         <table class="table table-striped table-hover table-sm">
             <thead>
                 <tr>
-                    <th scope="col" title="The time when event was created">Updated at</th>
+                    <th scope="col" title="The time when the rule was executed">Updated at</th>
                     <th scope="col" class="w-10 text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
                     {% if seriesFetchedEnabled %}<th scope="col" class="w-10 text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
                     <th scope="col" class="w-10 text-center" title="How many seconds request took">Duration</th>
-                    <th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
+                    <th scope="col" class="text-center" title="The time used in execution query request">Execution timestamp</th>
                     <th scope="col" class="text-center" title="cURL command with request example">cURL</th>
                 </tr>
             </thead>

app/vmalert/web.qtpl.go

@@ -1717,7 +1717,7 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule rule.Api
         <table class="table table-striped table-hover table-sm">
             <thead>
                 <tr>
-                    <th scope="col" title="The time when event was created">Updated at</th>
+                    <th scope="col" title="The time when the rule was executed">Updated at</th>
                     <th scope="col" class="w-10 text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
                     `)
 //line app/vmalert/web.qtpl:607
@@ -1729,7 +1729,7 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule rule.Api
 //line app/vmalert/web.qtpl:607
 	qw422016.N().S(`
                     <th scope="col" class="w-10 text-center" title="How many seconds request took">Duration</th>
-                    <th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
+                    <th scope="col" class="text-center" title="The time used in execution query request">Execution timestamp</th>
                     <th scope="col" class="text-center" title="cURL command with request example">cURL</th>
                 </tr>
             </thead>

app/vmauth/auth_config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"errors"
 	"flag"
 	"fmt"
 	"math"
@@ -94,6 +95,8 @@ type UserInfo struct {
 	rt http.RoundTripper
 
 	requests         *metrics.Counter
+	requestErrors    *metrics.Counter
+	backendRequests  *metrics.Counter
 	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
@@ -105,13 +108,29 @@ type HeadersConf struct {
 	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
 }
 
-func (ui *UserInfo) beginConcurrencyLimit() error {
+func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	select {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
 		ui.concurrencyLimitReached.Inc()
-		return fmt.Errorf("cannot handle more than %d concurrent requests from user %s", ui.getMaxConcurrentRequests(), ui.name())
+
+		// The per-user limit for the number of concurrent requests is reached.
+		// Wait until the currently executed requests are finished, so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case ui.concurrencyLimitCh <- struct{}{}:
+			return nil
+		case <-ctx.Done():
+			err := ctx.Err()
+			if errors.Is(err, context.DeadlineExceeded) {
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
+					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
+			}
+
+			return fmt.Errorf("cannot start executing the request because %d concurrent requests from the user %s are executed: %w",
+				ui.getMaxConcurrentRequests(), ui.name(), err)
+		}
 	}
 }
 
@@ -127,6 +146,18 @@ func (ui *UserInfo) getMaxConcurrentRequests() int {
 	return mcr
 }
 
+func (ui *UserInfo) stopHealthChecks() {
+	if ui == nil {
+		return
+	}
+	if ui.URLPrefix == nil {
+		return
+	}
+
+	bus := ui.URLPrefix.bus.Load()
+	bus.stopHealthChecks()
+}
+
 // Header is `Name: Value` http header, which must be added to the proxied request.
 type Header struct {
 	Name  string
@@ -262,7 +293,7 @@ type URLPrefix struct {
 	// the list of backend urls
 	//
 	// the list can be dynamically updated if `discover_backend_ips` option is set.
-	bus atomic.Pointer[[]*backendURL]
+	bus atomic.Pointer[backendURLs]
 
 	// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
 	discoverBackendIPs bool
@@ -286,21 +317,93 @@ func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
 	}
 }
 
+type backendURLs struct {
+	healthChecksContext context.Context
+	healthChecksCancel  func()
+	healthChecksWG      sync.WaitGroup
+
+	bus []*backendURL
+}
+
+func newBackendURLs() *backendURLs {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &backendURLs{
+		healthChecksContext: ctx,
+		healthChecksCancel:  cancel,
+	}
+}
+
+func (bus *backendURLs) add(u *url.URL) {
+	bus.bus = append(bus.bus, &backendURL{
+		url:                u,
+		healthCheckContext: bus.healthChecksContext,
+		healthCheckWG:      &bus.healthChecksWG,
+	})
+}
+
+func (bus *backendURLs) stopHealthChecks() {
+	bus.healthChecksCancel()
+	bus.healthChecksWG.Wait()
+}
+
 type backendURL struct {
-	brokenDeadline     atomic.Uint64
+	broken atomic.Bool
+
+	healthCheckContext context.Context
+	healthCheckWG      *sync.WaitGroup
+
 	concurrentRequests atomic.Int32
 
 	url *url.URL
 }
 
 func (bu *backendURL) isBroken() bool {
-	ct := fasttime.UnixTimestamp()
-	return ct < bu.brokenDeadline.Load()
+	return bu.broken.Load()
 }
 
 func (bu *backendURL) setBroken() {
-	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
-	bu.brokenDeadline.Store(deadline)
+	if bu.broken.CompareAndSwap(false, true) {
+		bu.healthCheckWG.Add(1)
+		go func() {
+			defer bu.healthCheckWG.Done()
+			bu.runHealthCheck()
+			bu.broken.Store(false)
+		}()
+	}
+}
+
+func (bu *backendURL) runHealthCheck() {
+	port := bu.url.Port()
+	if port == "" {
+		port = "80"
+	}
+	addr := net.JoinHostPort(bu.url.Hostname(), port)
+
+	t := time.NewTicker(*failTimeout)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			// Verify network connectivity via TCP dial before marking backend healthy.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9997
+			ctx, cancel := context.WithTimeout(bu.healthCheckContext, time.Second)
+			c, err := netutil.Dialer.DialContext(ctx, "tcp", addr)
+			cancel()
+			if err != nil {
+				if errors.Is(bu.healthCheckContext.Err(), context.Canceled) {
+					return
+				}
+				logger.Warnf("ignoring the backend at %s for %s becasue of dial error: %s", addr, *failTimeout, err)
+				continue
+			}
+
+			_ = c.Close()
+			return
+		case <-bu.healthCheckContext.Done():
+			return
+		}
+	}
 }
 
 func (bu *backendURL) get() {
@@ -312,8 +415,8 @@ func (bu *backendURL) put() {
 }
 
 func (up *URLPrefix) getBackendsCount() int {
-	pbus := up.bus.Load()
-	return len(*pbus)
+	bus := up.bus.Load()
+	return len(bus.bus)
 }
 
 // getBackendURL returns the backendURL depending on the load balance policy.
@@ -324,16 +427,15 @@ func (up *URLPrefix) getBackendsCount() int {
 func (up *URLPrefix) getBackendURL() *backendURL {
 	up.discoverBackendAddrsIfNeeded()
 
-	pbus := up.bus.Load()
-	bus := *pbus
-	if len(bus) == 0 {
+	bus := up.bus.Load()
+	if len(bus.bus) == 0 {
 		return nil
 	}
 
 	if up.loadBalancingPolicy == "first_available" {
-		return getFirstAvailableBackendURL(bus)
+		return getFirstAvailableBackendURL(bus.bus)
 	}
-	return getLeastLoadedBackendURL(bus, &up.n)
+	return getLeastLoadedBackendURL(bus.bus, &up.n)
 }
 
 func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
@@ -407,25 +509,24 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	cancel()
 
 	// generate new backendURLs for the resolved IPs
-	var busNew []*backendURL
+	busNew := newBackendURLs()
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
 			buCopy.Host = addr
-			busNew = append(busNew, &backendURL{
-				url: &buCopy,
-			})
+			busNew.add(&buCopy)
 		}
 	}
 
-	pbus := up.bus.Load()
-	if areEqualBackendURLs(*pbus, busNew) {
+	bus := up.bus.Load()
+	if areEqualBackendURLs(bus.bus, busNew.bus) {
 		return
 	}
 
 	// Store new backend urls
-	up.bus.Store(&busNew)
+	up.bus.Store(busNew)
+	bus.stopHealthChecks()
 }
 
 func areEqualBackendURLs(a, b []*backendURL) bool {
@@ -456,20 +557,23 @@ func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 	for i := 1; i < len(bus); i++ {
 		if !bus[i].isBroken() {
 			bu = bus[i]
-			break
+			bu.get()
+			return bu
 		}
 	}
-	bu.get()
-	return bu
+	return nil
 }
 
-// getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
+// getLeastLoadedBackendURL returns a non-broken backendURL with the lowest number of concurrent requests.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
 	if len(bus) == 1 {
 		// Fast path - return the only backend url.
 		bu := bus[0]
+		if bu.isBroken() {
+			return nil
+		}
 		bu.get()
 		return bu
 	}
@@ -494,7 +598,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
 	buMinIdx := n % uint32(len(bus))
 	minRequests := bus[buMinIdx].concurrentRequests.Load()
-	for i := uint32(0); i < uint32(len(bus)); i++ {
+	for i := uint32(1); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
 		bu := bus[idx]
 		if bu.isBroken() {
@@ -508,6 +612,9 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		}
 	}
 	buMin := bus[buMinIdx]
+	if buMin.isBroken() {
+		return nil
+	}
 	buMin.get()
 	atomicCounter.CompareAndSwap(n+1, buMinIdx+1)
 	return buMin
@@ -732,6 +839,11 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 
 	acPrev := authConfig.Load()
 	if acPrev != nil {
+		acPrev.UnauthorizedUser.stopHealthChecks()
+		for i := range acPrev.Users {
+			acPrev.Users[i].stopHealthChecks()
+		}
+
 		metrics.UnregisterSet(acPrev.ms, true)
 	}
 	metrics.RegisterSet(ac.ms)
@@ -778,6 +890,8 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
 		}
 		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
@@ -826,6 +940,8 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
 		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
@@ -1060,13 +1176,11 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 	}
 
 	// Initialize up.bus
-	bus := make([]*backendURL, len(up.busOriginal))
-	for i, bu := range up.busOriginal {
-		bus[i] = &backendURL{
-			url: bu,
-		}
+	bus := newBackendURLs()
+	for _, bu := range up.busOriginal {
+		bus.add(bu)
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 
 	return nil
 }

app/vmauth/auth_config_test.go

@@ -753,7 +753,7 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	up.loadBalancingPolicy = "least_loaded"
 
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	fn := func(ns ...int) {
 		t.Helper()
@@ -825,7 +825,7 @@ func TestBrokenBackend(t *testing.T) {
 	})
 	up.loadBalancingPolicy = "least_loaded"
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	// explicitly mark one of the backends as broken
 	bus[1].setBroken()
@@ -848,7 +848,7 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 
 		up.discoverBackendAddrsIfNeeded()
 		pbus := up.bus.Load()
-		bus := *pbus
+		bus := pbus.bus
 
 		if len(bus) != 1 {
 			t.Fatalf("expected url list to be of size 1; got %d instead", len(bus))
@@ -942,16 +942,14 @@ func mustParseURL(u string) *URLPrefix {
 }
 
 func mustParseURLs(us []string) *URLPrefix {
-	bus := make([]*backendURL, len(us))
+	bus := newBackendURLs()
 	urls := make([]*url.URL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
 			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
 		}
-		bus[i] = &backendURL{
-			url: pu,
-		}
+		bus.add(pu)
 		urls[i] = pu
 	}
 	up := &URLPrefix{}
@@ -960,7 +958,7 @@ func mustParseURLs(us []string) *URLPrefix {
 	} else {
 		up.vOriginal = us
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 	up.busOriginal = urls
 	return up
 }

app/vmauth/main.go

@@ -44,12 +44,17 @@ var (
 		"See also -maxConcurrentRequests")
 	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
-	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
+		"'429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
-		"in per-user config")
+		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options "+
+		"and max_concurrent_requests option in per-user config")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration the request waits for execution when the number of concurrently executed "+
+		"requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. "+
+		"This allows graceful handling of short spikes in the number of concurrent requests")
+
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
@@ -151,7 +156,6 @@ func requestHandlerWithInternalRoutes(w http.ResponseWriter, r *http.Request) bo
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-
 	ats := getAuthTokensFromRequest(r)
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
@@ -208,20 +212,45 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 	ui.requests.Inc()
 
+	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
+	defer cancel()
+
 	// Limit the concurrency of requests to backends
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		if err := ui.beginConcurrencyLimit(); err != nil {
+		if err := ui.beginConcurrencyLimit(ctx); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
 			return
 		}
 	default:
-		concurrentRequestsLimitReached.Inc()
-		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
-		handleConcurrencyLimitError(w, r, err)
-		return
+		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
+		// so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case concurrencyLimitCh <- struct{}{}:
+			if err := ui.beginConcurrencyLimit(ctx); err != nil {
+				handleConcurrencyLimitError(w, r, err)
+				<-concurrencyLimitCh
+				return
+			}
+		case <-ctx.Done():
+			err := ctx.Err()
+
+			concurrentRequestsLimitReached.Inc()
+
+			if errors.Is(err, context.DeadlineExceeded) {
+				err = fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+					*maxQueueDuration, cap(concurrencyLimitCh))
+				handleConcurrencyLimitError(w, r, err)
+				return
+			}
+
+			err = fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
+			handleConcurrencyLimitError(w, r, err)
+			return
+		}
 	}
 	processRequest(w, r, ui)
 	ui.endConcurrencyLimit()
@@ -285,16 +314,18 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			return
 		}
 		bu.setBroken()
+		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
 		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)
-	ui.backendErrors.Inc()
+	ui.requestErrors.Inc()
 }
 
 func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
 	req.URL = targetURL
@@ -325,7 +356,6 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			if errors.Is(err, context.DeadlineExceeded) {
 				// Timed out request must be counted as errors, since this usually means that the backend is slow.
 				logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
-				ui.backendErrors.Inc()
 			}
 			return false, false
 		}
@@ -337,6 +367,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
@@ -344,11 +375,11 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			return false, true
 		}
 
-		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
+		// Request body wasn't read yet, this usually means that the backend isn't reachable; retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, req.URL, targetURL, err)
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
@@ -357,12 +388,13 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
 			return true, false
 		}
 		// Retry requests at other backends if it matches retryStatusCodes.
@@ -370,7 +402,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
 			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
 		return false, false
 	}
@@ -386,6 +418,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		requestURI := httpserver.GetRequestURI(r)
 
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+		ui.requestErrors.Inc()
 		return true, false
 	}
 	return true, false
@@ -596,6 +629,13 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
+	ctx := r.Context()
+	if errors.Is(ctx.Err(), context.Canceled) {
+		// Do not return any response for the request canceled by the client,
+		// since the connection to the client is already closed.
+		return
+	}
+
 	w.Header().Add("Retry-After", "10")
 	err = &httpserver.ErrorWithStatusCode{
 		Err:        err,
@@ -652,6 +692,7 @@ type zeroReader struct{}
 func (r *zeroReader) Read(_ []byte) (int, error) {
 	return 0, io.EOF
 }
+
 func (r *zeroReader) Close() error {
 	return nil
 }

app/vmbackup/main.go

@@ -212,7 +212,7 @@ func newSrcFS() (*fslocal.FS, error) {
 }
 
 func newDstFS(ctx context.Context) (common.RemoteFS, error) {
-	fs, err := actions.NewRemoteFS(ctx, *dst)
+	fs, err := actions.NewRemoteFS(ctx, *dst, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-dst`=%q: %w", *dst, err)
 	}
@@ -255,7 +255,7 @@ func newOriginFS(ctx context.Context) (common.OriginFS, error) {
 	if len(*origin) == 0 {
 		return &fsnil.FS{}, nil
 	}
-	fs, err := actions.NewRemoteFS(ctx, *origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}
@@ -266,7 +266,7 @@ func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
 	if len(*origin) == 0 {
 		return nil, fmt.Errorf("-origin cannot be empty when -snapshotName and -snapshot.createURL aren't set")
 	}
-	fs, err := actions.NewRemoteFS(ctx, *origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}

app/vmctl/influx.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"log"
@@ -37,7 +38,7 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st
 	}
 }
 
-func (ip *influxProcessor) run() error {
+func (ip *influxProcessor) run(ctx context.Context) error {
 	series, err := ip.ic.Explore()
 	if err != nil {
 		return fmt.Errorf("explore query failed: %s", err)
@@ -47,7 +48,7 @@ func (ip *influxProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d timeseries to import. Continue?", len(series))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/main.go

@@ -103,7 +103,7 @@ func main() {
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency), c.Bool(globalVerbose))
-					return otsdbProcessor.run()
+					return otsdbProcessor.run(ctx)
 				},
 			},
 			{
@@ -164,7 +164,7 @@ func main() {
 						c.Bool(influxSkipDatabaseLabel),
 						c.Bool(influxPrometheusMode),
 						c.Bool(globalVerbose))
-					return processor.run()
+					return processor.run(ctx)
 				},
 			},
 			{
@@ -279,7 +279,7 @@ func main() {
 						cc:        c.Int(promConcurrency),
 						isVerbose: c.Bool(globalVerbose),
 					}
-					return pp.run()
+					return pp.run(ctx)
 				},
 			},
 			{

app/vmctl/opentsdb.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
@@ -37,7 +38,7 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, verbos
 	}
 }
 
-func (op *otsdbProcessor) run() error {
+func (op *otsdbProcessor) run(ctx context.Context) error {
 	log.Println("Loading all metrics from OpenTSDB for filters: ", op.oc.Filters)
 	var metrics []string
 	for _, filter := range op.oc.Filters {
@@ -53,7 +54,7 @@ func (op *otsdbProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d metrics to import. Continue?", len(metrics))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 	op.im.ResetStats()

app/vmctl/prometheus.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
@@ -30,7 +31,7 @@ type prometheusProcessor struct {
 	isVerbose bool
 }
 
-func (pp *prometheusProcessor) run() error {
+func (pp *prometheusProcessor) run(ctx context.Context) error {
 	blocks, err := pp.cl.Explore()
 	if err != nil {
 		return fmt.Errorf("explore failed: %s", err)
@@ -39,7 +40,7 @@ func (pp *prometheusProcessor) run() error {
 		return fmt.Errorf("found no blocks to import")
 	}
 	question := fmt.Sprintf("Found %d blocks to import. Continue?", len(blocks))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/remoteread.go

@@ -47,7 +47,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 
 	question := fmt.Sprintf("Selected time range %q - %q will be split into %d ranges according to %q step. Continue?",
 		rrp.filter.timeStart.String(), rrp.filter.timeEnd.String(), len(ranges), rrp.filter.chunk)
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/utils.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -15,7 +16,7 @@ const barTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "
 // isSilent should be inited in main
 var isSilent bool
 
-func prompt(question string) bool {
+func prompt(ctx context.Context, question string) bool {
 	if isSilent {
 		return true
 	}
@@ -25,15 +26,32 @@ func prompt(question string) bool {
 	}
 	reader := bufio.NewReader(os.Stdin)
 	fmt.Print(question, " [Y/n] ")
-	answer, err := reader.ReadString('\n')
-	if err != nil {
+
+	answerCh := make(chan string, 1)
+	errCh := make(chan error, 1)
+
+	go func() {
+		answer, err := reader.ReadString('\n')
+		if err != nil {
+			errCh <- err
+			return
+		}
+		answerCh <- answer
+	}()
+
+	select {
+	case <-ctx.Done():
+		fmt.Println("\nCanceled.")
+		return false
+	case err := <-errCh:
 		panic(err)
+	case answer := <-answerCh:
+		answer = strings.TrimSpace(strings.ToLower(answer))
+		if answer == "" || answer == "yes" || answer == "y" {
+			return true
+		}
+		return false
 	}
-	answer = strings.TrimSpace(strings.ToLower(answer))
-	if answer == "" || answer == "yes" || answer == "y" {
-		return true
-	}
-	return false
 }
 
 func wrapErr(vmErr *vm.ImportError, verbose bool) error {

app/vmctl/vm_native.go

@@ -79,7 +79,7 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 			return fmt.Errorf("failed to get tenants: %w", err)
 		}
 		question := fmt.Sprintf("The following tenants were discovered: %s.\n Continue?", tenants)
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	}
@@ -233,7 +233,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		// do not prompt for intercluster because there could be many tenants,
 		// and we don't want to interrupt the process when moving to the next tenant.
 		question := foundSeriesMsg + ". Continue?"
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	} else {

app/vminsert/common/insert_ctx.go

@@ -11,9 +11,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
@@ -50,8 +52,9 @@ var (
 type InsertCtx struct {
 	Labels sortedLabels
 
-	mrs            []storage.MetricRow
-	metricNamesBuf []byte
+	mrs           []storage.MetricRow
+	mms           []metricsmetadata.Row
+	metricNameBuf []byte
 
 	relabelCtx    relabel.Ctx
 	streamAggrCtx streamAggrCtx
@@ -73,8 +76,13 @@ func (ctx *InsertCtx) Reset(rowsLen int) {
 	}
 	mrs = slicesutil.SetLength(mrs, rowsLen)
 	ctx.mrs = mrs[:0]
+	mms := ctx.mms
+	for i := range mms {
+		cleanMetricMetadata(&mms[i])
+	}
+	ctx.mms = mms[:0]
 
-	ctx.metricNamesBuf = ctx.metricNamesBuf[:0]
+	ctx.metricNameBuf = ctx.metricNameBuf[:0]
 	ctx.relabelCtx.Reset()
 	ctx.streamAggrCtx.Reset()
 	ctx.skipStreamAggr = false
@@ -84,11 +92,20 @@ func cleanMetricRow(mr *storage.MetricRow) {
 	mr.MetricNameRaw = nil
 }
 
+func cleanMetricMetadata(mm *metricsmetadata.Row) {
+	mm.MetricFamilyName = nil
+	mm.Unit = nil
+	mm.Help = nil
+	mm.Type = 0
+	mm.ProjectID = 0
+	mm.AccountID = 0
+}
+
 func (ctx *InsertCtx) marshalMetricNameRaw(prefix []byte, labels []prompb.Label) []byte {
-	start := len(ctx.metricNamesBuf)
-	ctx.metricNamesBuf = append(ctx.metricNamesBuf, prefix...)
-	ctx.metricNamesBuf = storage.MarshalMetricNameRaw(ctx.metricNamesBuf, labels)
-	metricNameRaw := ctx.metricNamesBuf[start:]
+	start := len(ctx.metricNameBuf)
+	ctx.metricNameBuf = append(ctx.metricNameBuf, prefix...)
+	ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf, labels)
+	metricNameRaw := ctx.metricNameBuf[start:]
 	return metricNameRaw[:len(metricNameRaw):len(metricNameRaw)]
 }
 
@@ -143,7 +160,7 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	mr.MetricNameRaw = metricNameRaw
 	mr.Timestamp = timestamp
 	mr.Value = value
-	if len(ctx.metricNamesBuf) > 16*1024*1024 {
+	if len(ctx.metricNameBuf) > 16*1024*1024 {
 		if err := ctx.FlushBufs(); err != nil {
 			return err
 		}
@@ -151,6 +168,55 @@ func (ctx *InsertCtx) addRow(metricNameRaw []byte, timestamp int64, value float6
 	return nil
 }
 
+// WriteMetadata writes given prometheus protobuf  metadata into the storage.
+func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
+	if len(mmpbs) == 0 {
+		return nil
+	}
+	mms := ctx.mms
+	mms = slicesutil.SetLength(mms, len(mmpbs))
+	for idx, mmpb := range mmpbs {
+		mm := &mms[idx]
+		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
+		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
+		mm.Type = mmpb.Type
+		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
+	}
+
+	err := vmstorage.AddMetadataRows(mms)
+	if err != nil {
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot store metrics metadata: %w", err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+	}
+	return nil
+}
+
+// WritePromMetadata writes given prometheus metric metadata into the storage
+func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
+	if len(mmps) == 0 {
+		return nil
+	}
+	mms := ctx.mms
+	mms = slicesutil.SetLength(mms, len(mmps))
+	for idx, mmpb := range mmps {
+		mm := &mms[idx]
+		mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
+		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
+		mm.Type = mmpb.Type
+	}
+
+	err := vmstorage.AddMetadataRows(mms)
+	if err != nil {
+		return &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot store prometheus metrics metadata: %w", err),
+			StatusCode: http.StatusServiceUnavailable,
+		}
+	}
+	return nil
+}
+
 // AddLabelBytes adds (name, value) label to ctx.Labels.
 //
 // name and value must exist until ctx.Labels is used.

app/vminsert/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -231,6 +232,17 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusAccepted)
+		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -423,6 +435,9 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vm_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 

app/vminsert/opentelemetry/request_handler.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
@@ -14,8 +15,9 @@ import (
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="opentelemetry"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="opentelemetry"}`)
+	rowsInserted     = metrics.NewCounter(`vm_rows_inserted_total{type="opentelemetry"}`)
+	rowsPerInsert    = metrics.NewHistogram(`vm_rows_per_insert{type="opentelemetry"}`)
+	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="opentelemetry"}`)
 )
 
 // InsertHandler processes opentelemetry metrics.
@@ -33,12 +35,12 @@ func InsertHandler(req *http.Request) error {
 			return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
 		}
 	}
-	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, _ []prompb.MetricMetadata) error {
-		return insertRows(tss, extraLabels)
+	return stream.ParseStream(req.Body, encoding, processBody, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
+		return insertRows(tss, mms, extraLabels)
 	})
 }
 
-func insertRows(tss []prompb.TimeSeries, extraLabels []prompb.Label) error {
+func insertRows(tss []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -75,5 +77,14 @@ func insertRows(tss []prompb.TimeSeries, extraLabels []prompb.Label) error {
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
-	return ctx.FlushBufs()
+	if err := ctx.FlushBufs(); err != nil {
+		return fmt.Errorf("cannot flush metric bufs: %w", err)
+	}
+	if prommetadata.IsEnabled() {
+		if err := ctx.WriteMetadata(mms); err != nil {
+			return err
+		}
+		metadataInserted.Add(len(mms))
+	}
+	return nil
 }

app/vminsert/prometheusimport/request_handler.go

@@ -1,6 +1,7 @@
 package prometheusimport
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
@@ -15,8 +16,9 @@ import (
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="prometheus"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="prometheus"}`)
+	rowsInserted     = metrics.NewCounter(`vm_rows_inserted_total{type="prometheus"}`)
+	rowsPerInsert    = metrics.NewHistogram(`vm_rows_per_insert{type="prometheus"}`)
+	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="prometheus"}`)
 )
 
 // InsertHandler processes `/api/v1/import/prometheus` request.
@@ -30,14 +32,14 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	encoding := req.Header.Get("Content-Encoding")
-	return stream.Parse(req.Body, defaultTimestamp, encoding, true, prommetadata.IsEnabled(), func(rows []prometheus.Row, _ []prometheus.Metadata) error {
-		return insertRows(rows, extraLabels)
+	return stream.Parse(req.Body, defaultTimestamp, encoding, true, prommetadata.IsEnabled(), func(rows []prometheus.Row, mms []prometheus.Metadata) error {
+		return insertRows(rows, mms, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)
 	})
 }
 
-func insertRows(rows []prometheus.Row, extraLabels []prompb.Label) error {
+func insertRows(rows []prometheus.Row, mms []prometheus.Metadata, extraLabels []prompb.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -64,5 +66,15 @@ func insertRows(rows []prometheus.Row, extraLabels []prompb.Label) error {
 	}
 	rowsInserted.Add(len(rows))
 	rowsPerInsert.Update(float64(len(rows)))
-	return ctx.FlushBufs()
+	if err := ctx.FlushBufs(); err != nil {
+		return fmt.Errorf("cannot flush metric bufs: %w", err)
+	}
+
+	if prommetadata.IsEnabled() {
+		if err := ctx.WritePromMetadata(mms); err != nil {
+			return err
+		}
+		metadataInserted.Add(len(mms))
+	}
+	return nil
 }

app/vminsert/prompush/push.go

@@ -4,13 +4,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+	rowsInserted         = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
+	rowsPerInsert        = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+	metadataRowsInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="promscrape"}`)
 )
 
 const maxRowsPerBlock = 10000
@@ -41,6 +43,13 @@ func Push(wr *prompb.WriteRequest) {
 		}
 		push(ctx, tssBlock)
 	}
+	if prommetadata.IsEnabled() {
+		if err := ctx.WriteMetadata(wr.Metadata); err != nil {
+			logger.Errorf("cannot write promscrape metrics metadata to storage: %s", err)
+		} else {
+			metadataRowsInserted.Add(len(wr.Metadata))
+		}
+	}
 }
 
 func push(ctx *common.InsertCtx, tss []prompb.TimeSeries) {

app/vminsert/promremotewrite/request_handler.go

@@ -1,10 +1,12 @@
 package promremotewrite
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/promremotewrite/stream"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
@@ -12,8 +14,9 @@ import (
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promremotewrite"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promremotewrite"}`)
+	rowsInserted     = metrics.NewCounter(`vm_rows_inserted_total{type="promremotewrite"}`)
+	rowsPerInsert    = metrics.NewHistogram(`vm_rows_per_insert{type="promremotewrite"}`)
+	metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="promremotewrite"}`)
 )
 
 // InsertHandler processes remote write for prometheus.
@@ -23,12 +26,12 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isVMRemoteWrite := req.Header.Get("Content-Encoding") == "zstd"
-	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, _ []prompb.MetricMetadata) error {
-		return insertRows(tss, extraLabels)
+	return stream.Parse(req.Body, isVMRemoteWrite, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
+		return insertRows(tss, mms, extraLabels)
 	})
 }
 
-func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompb.Label) error {
+func insertRows(timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
 	ctx := common.GetInsertCtx()
 	defer common.PutInsertCtx(ctx)
 
@@ -68,5 +71,15 @@ func insertRows(timeseries []prompb.TimeSeries, extraLabels []prompb.Label) erro
 	}
 	rowsInserted.Add(rowsTotal)
 	rowsPerInsert.Update(float64(rowsTotal))
-	return ctx.FlushBufs()
+
+	if err := ctx.FlushBufs(); err != nil {
+		return fmt.Errorf("cannot flush metric bufs: %w", err)
+	}
+	if prommetadata.IsEnabled() {
+		if err := ctx.WriteMetadata(mms); err != nil {
+			return err
+		}
+		metadataInserted.Add(len(mms))
+	}
+	return nil
 }

app/vminsert/zabbixconnector/request_handler.go

@@ -0,0 +1,67 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="zabbixconnector"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(rows, extraLabels)
+	})
+}
+
+func insertRows(rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsTotal := len(rows)
+	ctx.Reset(rowsTotal)
+	hasRelabeling := relabel.HasRelabeling()
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		for k := range r.Tags {
+			t := &r.Tags[k]
+			ctx.AddLabelBytes(t.Key, t.Value)
+		}
+		for k := range extraLabels {
+			label := &extraLabels[k]
+			ctx.AddLabel(label.Name, label.Value)
+		}
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		ctx.SortLabelsIfNeeded()
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vmrestore/main.go

@@ -104,7 +104,7 @@ func newDstFS() (*fslocal.FS, error) {
 }
 
 func newSrcFS(ctx context.Context) (common.RemoteFS, error) {
-	fs, err := actions.NewRemoteFS(ctx, *src)
+	fs, err := actions.NewRemoteFS(ctx, *src, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-src`=%q: %w", *src, err)
 	}

app/vmselect/main.go

@@ -421,6 +421,16 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/metadata":
+		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+		metadataRequests.Inc()
+		if err := prometheus.MetadataHandler(qt, startTime, w, r); err != nil {
+			metadataErrors.Inc()
+			httpserver.SendPrometheusError(w, r, err)
+			return true
+		}
+		return true
+
 	default:
 		return false
 	}
@@ -574,12 +584,6 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		w.Header().Set("Content-Type", "application/json")
 		fmt.Fprint(w, `{"status":"success","data":{"notifiers":[]}}`)
 		return true
-	case "/api/v1/metadata":
-		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
-		metadataRequests.Inc()
-		w.Header().Set("Content-Type", "application/json")
-		fmt.Fprintf(w, "%s", `{"status":"success","data":{}}`)
-		return true
 	case "/api/v1/status/buildinfo":
 		buildInfoRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -708,7 +712,9 @@ var (
 	alertsRequests    = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`)
 	notifiersRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/notifiers"}`)
 
-	metadataRequests       = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
+	metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
+	metadataErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/metadata"}`)
+
 	buildInfoRequests      = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/buildinfo"}`)
 	queryExemplarsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/query_exemplars"}`)
 

app/vmselect/netstorage/netstorage.go

@@ -20,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 )
 
 var (
@@ -865,6 +866,23 @@ func LabelValues(qt *querytracer.Tracer, labelName string, sq *storage.SearchQue
 	return labelValues, nil
 }
 
+// GetMetricsMetadata returns time series metric names metadata for the given args
+func GetMetricsMetadata(qt *querytracer.Tracer, limit int, metricName string) ([]*metricsmetadata.Row, error) {
+	qt = qt.NewChild("get metrics metadata: limit=%d, metric_name=%q", limit, metricName)
+	defer qt.Done()
+
+	metadata := vmstorage.Storage.GetMetadataRows(qt, limit, metricName)
+
+	sort.Slice(metadata, func(i, j int) bool {
+		return string(metadata[i].MetricFamilyName) < string(metadata[j].MetricFamilyName)
+	})
+	if limit > 0 && len(metadata) >= limit {
+		metadata = metadata[:limit]
+	}
+
+	return metadata, nil
+}
+
 // GraphiteTagValues returns tag values for the given tagName until the given deadline.
 func GraphiteTagValues(qt *querytracer.Tracer, tagName, filter string, limit int, deadline searchutil.Deadline) ([]string, error) {
 	qt = qt.NewChild("get graphite tag values for tagName=%s, filter=%s, limit=%d", tagName, filter, limit)

app/vmselect/prometheus/metadata_response.qtpl

@@ -0,0 +1,35 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+) %}
+
+{% stripspace %}
+MetadataResponse generates response for /api/v1/metadata
+See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+{% func MetadataResponse( result []*metricsmetadata.Row, qt *querytracer.Tracer) %}
+{
+	"status":"success",
+	"data": {
+		{% code
+			mapItems := len(result)
+			currentItem := 0
+		 %}
+		{% for _, row := range result %}
+			"{%s string(row.MetricFamilyName) %}": [
+					{
+						"type": {%q= row.Type.String() %},
+            {% if len(row.Unit) > 0 -%}
+						"unit": {%q= string(row.Unit) %},
+            {% endif -%}
+						"help": {%q= string(row.Help) %}
+					}
+			]
+			{% if currentItem != mapItems-1 %},{% endif %}
+			{% code currentItem++ %}
+		{% endfor %}
+	}
+	{%= dumpQueryTrace(qt) %}
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/prometheus/metadata_response.qtpl.go

@@ -0,0 +1,108 @@
+// Code generated by qtc from "metadata_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/prometheus/metadata_response.qtpl:1
+package prometheus
+
+//line app/vmselect/prometheus/metadata_response.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+)
+
+// MetadataResponse generates response for /api/v1/metadataSee https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+func StreamMetadataResponse(qw422016 *qt422016.Writer, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/metadata_response.qtpl:9
+	qw422016.N().S(`{"status":"success","data": {`)
+//line app/vmselect/prometheus/metadata_response.qtpl:14
+	mapItems := len(result)
+	currentItem := 0
+
+//line app/vmselect/prometheus/metadata_response.qtpl:17
+	for _, row := range result {
+//line app/vmselect/prometheus/metadata_response.qtpl:17
+		qw422016.N().S(`"`)
+//line app/vmselect/prometheus/metadata_response.qtpl:18
+		qw422016.E().S(string(row.MetricFamilyName))
+//line app/vmselect/prometheus/metadata_response.qtpl:18
+		qw422016.N().S(`": [{"type":`)
+//line app/vmselect/prometheus/metadata_response.qtpl:20
+		qw422016.N().Q(row.Type.String())
+//line app/vmselect/prometheus/metadata_response.qtpl:20
+		qw422016.N().S(`,`)
+//line app/vmselect/prometheus/metadata_response.qtpl:21
+		if len(row.Unit) > 0 {
+//line app/vmselect/prometheus/metadata_response.qtpl:21
+			qw422016.N().S(`"unit":`)
+//line app/vmselect/prometheus/metadata_response.qtpl:22
+			qw422016.N().Q(string(row.Unit))
+//line app/vmselect/prometheus/metadata_response.qtpl:22
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/metadata_response.qtpl:23
+		}
+//line app/vmselect/prometheus/metadata_response.qtpl:23
+		qw422016.N().S(`"help":`)
+//line app/vmselect/prometheus/metadata_response.qtpl:24
+		qw422016.N().Q(string(row.Help))
+//line app/vmselect/prometheus/metadata_response.qtpl:24
+		qw422016.N().S(`}]`)
+//line app/vmselect/prometheus/metadata_response.qtpl:27
+		if currentItem != mapItems-1 {
+//line app/vmselect/prometheus/metadata_response.qtpl:27
+			qw422016.N().S(`,`)
+//line app/vmselect/prometheus/metadata_response.qtpl:27
+		}
+//line app/vmselect/prometheus/metadata_response.qtpl:28
+		currentItem++
+
+//line app/vmselect/prometheus/metadata_response.qtpl:29
+	}
+//line app/vmselect/prometheus/metadata_response.qtpl:29
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/metadata_response.qtpl:31
+	streamdumpQueryTrace(qw422016, qt)
+//line app/vmselect/prometheus/metadata_response.qtpl:31
+	qw422016.N().S(`}`)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+}
+
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+func WriteMetadataResponse(qq422016 qtio422016.Writer, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	StreamMetadataResponse(qw422016, result, qt)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+}
+
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+func MetadataResponse(result []*metricsmetadata.Row, qt *querytracer.Tracer) string {
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	WriteMetadataResponse(qb422016, result, qt)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qs422016 := string(qb422016.B)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+	return qs422016
+//line app/vmselect/prometheus/metadata_response.qtpl:33
+}

app/vmselect/prometheus/prometheus.go

@@ -639,6 +639,37 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 	return nil
 }
 
+// MetadataHandler processes /api/v1/metadata request.
+//
+// See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
+func MetadataHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+
+	limit, err := httputil.GetInt(r, "limit")
+	if err != nil {
+		return err
+	}
+	if limit < 0 {
+		limit = 0
+	}
+
+	metricName := r.FormValue("metric")
+
+	metadata, err := netstorage.GetMetricsMetadata(qt, limit, metricName)
+	if err != nil {
+		return fmt.Errorf("cannot get metadata: %w", err)
+	}
+	qt.Done()
+	w.Header().Set("Content-Type", "application/json")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	WriteMetadataResponse(bw, metadata, qt)
+	if err := bw.Flush(); err != nil {
+		return fmt.Errorf("cannot send metadata response to remote client: %w", err)
+	}
+
+	return nil
+}
+
 var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/labels"}`)
 
 // SeriesCountHandler processes /api/v1/series/count request.

app/vmselect/promql/eval.go

@@ -1169,60 +1169,6 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			},
 		}
 		return evalExpr(qt, ec, be)
-	case "rate":
-		if iafc != nil {
-			if !strings.EqualFold(iafc.ae.Name, "sum") {
-				qt.Printf("do not apply instant rollup optimization for incremental aggregate %s()", iafc.ae.Name)
-				return evalAt(qt, timestamp, window)
-			}
-			qt.Printf("optimized calculation for sum(rate(m[d])) as (sum(increase(m[d])) / d)")
-			afe := expr.(*metricsql.AggrFuncExpr)
-			fe := afe.Args[0].(*metricsql.FuncExpr)
-			feIncrease := *fe
-			feIncrease.Name = "increase"
-			// copy RollupExpr to drop possible offset,
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-			newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-			newArg.Offset = nil
-			feIncrease.Args = []metricsql.Expr{newArg}
-			d := newArg.Window.Duration(ec.Step)
-			if d == 0 {
-				d = ec.Step
-			}
-			afeIncrease := *afe
-			afeIncrease.Args = []metricsql.Expr{&feIncrease}
-			be := &metricsql.BinaryOpExpr{
-				Op:              "/",
-				KeepMetricNames: true,
-				Left:            &afeIncrease,
-				Right: &metricsql.NumberExpr{
-					N: float64(d) / 1000,
-				},
-			}
-			return evalExpr(qt, ec, be)
-		}
-		qt.Printf("optimized calculation for instant rollup rate(m[d]) as (increase(m[d]) / d)")
-		fe := expr.(*metricsql.FuncExpr)
-		feIncrease := *fe
-		feIncrease.Name = "increase"
-		// copy RollupExpr to drop possible offset,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-		newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-		newArg.Offset = nil
-		feIncrease.Args = []metricsql.Expr{newArg}
-		d := newArg.Window.Duration(ec.Step)
-		if d == 0 {
-			d = ec.Step
-		}
-		be := &metricsql.BinaryOpExpr{
-			Op:              "/",
-			KeepMetricNames: fe.KeepMetricNames,
-			Left:            &feIncrease,
-			Right: &metricsql.NumberExpr{
-				N: float64(d) / 1000,
-			},
-		}
-		return evalExpr(qt, ec, be)
 	case "max_over_time":
 		if iafc != nil {
 			if !strings.EqualFold(iafc.ae.Name, "max") {

app/vmselect/promql/rollup_result_cache.go

@@ -132,7 +132,7 @@ func InitRollupResultCache(cachePath string) {
 		c = workingsetcache.New(cacheSize)
 		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 	}
-	if *disableCache {
+	if *disableCache && len(rollupResultCachePath) > 0 && !*resetRollupResultCacheOnStartup {
 		c.Reset()
 	}
 

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-zpalCSif.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-DY9kCvzk.js">
+  <script type="module" crossorigin src="./assets/index-Clpj_g75.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-D5YL0cqB.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-CBxdwuZH.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-jEWkrqzO.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/mergeset"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
@@ -90,6 +91,9 @@ var (
 		"In most cases, this value should not be changed. The maximum allowed value is 23h.")
 
 	logNewSeriesAuthKey = flagutil.NewPassword("logNewSeriesAuthKey", "authKey, which must be passed in query string to /internal/log_new_series. It overrides -httpAuth.*")
+
+	metadataStorageSize = flagutil.NewBytes("storage.maxMetadataStorageSize", 0, "Overrides max size for metrics metadata entries in-memory storage. "+
+		"If set to 0 or a negative value, defaults to 1% of allowed memory.")
 )
 
 // CheckTimeRange returns true if the given tr is denied for querying.
@@ -114,12 +118,13 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	}
 
 	resetResponseCacheIfNeeded = resetCacheIfNeeded
-	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
+	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
 	storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
 	storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
+	storage.SetMetadataStorageSize(metadataStorageSize.IntN())
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
 	mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())
@@ -194,6 +199,19 @@ func AddRows(mrs []storage.MetricRow) error {
 	return nil
 }
 
+// AddMetadataRows adds mrs to the storage.
+//
+// The caller should limit the number of concurrent calls to AddMetadataRows() in order to limit memory usage.
+func AddMetadataRows(mms []metricsmetadata.Row) error {
+	if Storage.IsReadOnly() {
+		return errReadOnly
+	}
+	WG.Add(1)
+	Storage.AddMetadataRows(mms)
+	WG.Done()
+	return nil
+}
+
 var errReadOnly = errors.New("the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value")
 
 // RegisterMetricNames registers all the metrics from mrs in the storage.
@@ -482,7 +500,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
 	tm := &m.TableMetrics
-	idbm := &m.IndexDBMetrics
+	idbm := &m.TableMetrics.IndexDBMetrics
 
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *DataPath), fs.MustGetFreeSpace(*DataPath))
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *DataPath), uint64(minFreeDiskSpaceBytes.N))
@@ -610,75 +628,82 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 
 	metrics.WriteCounterUint64(w, `vm_missing_metric_names_for_metric_id_total`, idbm.MissingMetricNamesForMetricID)
 
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_syncs_total`, m.DateMetricIDCacheSyncsCount)
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_resets_total`, m.DateMetricIDCacheResetsCount)
-
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/tsid"}`, m.TSIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricIDs"}`, m.MetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricName"}`, m.MetricNameCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/date_metricID"}`, m.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/metricID"}`, idbm.MetricIDCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/tsid"}`, m.TSIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, storage.RegexpCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/metricID"}`, idbm.MetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/date_metricID"}`, m.DateMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, uint64(storage.RegexpCacheSizeBytes()))
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSizeBytes()))
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/tsid"}`, m.TSIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, storage.RegexpCacheMaxSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMaxSizeBytes())
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, uint64(storage.RegexpCacheMaxSizeBytes()))
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheMaxSizeBytes()))
 
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/tsid"}`, m.TSIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricIDs"}`, m.MetricIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricName"}`, m.MetricNameCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/tsid"}`, m.TSIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricIDs"}`, m.MetricIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricName"}`, m.MetricNameCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 
-	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_resets_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheResets)
 
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/tsid"}`, m.TSIDCacheCollisions)
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/metricName"}`, m.MetricNameCacheCollisions)
 
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/metricID"}`, idbm.MetricIDCacheSyncsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSyncsCount)
+
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/metricID"}`, idbm.MetricIDCacheRotationsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheRotationsCount)
+
+	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
+
 	metrics.WriteGaugeUint64(w, `vm_next_retention_seconds`, m.NextRetentionSeconds)
 
 	if *trackMetricNamesStats {
@@ -689,6 +714,11 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled`, tm.ScheduledDownsamplingPartitions)
 	metrics.WriteGaugeUint64(w, `vm_downsampling_partitions_scheduled_size_bytes`, tm.ScheduledDownsamplingPartitionsSize)
+
+	metrics.WriteGaugeUint64(w, `vm_metrics_metadata_storage_items`, m.MetadataStorageItemsCurrent)
+	metrics.WriteCounterUint64(w, `vm_metrics_metadata_storage_size_bytes`, m.MetadataStorageCurrentSizeBytes)
+	metrics.WriteCounterUint64(w, `vm_metrics_metadata_storage_max_size_bytes`, m.MetadataStorageMaxSizeBytes)
+
 }
 
 func jsonResponseError(w http.ResponseWriter, err error) {

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.4 AS build-web-stage
+FROM golang:1.25.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/package-lock.json

@@ -5194,9 +5194,9 @@
       "license": "MIT"
     },
     "node_modules/js-yaml": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
+      "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -20,6 +20,7 @@ export interface ChartTooltipProps {
   info?: ReactNode;
   marker?: string;
   show?: boolean;
+  duplicateCount?: number;
   onClose?: (id: string) => void;
 }
 
@@ -35,6 +36,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
   statsFormatted,
   isSticky,
   marker,
+  duplicateCount = 0,
   onClose
 }) => {
   const tooltipRef = useRef<HTMLDivElement>(null);
@@ -156,6 +158,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
         <p className="vm-chart-tooltip-data__value">
           <b>{value}</b>{unit}
         </p>
+        {duplicateCount > 1 && <p>(overlapping points: {duplicateCount})</p>}
       </div>
       {statsFormatted && (
         <table className="vm-chart-tooltip-stats">

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -8,7 +8,8 @@ import { useHideDuplicateFields } from "./hooks/useHideDuplicateFields";
 import Accordion from "../../../Main/Accordion/Accordion";
 import { useLegendGroup } from "./hooks/useLegendGroup";
 import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
-import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
+import { LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import { getFromStorage } from "../../../../utils/storage";
 
 export type LegendProps = {
   labels: LegendItemType[];
@@ -38,17 +39,26 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
 
   const Content = isTableView ? LegendTable : LegendLines;
 
+  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false"
+  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT
+
+  const expandedWarning = (
+    <span className="vm-legend-group-header__warning">
+      Legend collapsed by default ({sortedLabels.length} series) — click to expand.
+    </span>
+  )
+
   return (
     <div
       className="vm-legend-group"
       key={group}
     >
       <Accordion
-        defaultExpanded={sortedLabels.length < DEFAULT_MAX_SERIES.chart}
+        defaultExpanded={defaultExpanded}
         title={(
           <div className="vm-legend-group-header">
             <div className="vm-legend-group-header-title">
-              Group by{groupByLabel ? "" : " query"}: <b>{group}</b>
+              Group by{groupByLabel ? "" : " query"}: <b>{group}</b> {!defaultExpanded && expandedWarning}
             </div>
             {!!duplicateFields.length && (
               <div className="vm-legend-group-header-labels">

app/vmui/packages/vmui/src/components/Chart/Line/Legend/style.scss

@@ -32,6 +32,14 @@
         }
       }
 
+      &__warning {
+        flex-grow: 1;
+        text-align: right;
+        padding-right: calc($padding-large * 2);
+        font-size: $font-size-small;
+        color: $color-warning;
+      }
+
       &-labels {
         display: flex;
         flex-wrap: wrap;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -1,15 +1,17 @@
-import { forwardRef, useCallback, useImperativeHandle, useState } from "preact/compat";
+import { forwardRef, useCallback, useEffect, useImperativeHandle, useState } from "preact/compat";
 import { DisplayType, ErrorTypes } from "../../../../types";
 import TextField from "../../../Main/TextField/TextField";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import { InfoIcon, RestartIcon } from "../../../Main/Icons";
 import Button from "../../../Main/Button/Button";
-import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
+import { DEFAULT_MAX_SERIES, LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../../state/customPanel/CustomPanelStateContext";
+import Switch from "../../../Main/Switch/Switch";
+import { getFromStorage, saveToStorage } from "../../../../utils/storage";
 
 interface ServerConfiguratorProps {
   onClose: () => void
@@ -27,6 +29,9 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   const { seriesLimits } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
+  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE")
+  const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
+
   const [limits, setLimits] = useState(seriesLimits);
   const [error, setError] = useState({
     table: "",
@@ -52,6 +57,10 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     onClose();
   }, [limits]);
 
+  useEffect(() => {
+    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`)
+  }, [legendCollapse]);
+
   useImperativeHandle(ref, () => ({ handleApply }), [handleApply]);
 
   return (
@@ -97,6 +106,19 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
           </div>
         ))}
       </div>
+
+      <div className="vm-graph-settings-row">
+        <span className="vm-graph-settings-row__label">Auto-collapse legend</span>
+        <Switch
+          value={legendCollapse}
+          onChange={setLegendCollapse}
+          label={legendCollapse ? "Enabled" : "Disabled"}
+          fullWidth={isMobile}
+        />
+        <span className="vm-legend-configs-item__info">
+          Collapses the legend when series count exceeds {LEGEND_COLLAPSE_SERIES_LIMIT} to reduce UI load.
+        </span>
+      </div>
     </div>
   );
 });

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/style.scss

@@ -18,6 +18,7 @@
     align-items: center;
     justify-content: space-between;
     gap: $padding-global;
+    margin-bottom: $padding-global;
 
     &_mobile {
       gap: $padding-small;

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/index.tsx

@@ -132,7 +132,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
                 <th>Series returned</th>
                 <th>Series fetched</th>
                 <th>Duration</th>
-                <th>Executed at</th>
+                <th>Execution timestamp</th>
               </tr>
             </thead>
             <tbody>
@@ -154,7 +154,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
       {!!item?.alerts?.length && (
         <>
           <span className="vm-alerts-title">Alerts</span>
-          <table>
+          <table className="vm-alerts-table">
             <colgroup>
               <col className="vm-col-sm"/>
               <col className="vm-col-sm"/>
@@ -190,7 +190,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
                   </td>
                   <td>
                     <Badges
-                      align="center"
+                      align="start"
                       items={Object.fromEntries(Object.entries(alert.labels || {}).map(([name, value]) => [name, {
                         color: "passive",
                         value: value,

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/style.scss

@@ -44,6 +44,7 @@
     word-break: break-word;
     table-layout: fixed;
     width: 100%;
+
     td, th {
       line-height: 30px;
       padding: 4px $padding-small;
@@ -52,15 +53,33 @@
       overflow: hidden;
       text-overflow: ellipsis;
     }
+
     th {
       white-space: nowrap;
     }
+
     td.align-center {
       text-align: center
     }
+
     th {
       font-weight: bold;
       padding: 0 $padding-small;
     }
   }
+
+  .vm-alerts-table {
+    tr {
+      border-bottom: $border-divider;
+
+      &:hover {
+        background: $color-background-hover;
+      }
+    }
+
+    td {
+      vertical-align: top;
+      padding-block: $padding-small;
+    }
+  }
 }

app/vmui/packages/vmui/src/components/ExploreAlerts/NotifiersHeader/style.scss

@@ -46,9 +46,6 @@
     .vm-text-field__input {
       padding: 11px 28px;
     }
-    .vm-text-field__icon-start {
-      height: 42px;
-    }
   }
 
   &__clear-icon {

app/vmui/packages/vmui/src/components/ExploreAlerts/Rule/style.scss

@@ -8,7 +8,7 @@
   flex-direction: column;
   position: relative;
 
-  &:has(>details[open]) {
+  &:has(>.vm-accordion-header_open) {
     background-color: $color-background-item;
   }
 

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/index.tsx

@@ -61,7 +61,7 @@ const RulesHeader: FC<RulesHeaderProps> = ({
             value={states}
             list={allStates}
             label="State"
-            placeholder="Please rule state"
+            placeholder="Please select rule state"
             onChange={onChangeStates}
             noOptionsText={noStateText}
             includeAll

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/style.scss

@@ -26,9 +26,6 @@
     .vm-text-field__input {
       padding: 11px 28px;
     }
-    .vm-text-field__icon-start {
-      height: 42px;
-    }
   }
 
   &__clear-icon {

app/vmui/packages/vmui/src/components/ExploreAlerts/Target/style.scss

@@ -34,7 +34,7 @@
   position: relative;
   border-radius: $border-radius-small;
 
-  &:has(>details[open]) {
+  &:has(>.vm-accordion-header_open) {
     background-color: $color-background-item;
   }
 

app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricGraph/ExploreMetricItemGraph.tsx

@@ -43,7 +43,7 @@ const ExploreMetricItem: FC<ExploreMetricItemGraphProps> = ({
   const step = isHeatmap && customStep === defaultStep ? heatmapStep : customStep;
 
 
-  const query = useMemo(() => {
+  const queries = useMemo(() => {
     const params = Object.entries({ job, instance })
       .filter(val => val[1])
       .map(([key, val]) => `${key}=${JSON.stringify(val)}`);
@@ -55,19 +55,19 @@ const ExploreMetricItem: FC<ExploreMetricItemGraphProps> = ({
 
     const base = `{${params.join(",")}}`;
     if (isBucket) {
-      return `sum(rate(${base})) by (vmrange, le)`;
+      return [`sum(rate(${base})) by (vmrange, le)`];
     }
     const queryBase = rateEnabled ? `rollup_rate(${base})` : `rollup(${base})`;
-    return `
+    return [`
 with (q = ${queryBase}) (
   alias(min(label_match(q, "rollup", "min")), "min"),
   alias(max(label_match(q, "rollup", "max")), "max"),
   alias(avg(label_match(q, "rollup", "avg")), "avg"),
-)`;
+)`];
   }, [name, job, instance, rateEnabled, isBucket]);
 
   const { isLoading, graphData, error, queryErrors, warning, isHistogram } = useFetchQuery({
-    predefinedQuery: [query],
+    predefinedQuery: queries,
     visible: true,
     customStep: step,
     showAllSeries
@@ -98,7 +98,7 @@ with (q = ${queryBase}) (
       {warning && (
         <WarningLimitSeries
           warning={warning}
-          query={[query]}
+          query={queries}
           onChange={setShowAllSeries}
         />
       )}
@@ -107,7 +107,7 @@ with (q = ${queryBase}) (
           data={graphData}
           period={period}
           customStep={step}
-          query={[query]}
+          query={queries}
           yaxis={yaxis}
           setYaxisLimits={setYaxisLimits}
           setPeriod={setPeriod}

app/vmui/packages/vmui/src/components/Main/Accordion/Accordion.tsx

@@ -1,4 +1,5 @@
 import { FC, useState, useEffect } from "preact/compat";
+import classNames from "classnames";
 import { JSX } from "preact";
 import { ArrowDownIcon } from "../Icons";
 import "./style.scss";
@@ -31,9 +32,12 @@ const Accordion: FC<AccordionProps> = ({
       event.preventDefault();
       return; // If the text is selected, cancel the execution of toggle.
     }
-    const details = event.currentTarget.parentElement as HTMLDetailsElement;
-    onChange && onChange(details.open);
-    setIsOpen(details.open);
+
+    setIsOpen((prev) => {
+      const newState = !prev;
+      onChange && onChange(newState);
+      return newState;
+    });
   };
 
   useEffect(() => {
@@ -42,23 +46,32 @@ const Accordion: FC<AccordionProps> = ({
 
   return (
     <>
-      <details
-        className="vm-accordion-section"
-        key="content"
-        open={isOpen}
+      <header
+        className={classNames({
+          "vm-accordion-header": true,
+          "vm-accordion-header_open": isOpen,
+        })}
+        onClick={toggleOpen}
         id={id}
       >
-        <summary
-          className="vm-accordion-header"
-          onClick={toggleOpen}
+        {title}
+        <div
+          className={classNames({
+            "vm-accordion-header__arrow": true,
+            "vm-accordion-header__arrow_open": isOpen,
+          })}
         >
-          {title}
-          <div className="vm-accordion-header__arrow">
-            <ArrowDownIcon />
-          </div>
-        </summary>
-        {children}
-      </details>
+          <ArrowDownIcon />
+        </div>
+      </header>
+      {isOpen && (
+        <section
+          className="vm-accordion-section"
+          key="content"
+        >
+          {children}
+        </section>
+      )}
     </>
   );
 };

app/vmui/packages/vmui/src/components/Main/Accordion/style.scss

@@ -17,6 +17,10 @@
     transform: rotate(0);
     transition: transform 200ms ease-in-out;
 
+    &_open {
+      transform: rotate(180deg);
+    }
+
     svg {
       width: 14px;
       height: auto;
@@ -24,14 +28,6 @@
   }
 }
 
-.vm-accordion-section[open] > summary {
-  & > .vm-accordion-header {
-    &__arrow {
-      transform: rotate(180deg);
-    }
-  }
-}
-
 .accordion-section {
   overflow: hidden;
 }

app/vmui/packages/vmui/src/components/Main/Select/Select.tsx

@@ -137,6 +137,7 @@ const Select: FC<SelectProps> = ({
         "vm-select_disabled": disabled
       })}
     >
+      {label && <span className="vm-text-field__label">{label}</span>}
       <div
         className="vm-select-input"
         onClick={handleToggleList}
@@ -150,7 +151,7 @@ const Select: FC<SelectProps> = ({
               onRemoveItem={handleSelected}
             />
           )}
-          {!hideInput && !selectedValues?.length && (
+          {!hideInput && (
             <input
               value={textFieldValue}
               type="text"
@@ -164,7 +165,6 @@ const Select: FC<SelectProps> = ({
             />
           )}
         </div>
-        {label && <span className="vm-text-field__label">{label}</span>}
         {clearable && value && (
           <div
             className="vm-select-input__icon"

app/vmui/packages/vmui/src/components/Main/Select/style.scss

@@ -1,6 +1,8 @@
 @use "src/styles/variables" as *;
 
 .vm-select {
+  position: relative;
+  display: grid;
   &-input {
     position: relative;
     display: flex;

app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx

@@ -89,8 +89,8 @@ const GraphView: FC<GraphViewProps> = ({
   const [legendValue, setLegendValue] = useState<ChartTooltipProps | null>(null);
 
   const getSeriesItem = useMemo(() => {
-    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isAnomalyView);
-  }, [data, hideSeries, alias, showAllPoints, isAnomalyView]);
+    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isAnomalyView, isRawQuery);
+  }, [data, hideSeries, alias, showAllPoints, isAnomalyView, isRawQuery]);
 
   const setLimitsYaxis = (minVal: number, maxVal: number) => {
     let min = Number.isFinite(minVal) ? minVal : 0;
@@ -144,8 +144,8 @@ const GraphView: FC<GraphViewProps> = ({
   useEffect(() => {
     const dLen = data.length;
 
-    const tsAnchor = data?.[0]?.values?.[0]?.[0]
-    const tsSet = new Set<number>([])
+    const tsAnchor = data?.[0]?.values?.[0]?.[0];
+    const tsArray: number[] = [];
     const tempLegend = new Array<LegendItemType>(dLen);
     const tempSeries = new Array<uPlotSeries>(dLen + 1);
     tempSeries[0] = {};
@@ -162,7 +162,7 @@ const GraphView: FC<GraphViewProps> = ({
       const vals = d.values;
       for (let j = 0, vLen = vals.length; j < vLen; j++) {
         const v = vals[j];
-        if (isRawQuery) tsSet.add(v[0])
+        if (isRawQuery) tsArray.push(v[0]);
         const num = promValueToNumber(v[1]);
         if (Number.isFinite(num)) {
           if (num < minVal) minVal = num;
@@ -171,12 +171,12 @@ const GraphView: FC<GraphViewProps> = ({
       }
     }
 
-    const dpr = window.devicePixelRatio || 1
+    const dpr = window.devicePixelRatio || 1;
     const widthPx = containerSize.width || window.innerWidth || 4096;
     const pixels = Math.max(1, Math.floor(widthPx * Math.max(1, dpr)));
 
     const timeSeries = isRawQuery
-      ? Array.from(tsSet).sort((a,b) => a - b)
+      ? tsArray.sort((a, b) => a - b)
       : getTimeSeries(currentStep, period, pixels, tsAnchor);
 
     const timeDataSeries: (number | null)[][] = data.map(d => {
@@ -195,6 +195,8 @@ const GraphView: FC<GraphViewProps> = ({
           // Treat special values as nulls in order to satisfy uPlot.
           // Otherwise it may draw unexpected graphs.
           v = Number.isFinite(num) ? num : null;
+          // Advance to next value
+          j++;
         }
         results[k] = v;
       }
@@ -281,7 +283,7 @@ const GraphView: FC<GraphViewProps> = ({
           height={height}
           isAnomalyView={isAnomalyView}
           spanGaps={spanGaps}
-          showAllPoints={showAllPoints}
+          showAllPoints={isRawQuery ? true : showAllPoints}
         />
       )}
       {isHistogram && (

app/vmui/packages/vmui/src/constants/graph.ts

@@ -8,6 +8,8 @@ export const DEFAULT_MAX_SERIES = {
   code: 1000,
 };
 
+export const LEGEND_COLLAPSE_SERIES_LIMIT = 100;
+
 export const GRAPH_SIZES: GraphSize[] = [
   {
     id: "small",

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -49,6 +49,26 @@ const useLineTooltip = ({ u, metrics, series, unit, isAnomalyView }: LineTooltip
     const max = u?.scales?.[1]?.max || 1;
     const date = u?.data?.[0]?.[dataIdx] || 0;
 
+    let duplicateCount = 1;
+
+    if (u && seriesIdx > 0 && dataIdx >= 0) {
+      const xs = u.data[0] as (number | null)[];
+      const ys = u.data[seriesIdx] as (number | null)[];
+
+      const xVal = xs[dataIdx];
+      const yVal = ys[dataIdx];
+
+      if (xVal != null && yVal != null) {
+        duplicateCount = 0;
+
+        for (let i = 0; i < xs.length; i++) {
+          if (xs[i] === xVal && ys[i] === yVal) {
+            duplicateCount++;
+          }
+        }
+      }
+    }
+
     const point = {
       top: u ? u.valToPos((value || 0), seriesItem?.scale || "1") : 0,
       left: u ? u.valToPos(date, "x") : 0,
@@ -65,6 +85,7 @@ const useLineTooltip = ({ u, metrics, series, unit, isAnomalyView }: LineTooltip
       info: getMetricName(metricItem, seriesItem),
       statsFormatted: seriesItem?.statsFormatted,
       marker: `${seriesItem?.stroke}`,
+      duplicateCount,
     };
   }, [u, tooltipIdx, metrics, series, unit, isAnomalyView]);
 

app/vmui/packages/vmui/src/pages/ExploreAlerts/ExploreNotifiers.tsx

@@ -1,5 +1,4 @@
-import { FC, useEffect, useState } from "preact/compat";
-import { useLocation } from "react-router";
+import { FC, useState } from "preact/compat";
 import { useNotifiersSetQueryParams as useSetQueryParams } from "./hooks/useSetQueryParams";
 import Spinner from "../../components/Main/Spinner/Spinner";
 import Alert from "../../components/Main/Alert/Alert";
@@ -33,37 +32,6 @@ const ExploreNotifiers: FC = () => {
     search: searchInput,
   });
 
-  const location = useLocation();
-  const pageLoaded = !isLoading && !error && !!notifiers?.length;
-  const savedScrollTop = localStorage.getItem("scrollTop");
-
-  useEffect(() => {
-    if (!pageLoaded) return;
-    if (location.hash) {
-      const target = document.querySelector(location.hash);
-      if (target) {
-        let parent = target.closest("details");
-        while (parent) {
-          parent.open = true;
-          if (!parent?.parentElement) return;
-          parent = parent.parentElement.closest("details");
-        }
-        target.scrollIntoView();
-      }
-    } else {
-      if (savedScrollTop) {
-        window.scrollTo(0, parseInt(savedScrollTop));
-      }
-      const handleBeforeUnload = () => {
-        localStorage.setItem("scrollTop", (window.scrollY || 0).toString());
-      };
-      window.addEventListener("beforeunload", handleBeforeUnload);
-      return () => {
-        window.removeEventListener("beforeunload", handleBeforeUnload);
-      };
-    }
-  }, [location, savedScrollTop, pageLoaded]);
-
   const handleChangeSearch = (input: string) => {
     if (!input) {
       setSearchInput("");

app/vmui/packages/vmui/src/pages/ExploreAlerts/ExploreRules.tsx

@@ -1,5 +1,5 @@
 import { FC, useEffect, useMemo, useState, useCallback } from "preact/compat";
-import { useNavigate, useLocation, useSearchParams } from "react-router";
+import { useSearchParams } from "react-router";
 import { useRulesSetQueryParams as useSetQueryParams } from "./hooks/useSetQueryParams";
 import Spinner from "../../components/Main/Spinner/Spinner";
 import Alert from "../../components/Main/Alert/Alert";
@@ -33,16 +33,9 @@ const ExploreRules: FC = () => {
   const [modalOpen, setModalOpen] = useState(true);
   const [searchParams, setSearchParams] = useSearchParams();
 
-  const navigate = useNavigate();
-  const location = useLocation();
-
   useEffect(() => {
-    if (!location.hash && groupId) {
-      setModalOpen(true);
-    } else {
-      setModalOpen(false);
-    }
-  }, [location.hash, groupId]);
+    setModalOpen(!!groupId);
+  }, [groupId]);
 
   useSetQueryParams({
     types: types.join("&"),
@@ -62,29 +55,29 @@ const ExploreRules: FC = () => {
   }, [searchInput]);
 
   const getModal = () => {
-    if (ruleId !== "") {
+    if (ruleId) {
       return (
         <ExploreRule
           groupId={groupId}
           id={ruleId}
-          mode={ruleId !== "" ? "rule" : "alert"}
-          onClose={handleClose(`rule-${ruleId}`)}
+          mode={ruleId ? "rule" : "alert"}
+          onClose={handleClose}
         />
       );
-    } else if (alertId !== "") {
+    } else if (alertId) {
       return (
         <ExploreAlert
           groupId={groupId}
           id={alertId}
-          mode={ruleId !== "" ? "rule" : "alert"}
-          onClose={handleClose(`alert-${alertId}`)}
+          mode={ruleId ? "rule" : "alert"}
+          onClose={handleClose}
         />
       );
-    } else if (groupId !== "") {
+    } else if (groupId) {
       return (
         <ExploreGroup
           id={groupId}
-          onClose={handleClose(`group-${groupId}`)}
+          onClose={handleClose}
         />
       );
     }
@@ -92,18 +85,13 @@ const ExploreRules: FC = () => {
 
   const noRuleFound = "No rules found!";
 
-  const handleClose = (id: string) => {
-    return () => {
-      const newParams = new URLSearchParams(searchParams);
-      newParams.delete("group_id");
-      newParams.delete("rule_id");
-      newParams.delete("alert_id");
-      setSearchParams(newParams);
-      setModalOpen(false);
-      navigate({
-        hash: `#${id}`,
-      });
-    };
+  const handleClose = () => {
+    const newParams = new URLSearchParams(searchParams);
+    newParams.delete("group_id");
+    newParams.delete("rule_id");
+    newParams.delete("alert_id");
+    setSearchParams(newParams);
+    setModalOpen(false);
   };
 
   const {
@@ -112,36 +100,6 @@ const ExploreRules: FC = () => {
     error,
   } = useFetchGroups({ blockFetch: modalOpen });
 
-  const pageLoaded = !isLoading && !error && !!groups?.length;
-  const savedScrollTop = localStorage.getItem("scrollTop");
-
-  useEffect(() => {
-    if (!pageLoaded) return;
-    if (location.hash) {
-      const target = document.querySelector(location.hash);
-      if (target) {
-        let parent = target.closest("details");
-        while (parent) {
-          parent.open = true;
-          if (!parent?.parentElement) return;
-          parent = parent.parentElement.closest("details");
-        }
-        target.scrollIntoView();
-      }
-    } else {
-      if (savedScrollTop) {
-        window.scrollTo(0, parseInt(savedScrollTop));
-      }
-      const updateScrollPosition = () => {
-        localStorage.setItem("scrollTop", (window.scrollY || 0).toString());
-      };
-      window.addEventListener("scroll", updateScrollPosition);
-      return () => {
-        window.removeEventListener("scroll", updateScrollPosition);
-      };
-    }
-  }, [location, savedScrollTop, pageLoaded]);
-
   const { filteredGroups, allTypes, allStates } = useMemo(
     () => filterGroups(groups || [], types, states, searchInput),
     [groups, types, states, searchInput]

app/vmui/packages/vmui/src/pages/ExploreAlerts/style.scss

@@ -1,11 +1,8 @@
 @use "src/styles/variables" as *;
 
 .vm-explore-alert-group {
+  content-visibility: auto;
   width: 100%;
-  &:has(.vm-accordion-header_open) {
-    border: $border-divider;
-    border-radius: $border-radius-small;
-  }
 }
 
 .vm-explore-alerts.vm-modal {

app/vmui/packages/vmui/src/router/index.ts

@@ -71,7 +71,15 @@ export const routerOptions: { [key: string]: RouterOptions } = {
   [router.home]: getDefaultOptions(APP_TYPE),
   [router.rawQuery]: {
     title: "Raw query",
-    ...routerOptionsDefault,
+    header: {
+      tenant: true,
+      stepControl: false,
+      timeSelector: true,
+      executionControls: {
+        tooltip: "Refresh dashboard",
+        useAutorefresh: true,
+      }
+    },
   },
   [router.metrics]: {
     title: "Explore Prometheus metrics",

app/vmui/packages/vmui/src/utils/default-server-url.test.ts

@@ -20,7 +20,7 @@ describe("test server urls", () => {
 
     it("https://play.vm.com/#/rules?q=test", () => {
       const result = getDefaultURL("https://play.vm.com/#/rules?q=test");
-      expect(result).toBe("https://play.vm.com");
+      expect(result).toBe("https://play.vm.com/prometheus");
     });
   });
 });

app/vmui/packages/vmui/src/utils/default-server-url.ts

@@ -4,7 +4,7 @@ import { APP_TYPE, AppType } from "../constants/appType";
 import { getFromStorage } from "./storage";
 
 export const getDefaultURL = (u: string) => {
-  return u.replace(/(\/(?:prometheus\/)?(?:graph|vmui)\/.*|\/#\/.*)/, "").replace(/(\/select\/[^/]+)$/, "$1/prometheus");
+  return u.replace(/(\/(?:prometheus\/)?(?:graph|vmui)\/.*|\/#\/.*)/, "/prometheus");
 };
 
 export const getDefaultServer = (tenantId?: string): string => {

app/vmui/packages/vmui/src/utils/storage.ts

@@ -8,6 +8,7 @@ export type StorageKeys = "AUTOCOMPLETE"
   | "NO_CACHE"
   | "QUERY_TRACING"
   | "SERIES_LIMITS"
+  | "LEGEND_AUTO_COLLAPSE"
   | "TABLE_COMPACT"
   | "TIMEZONE"
   | "DISABLED_DEFAULT_TIMEZONE"

app/vmui/packages/vmui/src/utils/uplot/scatter.ts

@@ -0,0 +1,111 @@
+import uPlot, { OrientCallback } from "uplot";
+
+const deg360 = 2 * Math.PI;
+
+// Base point size multiplier (in device pixels)
+const BASE_POINT_SIZE = 4;
+
+// Square size scale relative to circle size
+const SQUARE_SIZE_SCALE = 1.2;
+
+export const drawPoints = (u: uPlot, seriesIdx: number) => {
+  const size = BASE_POINT_SIZE * uPlot.pxRatio;
+  const r = size / 2;
+  const squareSize = size * SQUARE_SIZE_SCALE;
+  const squareHalf = squareSize / 2;
+
+  const orientCallback: OrientCallback = (
+    series,
+    dataX,
+    dataY,
+    scaleX,
+    scaleY,
+    valToPosX,
+    valToPosY,
+    xOff,
+    yOff,
+    xDim,
+    yDim,
+    _moveTo,
+    _lineTo,
+    rect,
+    arc,
+  ) => {
+    const stroke = series?.stroke as unknown;
+    if (typeof stroke === "function") {
+      u.ctx.fillStyle = (stroke as () => string)();
+    }
+
+    const circlesPath = new Path2D();
+    const squaresPath = new Path2D();
+
+    const xMin = Number(scaleX.min);
+    const xMax = Number(scaleX.max);
+    const yMin = Number(scaleY.min);
+    const yMax = Number(scaleY.max);
+
+    const counts = new Map<string, number>();
+    const len = dataX.length;
+
+    for (let i = 0; i < len; i++) {
+      const xv = dataX[i];
+      const yv = dataY[i];
+
+      if (xv == null || yv == null) continue;
+
+      const xVal = Number(xv);
+      const yVal = Number(yv);
+
+      if (!Number.isFinite(xVal) || !Number.isFinite(yVal)) continue;
+
+      const key = `${xVal}|${yVal}`;
+      counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+
+    const duplicates = new Set<string>();
+    for (const [key, count] of counts) {
+      if (count > 1) duplicates.add(key);
+    }
+
+    for (let i = 0; i < len; i++) {
+      const xv = dataX[i];
+      const yv = dataY[i];
+
+      if (xv == null || yv == null) continue;
+
+      const xVal = Number(xv);
+      const yVal = Number(yv);
+
+      if (
+        !Number.isFinite(xVal) ||
+        !Number.isFinite(yVal) ||
+        xVal < xMin || xVal > xMax ||
+        yVal < yMin || yVal > yMax
+      ) {
+        continue;
+      }
+
+      const cx = valToPosX(xVal, scaleX, xDim, xOff);
+      const cy = valToPosY(yVal, scaleY, yDim, yOff);
+
+      const key = `${xVal}|${yVal}`;
+      const isDuplicate = duplicates.has(key);
+
+      if (isDuplicate) {
+        rect(squaresPath, cx - squareHalf, cy - squareHalf, squareSize, squareSize);
+      } else {
+        circlesPath.moveTo(cx + r, cy);
+        arc(circlesPath, cx, cy, r, 0, deg360);
+      }
+    }
+
+    u.ctx.fill(circlesPath);
+    u.ctx.lineWidth = 1.4 * uPlot.pxRatio;
+    u.ctx.strokeStyle = u.ctx.fillStyle;
+    u.ctx.stroke(squaresPath);
+  };
+
+  uPlot.orient(u, seriesIdx, orientCallback);
+
+  return null;
+};

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -5,6 +5,7 @@ import { ForecastType, HideSeriesArgs, LegendItemType, SeriesItem } from "../../
 import { anomalyColors, baseContrastColors, getColorFromString } from "../color";
 import { getMathStats } from "../math";
 import { formatPrettyNumber } from "./helpers";
+import { drawPoints } from "./scatter";
 
 // Helper function to extract freeFormFields values as a comma-separated string
 export const extractFields = (metric: MetricBase["metric"]): string => {
@@ -31,7 +32,7 @@ export const isForecast = (metric: MetricBase["metric"]): ForecastMetricInfo =>
   };
 };
 
-export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[], alias: string[], showPoints?: boolean, isAnomalyUI?: boolean) => {
+export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[], alias: string[], showPoints?: boolean, isAnomalyUI?: boolean, isRawQuery?: boolean) => {
   const colorState: {[key: string]: string} = {};
   const maxColors = isAnomalyUI ? 0 : Math.min(data.length, baseContrastColors.length);
 
@@ -51,13 +52,14 @@ export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[],
       dash: getDashSeries(metricInfo),
       width: getWidthSeries(metricInfo),
       stroke: getStrokeSeries({ metricInfo, label, isAnomalyUI, colorState }),
-      points: getPointsSeries(metricInfo, showPoints),
+      points: getPointsSeries(metricInfo, showPoints, isRawQuery),
       spanGaps: false,
       forecast: metricInfo?.value,
       forecastGroup: metricInfo?.group,
       freeFormFields: d.metric,
       show: !includesHideSeries(label, hideSeries),
       scale: "1",
+      paths: isRawQuery ? drawPoints : undefined,
       ...getSeriesStatistics(d),
     };
   };
@@ -118,10 +120,10 @@ export const delSeries = (u: uPlot) => {
   }
 };
 
-export const addSeries = (u: uPlot, series: uPlotSeries[], spanGaps = false, showPoints = false) => {
+export const addSeries = (u: uPlot, series: uPlotSeries[], spanGaps = false, showPoints = false, isRawQuery?: boolean) => {
   series.forEach((s,i) => {
     if (s.label) s.spanGaps = spanGaps;
-    if (s.points) s.points.filter = showPoints ? undefined : filterPoints;
+    if (s.points) s.points.filter = showPoints || isRawQuery ? undefined : filterPoints;
     i && u.addSeries(s);
   });
 };
@@ -157,17 +159,17 @@ const getWidthSeries = (metricInfo: ForecastMetricInfo | null): number => {
   return 1.4;
 };
 
-const getPointsSeries = (metricInfo: ForecastMetricInfo | null, showPoints: boolean = false): uPlotSeries.Points => {
+const getPointsSeries = (metricInfo: ForecastMetricInfo | null, showPoints: boolean = false, isRawQuery?: boolean): uPlotSeries.Points => {
   const isAnomalyMetric = metricInfo?.value === ForecastType.anomaly;
 
   if (isAnomalyMetric) {
     return { size: 8, width: 4, space: 0 };
   }
   return {
-    size: 4,
+    size: isRawQuery ? 0 : 4,
     width: 0,
     show: true,
-    filter: showPoints ? null : filterPoints,
+    filter: showPoints || isRawQuery ? null : filterPoints,
   };
 };
 

apptest/model.go

@@ -25,6 +25,7 @@ type PrometheusQuerier interface {
 	PrometheusAPIV1Labels(t *testing.T, query string, opts QueryOpts) *PrometheusAPIV1LabelsResponse
 	PrometheusAPIV1LabelValues(t *testing.T, labelName, query string, opts QueryOpts) *PrometheusAPIV1LabelValuesResponse
 	PrometheusAPIV1ExportNative(t *testing.T, query string, opts QueryOpts) []byte
+	PrometheusAPIV1Metadata(t *testing.T, metric string, limit int, opts QueryOpts) *PrometheusAPIV1Metadata
 
 	APIV1AdminTSDBDeleteSeries(t *testing.T, matchQuery string, opts QueryOpts)
 
@@ -37,7 +38,7 @@ type PrometheusQuerier interface {
 // Writer contains methods for writing new data
 type Writer interface {
 	// Prometheus APIs
-	PrometheusAPIV1Write(t *testing.T, records []prompb.TimeSeries, opts QueryOpts)
+	PrometheusAPIV1Write(t *testing.T, wr prompb.WriteRequest, opts QueryOpts)
 	PrometheusAPIV1ImportPrometheus(t *testing.T, records []string, opts QueryOpts)
 	PrometheusAPIV1ImportCSV(t *testing.T, records []string, opts QueryOpts)
 	PrometheusAPIV1ImportNative(t *testing.T, data []byte, opts QueryOpts)
@@ -350,6 +351,33 @@ func NewPrometheusAPIV1LabelValuesResponse(t *testing.T, s string) *PrometheusAP
 	return res
 }
 
+// PrometheusAPIV1Metadata is an inmemory representation of the
+// /prometheus/api/v1/metadata response.
+type PrometheusAPIV1Metadata struct {
+	Status    string
+	IsPartial bool
+	Data      map[string][]MetadataEntry
+	Trace     *Trace
+}
+
+type MetadataEntry struct {
+	Type string
+	Help string
+	Unit string
+}
+
+// NewPrometheusAPIV1Metadata is a test helper function that creates a new
+// instance of PrometheusAPIV1Metadata by unmarshalling a json string.
+func NewPrometheusAPIV1Metadata(t *testing.T, s string) *PrometheusAPIV1Metadata {
+	t.Helper()
+
+	res := &PrometheusAPIV1Metadata{}
+	if err := json.Unmarshal([]byte(s), res); err != nil {
+		t.Fatalf("could not unmarshal series response data:\n%s\n err: %v", string(s), err)
+	}
+	return res
+}
+
 // Trace provides the description and the duration of some unit of work that has
 // been performed during the request processing.
 type Trace struct {

apptest/tests/dedup_test.go

@@ -99,37 +99,39 @@ func testDeduplication(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier,
 	ts3 := start.Add(3 * time.Second).UnixMilli()
 	ts5 := start.Add(5 * time.Second).UnixMilli()
 	ts10 := start.Add(10 * time.Second).UnixMilli()
-	data := []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{{Name: "__name__", Value: "metric1"}},
-			Samples: []prompb.Sample{
-				{Timestamp: ts1, Value: 3},
-				{Timestamp: ts3, Value: 10},
-				{Timestamp: ts5, Value: 5},
+	data := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{{Name: "__name__", Value: "metric1"}},
+				Samples: []prompb.Sample{
+					{Timestamp: ts1, Value: 3},
+					{Timestamp: ts3, Value: 10},
+					{Timestamp: ts5, Value: 5},
+				},
 			},
-		},
-		{
-			Labels: []prompb.Label{{Name: "__name__", Value: "metric2"}},
-			Samples: []prompb.Sample{
-				{Timestamp: ts1, Value: 3},
-				{Timestamp: ts3, Value: decimal.StaleNaN},
-				{Timestamp: ts5, Value: 5},
+			{
+				Labels: []prompb.Label{{Name: "__name__", Value: "metric2"}},
+				Samples: []prompb.Sample{
+					{Timestamp: ts1, Value: 3},
+					{Timestamp: ts3, Value: decimal.StaleNaN},
+					{Timestamp: ts5, Value: 5},
+				},
 			},
-		},
-		{
-			Labels: []prompb.Label{{Name: "__name__", Value: "metric3"}},
-			Samples: []prompb.Sample{
-				{Timestamp: ts10, Value: 30},
-				{Timestamp: ts10, Value: 100},
-				{Timestamp: ts10, Value: 50},
+			{
+				Labels: []prompb.Label{{Name: "__name__", Value: "metric3"}},
+				Samples: []prompb.Sample{
+					{Timestamp: ts10, Value: 30},
+					{Timestamp: ts10, Value: 100},
+					{Timestamp: ts10, Value: 50},
+				},
 			},
-		},
-		{
-			Labels: []prompb.Label{{Name: "__name__", Value: "metric4"}},
-			Samples: []prompb.Sample{
-				{Timestamp: ts10, Value: 30},
-				{Timestamp: ts10, Value: decimal.StaleNaN},
-				{Timestamp: ts10, Value: 50},
+			{
+				Labels: []prompb.Label{{Name: "__name__", Value: "metric4"}},
+				Samples: []prompb.Sample{
+					{Timestamp: ts10, Value: 30},
+					{Timestamp: ts10, Value: decimal.StaleNaN},
+					{Timestamp: ts10, Value: 50},
+				},
 			},
 		},
 	}

apptest/tests/ingestprotocols_test.go

@@ -158,7 +158,11 @@ func TestSingleIngestionProtocols(t *testing.T) {
 
 	// prometheus text exposition format
 	sut.PrometheusAPIV1ImportPrometheus(t, []string{
-		`importprometheus_series 10 1707123456700`,                               // 2024-02-05T08:57:36.700Z
+		`# HELP importprometheus_series some help message`,
+		`# TYPE importprometheus_series gauge`,
+		`importprometheus_series 10 1707123456700`, // 2024-02-05T08:57:36.700Z
+		`# HELP importprometheus_series2 some help message second one`,
+		`# TYPE importprometheus_series2 gauge`,
 		`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
 	}, apptest.QueryOpts{
 		ExtraLabels: []string{"el1=elv1", "el2=elv2"},
@@ -187,42 +191,58 @@ func TestSingleIngestionProtocols(t *testing.T) {
 	})
 
 	// prometheus remote write format
-	pbData := []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{
-				{
-					Name:  "__name__",
-					Value: "prometheusrw_series",
+	pbData := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{
+						Name:  "__name__",
+						Value: "prometheusrw_series",
+					},
+				},
+				Samples: []prompb.Sample{
+					{
+						Value:     10,
+						Timestamp: 1707123456700, // 2024-02-05T08:57:36.700Z
+
+					},
 				},
 			},
-			Samples: []prompb.Sample{
-				{
-					Value:     10,
-					Timestamp: 1707123456700, // 2024-02-05T08:57:36.700Z
-
+			{
+				Labels: []prompb.Label{
+					{
+						Name:  "__name__",
+						Value: "prometheusrw_series2",
+					},
+					{
+						Name:  "label",
+						Value: "foo2",
+					},
+					{
+						Name:  "label1",
+						Value: "value1",
+					},
+				},
+				Samples: []prompb.Sample{
+					{
+						Value:     20,
+						Timestamp: 1707123456800, // 2024-02-05T08:57:36.800Z
+					},
 				},
 			},
 		},
-		{
-			Labels: []prompb.Label{
-				{
-					Name:  "__name__",
-					Value: "prometheusrw_series2",
-				},
-				{
-					Name:  "label",
-					Value: "foo2",
-				},
-				{
-					Name:  "label1",
-					Value: "value1",
-				},
+		Metadata: []prompb.MetricMetadata{
+			{
+				Type:             1,
+				MetricFamilyName: "prometheusrw_series",
+				Help:             "some help",
+				Unit:             "",
 			},
-			Samples: []prompb.Sample{
-				{
-					Value:     20,
-					Timestamp: 1707123456800, // 2024-02-05T08:57:36.800Z
-				},
+			{
+				Type:             1,
+				MetricFamilyName: "prometheusrw_series2",
+				Help:             "some help2",
+				Unit:             "",
 			},
 		},
 	}
@@ -245,7 +265,6 @@ func TestSingleIngestionProtocols(t *testing.T) {
 			{Timestamp: 1707123456800, Value: 20}, // 2024-02-05T08:57:36.700Z
 		},
 	})
-
 }
 
 func TestClusterIngestionProtocols(t *testing.T) {
@@ -297,7 +316,11 @@ func TestClusterIngestionProtocols(t *testing.T) {
 
 	// prometheus text exposition format
 	vminsert.PrometheusAPIV1ImportPrometheus(t, []string{
-		`importprometheus_series 10 1707123456700`,                               // 2024-02-05T08:57:36.700Z
+		`# HELP importprometheus_series some help message`,
+		`# TYPE importprometheus_series gauge`,
+		`importprometheus_series 10 1707123456700`, // 2024-02-05T08:57:36.700Z
+		`# HELP importprometheus_series2 some help message second one`,
+		`# TYPE importprometheus_series2 gauge`,
 		`importprometheus_series2{label="foo",label1="value1"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
 	}, apptest.QueryOpts{
 		ExtraLabels: []string{"el1=elv1", "el2=elv2"},
@@ -434,42 +457,58 @@ func TestClusterIngestionProtocols(t *testing.T) {
 	})
 
 	// prometheus remote write format
-	pbData := []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{
-				{
-					Name:  "__name__",
-					Value: "prometheusrw_series",
+	pbData := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{
+						Name:  "__name__",
+						Value: "prometheusrw_series",
+					},
+				},
+				Samples: []prompb.Sample{
+					{
+						Value:     10,
+						Timestamp: 1707123456700, // 2024-02-05T08:57:36.700Z
+
+					},
 				},
 			},
-			Samples: []prompb.Sample{
-				{
-					Value:     10,
-					Timestamp: 1707123456700, // 2024-02-05T08:57:36.700Z
-
+			{
+				Labels: []prompb.Label{
+					{
+						Name:  "__name__",
+						Value: "prometheusrw_series2",
+					},
+					{
+						Name:  "label",
+						Value: "foo2",
+					},
+					{
+						Name:  "label1",
+						Value: "value1",
+					},
+				},
+				Samples: []prompb.Sample{
+					{
+						Value:     20,
+						Timestamp: 1707123456800, // 2024-02-05T08:57:36.800Z
+					},
 				},
 			},
 		},
-		{
-			Labels: []prompb.Label{
-				{
-					Name:  "__name__",
-					Value: "prometheusrw_series2",
-				},
-				{
-					Name:  "label",
-					Value: "foo2",
-				},
-				{
-					Name:  "label1",
-					Value: "value1",
-				},
+		Metadata: []prompb.MetricMetadata{
+			{
+				Type:             1,
+				MetricFamilyName: "prometheusrw_series",
+				Help:             "some help",
+				Unit:             "",
 			},
-			Samples: []prompb.Sample{
-				{
-					Value:     20,
-					Timestamp: 1707123456800, // 2024-02-05T08:57:36.800Z
-				},
+			{
+				Type:             1,
+				MetricFamilyName: "prometheusrw_series2",
+				Help:             "some help2",
+				Unit:             "",
 			},
 		},
 	}

apptest/tests/legacy_indexdb_test.go

@@ -0,0 +1,898 @@
+package tests
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+var (
+	legacyVmsinglePath  = os.Getenv("VM_LEGACY_VMSINGLE_PATH")
+	legacyVmstoragePath = os.Getenv("VM_LEGACY_VMSTORAGE_PATH")
+)
+
+type testLegacyDeleteSeriesOpts struct {
+	startLegacySUT func() at.PrometheusWriteQuerier
+	startNewSUT    func() at.PrometheusWriteQuerier
+	stopLegacySUT  func()
+	stopNewSUT     func()
+}
+
+func TestLegacySingleDeleteSeries(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testLegacyDeleteSeriesOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle-new", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vmsingle-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vmsingle-new")
+		},
+	}
+
+	testLegacyDeleteSeries(tc, opts)
+}
+
+func TestLegacyClusterDeleteSeries(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testLegacyDeleteSeriesOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-new",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-new",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-legacy")
+			tc.StopApp("vmstorage2-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-new")
+			tc.StopApp("vmstorage2-new")
+		},
+	}
+
+	testLegacyDeleteSeries(tc, opts)
+}
+
+func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
+	t := tc.T()
+
+	type want struct {
+		series       []map[string]string
+		queryResults []*at.QueryResult
+	}
+
+	genData := func(prefix string, start, end, step int64, value float64) (recs []string, w *want) {
+		count := (end - start) / step
+		recs = make([]string, count)
+		w = &want{
+			series:       make([]map[string]string, count),
+			queryResults: make([]*at.QueryResult, count),
+		}
+		for i := range count {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			timestamp := start + int64(i)*step
+
+			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
+			w.series[i] = map[string]string{"__name__": name}
+			w.queryResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, w
+	}
+
+	assertSearchResults := func(app at.PrometheusQuerier, query string, start, end int64, step string, want *want) {
+		t.Helper()
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want.series,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  step,
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     want.queryResults,
+				},
+			},
+			FailNow: true,
+		})
+	}
+
+	// - start legacy vmsingle
+	// - insert data1
+	// - confirm that metric names and samples are searcheable
+	// - stop legacy vmsingle
+	const step = 24 * 3600 * 1000 // 24h
+	start1 := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end1 := time.Date(2000, 1, 10, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data1, want1 := genData("metric", start1, end1, step, 1)
+	legacySUT := opts.startLegacySUT()
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, data1, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertSearchResults(legacySUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
+	opts.stopLegacySUT()
+
+	// - start new vmsingle
+	// - confirm that data1 metric names and samples are searcheable
+	// - delete data1
+	// - confirm that data1 metric names and samples are not searcheable anymore
+	// - insert data2 (same metric names, different dates)
+	// - confirm that metric names become searcheable again
+	// - confirm that data1 samples are not searchable and data2 samples are searcheable
+
+	newSUT := opts.startNewSUT()
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
+
+	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	wantNoResults := &want{
+		series:       []map[string]string{},
+		queryResults: []*at.QueryResult{},
+	}
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", wantNoResults)
+
+	start2 := time.Date(2000, 1, 11, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end2 := time.Date(2000, 1, 20, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data2, want2 := genData("metric", start2, end2, step, 2)
+	newSUT.PrometheusAPIV1ImportPrometheus(t, data2, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end2, "1d", want2)
+
+	// - restart new vmsingle
+	// - confirm that metric names still searchable, data1 samples are not
+	//   searchable, and data2 samples are searcheable
+
+	opts.stopNewSUT()
+	newSUT = opts.startNewSUT()
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end2, "1d", want2)
+	opts.stopNewSUT()
+}
+
+type testLegacyBackupRestoreOpts struct {
+	startLegacySUT     func() at.PrometheusWriteQuerier
+	startNewSUT        func() at.PrometheusWriteQuerier
+	stopLegacySUT      func()
+	stopNewSUT         func()
+	storageDataPaths   []string
+	snapshotCreateURLs func(at.PrometheusWriteQuerier) []string
+}
+
+func TestLegacySingleBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testLegacyBackupRestoreOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle-new", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vmsingle-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vmsingle-new")
+		},
+		storageDataPaths: []string{
+			storageDataPath,
+		},
+		snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
+			return []string{
+				sut.(*at.Vmsingle).SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testLegacyBackupRestore(tc, opts)
+}
+
+func TestLegacyClusterBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testLegacyBackupRestoreOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-new",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-new",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-legacy")
+			tc.StopApp("vmstorage2-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-new")
+			tc.StopApp("vmstorage2-new")
+		},
+		storageDataPaths: []string{
+			storage1DataPath,
+			storage2DataPath,
+		},
+		snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
+			c := sut.(*at.Vmcluster)
+			return []string{
+				c.Vmstorages[0].SnapshotCreateURL(),
+				c.Vmstorages[1].SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testLegacyBackupRestore(tc, opts)
+}
+
+func testLegacyBackupRestore(tc *at.TestCase, opts testLegacyBackupRestoreOpts) {
+	t := tc.T()
+
+	const msecPerMinute = 60 * 1000
+	// Use the same number of metrics and time range for all the data ingestions
+	// below.
+	const numMetrics = 1000
+	start := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).Add(-numMetrics * time.Minute).UnixMilli()
+	end := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
+	genData := func(prefix string) (recs []string, wantSeries []map[string]string, wantQueryResults []*at.QueryResult) {
+		recs = make([]string, numMetrics)
+		wantSeries = make([]map[string]string, numMetrics)
+		wantQueryResults = make([]*at.QueryResult, numMetrics)
+		for i := range numMetrics {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			value := float64(i)
+			timestamp := start + int64(i)*msecPerMinute
+
+			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
+			wantSeries[i] = map[string]string{"__name__": name}
+			wantQueryResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, wantSeries, wantQueryResults
+	}
+
+	backupBaseDir, err := filepath.Abs(filepath.Join(tc.Dir(), "backups"))
+	if err != nil {
+		t.Fatalf("could not get absolute path for the backup base dir")
+	}
+
+	// assertSeries issues various queries to the app and compares the query
+	// results with the expected ones.
+	assertQueries := func(app at.PrometheusQuerier, query string, wantSeries []map[string]string, wantQueryResults []*at.QueryResult) {
+		t.Helper()
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   wantSeries,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  "60s",
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     wantQueryResults,
+				},
+			},
+			Retries: 300,
+			FailNow: true,
+		})
+	}
+
+	createBackup := func(sut at.PrometheusWriteQuerier, name string) {
+		t.Helper()
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmbackup-%s-%s", name, replica)
+			snapshotCreateURL := opts.snapshotCreateURLs(sut)[i]
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, backupPath)
+		}
+	}
+
+	restoreFromBackup := func(name string) {
+		t.Helper()
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmrestore-%s-%s", name, replica)
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmrestore(instance, backupPath, storageDataPath)
+		}
+	}
+
+	legacy1Data, wantLegacy1Series, wantLegacy1QueryResults := genData("legacy1")
+	legacy2Data, wantLegacy2Series, wantLegacy2QueryResults := genData("legacy2")
+	new1Data, wantNew1Series, wantNew1QueryResults := genData("new1")
+	new2Data, wantNew2Series, wantNew2QueryResults := genData("new2")
+	wantLegacy12Series := slices.Concat(wantLegacy1Series, wantLegacy2Series)
+	wantLegacy12QueryResults := slices.Concat(wantLegacy1QueryResults, wantLegacy2QueryResults)
+	wantLegacy1New1Series := slices.Concat(wantLegacy1Series, wantNew1Series)
+	wantLegacy1New1QueryResults := slices.Concat(wantLegacy1QueryResults, wantNew1QueryResults)
+	wantLegacy1New12Series := slices.Concat(wantLegacy1New1Series, wantNew2Series)
+	wantLegacy1New12QueryResults := slices.Concat(wantLegacy1New1QueryResults, wantNew2QueryResults)
+	var legacySUT, newSUT at.PrometheusWriteQuerier
+
+	// Verify backup/restore with legacy SUT.
+
+	// Start legacy SUT with empty storage data dir.
+	legacySUT = opts.startLegacySUT()
+
+	// Ingest legacy1 records, ensure the queries return legacy1, and create
+	// legacy1 backup.
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy1Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+	createBackup(legacySUT, "legacy1")
+
+	// Ingest legacy2 records, ensure the queries return legacy1+legacy2, and
+	// create legacy1+legacy2 backup.
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy2Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertQueries(legacySUT, `{__name__=~"legacy.*"}`, wantLegacy12Series, wantLegacy12QueryResults)
+	createBackup(legacySUT, "legacy12")
+
+	// Stop legacy SUT and restore legacy1 data.
+	// Start legacy SUT and ensure the queries return legacy1.
+	opts.stopLegacySUT()
+	restoreFromBackup("legacy1")
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+
+	opts.stopLegacySUT()
+
+	// Verify backup/restore with new SUT.
+
+	// Start new SUT (with partition indexDBs) with storage containing legacy1
+	// data and Ensure that queries return legacy1 data.
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+
+	// Ingest new1 records, ensure that queries now return legacy1+new1, and
+	// create the legacy1+new1 backup.
+	newSUT.PrometheusAPIV1ImportPrometheus(t, new1Data, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertQueries(newSUT, `{__name__=~"(legacy|new).*"}`, wantLegacy1New1Series, wantLegacy1New1QueryResults)
+	createBackup(newSUT, "legacy1-new1")
+
+	// Ingest new2 records, ensure that queries now return legacy1+new1+new2,
+	// and create the legacy1+new1+new2 backup.
+	newSUT.PrometheusAPIV1ImportPrometheus(t, new2Data, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertQueries(newSUT, `{__name__=~"(legacy|new1|new2).*"}`, wantLegacy1New12Series, wantLegacy1New12QueryResults)
+	createBackup(newSUT, "legacy1-new12")
+
+	// Stop new SUT and restore legacy1+new1 data.
+	// Start new SUT and ensure queries return legacy1+new1 data.
+	opts.stopNewSUT()
+	restoreFromBackup("legacy1-new1")
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy1New1Series, wantLegacy1New1QueryResults)
+
+	opts.stopNewSUT()
+
+	// Verify backup/restore with legacy SUT again.
+
+	// Start legacy SUT with storage containing legacy1+new1 data.
+	//
+	// Ensure that the /series and /query_range queries return legacy1 data only.
+	// new1 data is not returned because legacy vmsingle does not know about
+	// partition indexDBs.
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+
+	// Stop legacy SUT and restore legacy1+legacy2 data.
+	// Start legacy SUT and ensure that queries now return legacy1+legacy2 data.
+	opts.stopLegacySUT()
+	restoreFromBackup("legacy12")
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy12Series, wantLegacy12QueryResults)
+
+	opts.stopLegacySUT()
+
+	// Verify backup/restore with new vmsingle again.
+
+	// Start new vmsingle with storage containing legacy1+legacy2 data and
+	// ensure that queries return legacy1+legacy2 data.
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy12Series, wantLegacy12QueryResults)
+
+	// Stop new SUT and restore legacy1+new1+new2 data.
+	// Start new SUT and ensure that queries return legacy1+new1+new2 data.
+	opts.stopNewSUT()
+	restoreFromBackup("legacy1-new12")
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~"(legacy|new).*"}`, wantLegacy1New12Series, wantLegacy1New12QueryResults)
+
+	opts.stopNewSUT()
+}
+
+type testLegacyDowngradeOpts struct {
+	startLegacySUT func() at.PrometheusWriteQuerier
+	startNewSUT    func() at.PrometheusWriteQuerier
+	stopLegacySUT  func()
+	stopNewSUT     func()
+}
+
+func TestLegacySingleDowngrade(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testLegacyDowngradeOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle-new", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vmsingle-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vmsingle-new")
+		},
+	}
+
+	testLegacyDowngrade(tc, opts)
+}
+
+func TestLegacyClusterDowngrade(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testLegacyDowngradeOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-new",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-new",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-legacy")
+			tc.StopApp("vmstorage2-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-new")
+			tc.StopApp("vmstorage2-new")
+		},
+	}
+
+	testLegacyDowngrade(tc, opts)
+}
+
+func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
+	t := tc.T()
+
+	type want struct {
+		series            []map[string]string
+		labels            []string
+		labelValues       []string
+		queryResults      []*at.QueryResult
+		queryRangeResults []*at.QueryResult
+	}
+
+	uniq := func(s []string) []string {
+		slices.Sort(s)
+		return slices.Compact(s)
+	}
+
+	mergeWant := func(want1, want2 want) want {
+		var result want
+		result.series = slices.Concat(want1.series, want2.series)
+		result.labels = uniq(slices.Concat(want1.labels, want2.labels))
+		result.labelValues = slices.Concat(want1.labelValues, want2.labelValues)
+		result.queryResults = slices.Concat(want1.queryResults, want2.queryResults)
+		result.queryRangeResults = slices.Concat(want1.queryRangeResults, want2.queryRangeResults)
+
+		return result
+	}
+
+	// Use the same number of metrics and time range for all the data batches below.
+	const numMetrics = 1000
+	const labelName = "prefix"
+	start := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
+	end := start
+	genData := func(prefix string) (recs []string, want want) {
+		labelValue := prefix
+		recs = make([]string, numMetrics)
+		want.series = make([]map[string]string, numMetrics)
+		want.labels = []string{"__name__", labelName}
+		want.labelValues = []string{labelValue}
+		want.queryResults = make([]*at.QueryResult, numMetrics)
+		want.queryRangeResults = make([]*at.QueryResult, numMetrics)
+		for i := range numMetrics {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			value := float64(i)
+			timestamp := start
+
+			recs[i] = fmt.Sprintf("%s{%s=\"%s\"} %f %d", name, labelName, labelValue, value, timestamp)
+			want.series[i] = map[string]string{"__name__": name, labelName: labelValue}
+			want.queryResults[i] = &at.QueryResult{
+				Metric: map[string]string{"__name__": name, labelName: labelValue},
+				Sample: &at.Sample{Timestamp: timestamp, Value: value},
+			}
+			want.queryRangeResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name, labelName: labelValue},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, want
+	}
+
+	// assertSeries issues various queries to the app and compares the query
+	// results with the expected ones.
+	assertQueries := func(app at.PrometheusQuerier, query string, want want, wantSeriesCount uint64) {
+		t.Helper()
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want.series,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series/count response",
+			Got: func() any {
+				return app.PrometheusAPIV1SeriesCount(t, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				})
+			},
+			Want: &at.PrometheusAPIV1SeriesCountResponse{
+				Status: "success",
+				Data:   []uint64{wantSeriesCount},
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/labels response",
+			Got: func() any {
+				return app.PrometheusAPIV1Labels(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				})
+			},
+			Want: &at.PrometheusAPIV1LabelsResponse{
+				Status: "success",
+				Data:   want.labels,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/label/../values response",
+			Got: func() any {
+				return app.PrometheusAPIV1LabelValues(t, labelName, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				})
+			},
+			Want: &at.PrometheusAPIV1LabelValuesResponse{
+				Status: "success",
+				Data:   want.labelValues,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query response",
+			Got: func() any {
+				return app.PrometheusAPIV1Query(t, query, at.QueryOpts{
+					Time: fmt.Sprintf("%d", start),
+					Step: "10m",
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "vector",
+					Result:     want.queryResults,
+				},
+			},
+			Retries: 300,
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  "60s",
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     want.queryRangeResults,
+				},
+			},
+			Retries: 300,
+			FailNow: true,
+		})
+	}
+
+	wantEmpty := want{
+		series:            []map[string]string{},
+		labels:            []string{"__name__"},
+		labelValues:       []string{},
+		queryResults:      []*at.QueryResult{},
+		queryRangeResults: []*at.QueryResult{},
+	}
+
+	legacy1Data, wantLegacy1 := genData("legacy1")
+	legacy2Data, wantLegacy2 := genData("legacy2")
+	new1Data, wantNew1 := genData("new1")
+	wantLegacy1New1 := mergeWant(wantLegacy1, wantNew1)
+	wantLegacy2New1 := mergeWant(wantLegacy2, wantNew1)
+	var legacySUT, newSUT at.PrometheusWriteQuerier
+
+	// Start legacy SUT with empty storage data dir.
+	// Ingest legacy1 records, ensure the queries return legacy1
+	legacySUT = opts.startLegacySUT()
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy1Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1, numMetrics)
+	opts.stopLegacySUT()
+
+	// Start new SUT (with partition indexDBs) with storage containing legacy1
+	// data and ensure that queries return new1 and legacy1 data.
+	newSUT = opts.startNewSUT()
+	newSUT.PrometheusAPIV1ImportPrometheus(t, new1Data, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy1New1, 2*numMetrics)
+	opts.stopNewSUT()
+
+	// Downgrade to legacy SUT, ensure the queries return only legacy1.
+	// Delete all series, ensure that queries return no series.
+	// Ingest legacy2 records, ensure the queries return only legacy2.
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1, numMetrics)
+	legacySUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantEmpty, numMetrics)
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy2Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	// series count includes deleted metrics
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy2, 2*numMetrics)
+	opts.stopLegacySUT()
+
+	// Upgrade to new SUT, ensure the queries return recently ingested legacy2 and new1
+	// since legacy SUT cannot delete them.
+	// Delete all series, ensure that queries return no series.
+	newSUT = opts.startNewSUT()
+	// series count includes deleted metrics
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy2New1, 3*numMetrics)
+	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	// series count includes deleted metrics
+	assertQueries(newSUT, `{__name__=~".*"}`, wantEmpty, 3*numMetrics)
+	opts.stopNewSUT()
+}

apptest/tests/metricsmetadata_test.go

@@ -0,0 +1,225 @@
+package tests
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+)
+
+func TestSingleMetricsMetadata(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+	sut := tc.MustStartVmsingle("vmsingle", []string{
+		"-storageDataPath=" + tc.Dir(),
+		"-retentionPeriod=100y",
+		"-enableMetadata",
+	})
+	// verify empty stats
+	resp := sut.PrometheusAPIV1Metadata(t, "", 0, apptest.QueryOpts{})
+	if len(resp.Data) != 0 {
+		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
+	}
+
+	const ingestTimestamp = 1707123456700
+	prometheusTextDataSet := []string{
+		`# HELP metric_name_1 some help message`,
+		`# TYPE metric_name_1 gauge`,
+		`metric_name_1{label="foo"} 10`,
+		`metric_name_1{label="bar"} 10`,
+		`metric_name_1{label="baz"} 10`,
+		`# HELP metric_name_2 some help message`,
+		`# TYPE metric_name_2 counter`,
+		`metric_name_2{label="baz"} 20`,
+		`# HELP metric_name_3 some help message`,
+		`# TYPE metric_name_3 gauge`,
+		`metric_name_3{label="baz"} 30`,
+	}
+	prometheusRemoteWriteDataSet := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_5"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_6"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+		},
+		Metadata: []prompb.MetricMetadata{
+			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
+		},
+	}
+
+	sut.PrometheusAPIV1ImportPrometheus(t, prometheusTextDataSet, apptest.QueryOpts{})
+	sut.PrometheusAPIV1Write(t, prometheusRemoteWriteDataSet, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	expected := &apptest.PrometheusAPIV1Metadata{
+		Status: "success",
+		Data: map[string][]apptest.MetadataEntry{
+			"metric_name_1": {{Help: "some help message", Type: "gauge"}},
+			"metric_name_2": {{Help: "some help message", Type: "counter"}},
+			"metric_name_3": {{Help: "some help message", Type: "gauge"}},
+			"metric_name_4": {{Help: "some help message", Type: "summary"}},
+			"metric_name_5": {{Help: "some help message", Type: "summary"}},
+			"metric_name_6": {{Help: "some help message", Type: "stateset"}},
+		},
+	}
+	gotStats := sut.PrometheusAPIV1Metadata(t, "", 0, apptest.QueryOpts{})
+	if diff := cmp.Diff(expected, gotStats); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	// check query metric name filter
+	tc.Assert(&apptest.AssertOptions{
+		Msg: "unexpected /api/v1/metadata response",
+		Got: func() any {
+			return sut.PrometheusAPIV1Metadata(t, "metric_name_4", 0, apptest.QueryOpts{})
+		},
+		Want: &apptest.PrometheusAPIV1Metadata{
+			Status: "success",
+			Data: map[string][]apptest.MetadataEntry{
+				"metric_name_4": {{Help: "some help message", Type: "summary"}},
+			},
+		},
+	})
+
+	// check query limit filter
+	tc.Assert(&apptest.AssertOptions{
+		Msg: "unexpected /api/v1/metadata response",
+		Got: func() any {
+			return sut.PrometheusAPIV1Metadata(t, "", 3, apptest.QueryOpts{})
+		},
+		Want: &apptest.PrometheusAPIV1Metadata{
+			Status: "success",
+			Data: map[string][]apptest.MetadataEntry{
+				"metric_name_1": {{Help: "some help message", Type: "gauge"}},
+				"metric_name_2": {{Help: "some help message", Type: "counter"}},
+				"metric_name_3": {{Help: "some help message", Type: "gauge"}},
+			},
+		},
+	})
+}
+
+func TestClusterMetricsMetadata(t *testing.T) {
+	fs.MustRemoveDir(t.Name())
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+	vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
+		"-retentionPeriod=100y",
+	})
+	vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
+		"-retentionPeriod=100y",
+	})
+
+	vminsert1 := tc.MustStartVminsert("vminsert1", []string{
+		fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VminsertAddr(), vmstorage2.VminsertAddr()),
+		"-enableMetadata",
+	})
+	vminsert2 := tc.MustStartVminsert("vminsert-2", []string{
+		fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VminsertAddr(), vmstorage2.VminsertAddr()),
+		"-enableMetadata",
+	})
+	vminsertGlobal := tc.MustStartVminsert("vminsert-global", []string{
+		fmt.Sprintf("-storageNode=%s,%s", vminsert1.ClusternativeListenAddr(), vminsert2.ClusternativeListenAddr()),
+		"-enableMetadata",
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		fmt.Sprintf("-storageNode=%s,%s", vmstorage1.VmselectAddr(), vmstorage2.VmselectAddr()),
+	})
+	// verify empty stats
+	resp := vmselect.PrometheusAPIV1Metadata(t, "", 0, apptest.QueryOpts{Tenant: "0:0"})
+	if len(resp.Data) != 0 {
+		t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
+	}
+
+	const ingestTimestamp = 1707123456700
+	prometheusTextDataSet := []string{
+		`# HELP metric_name_1 some help message`,
+		`# TYPE metric_name_1 gauge`,
+		`metric_name_1{label="foo"} 10`,
+		`metric_name_1{label="bar"} 10`,
+		`metric_name_1{label="baz"} 10`,
+		`# HELP metric_name_2 some help message`,
+		`# TYPE metric_name_2 counter`,
+		`metric_name_2{label="baz"} 20`,
+		`# HELP metric_name_3 some help message`,
+		`# TYPE metric_name_3 gauge`,
+		`metric_name_3{label="baz"} 30`,
+	}
+	prometheusRemoteWriteDataSet := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_5"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_6"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
+		},
+		Metadata: []prompb.MetricMetadata{
+			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
+		},
+	}
+
+	assertMetadataIngestOn := func(t *testing.T, vminsert *apptest.Vminsert, tenantID string) {
+		t.Helper()
+		vminsert.PrometheusAPIV1ImportPrometheus(t, prometheusTextDataSet, apptest.QueryOpts{Tenant: tenantID})
+		vminsert.PrometheusAPIV1Write(t, prometheusRemoteWriteDataSet, apptest.QueryOpts{Tenant: tenantID})
+		vmstorage1.ForceFlush(t)
+		vmstorage2.ForceFlush(t)
+		expected := &apptest.PrometheusAPIV1Metadata{
+			Status: "success",
+			Data: map[string][]apptest.MetadataEntry{
+				"metric_name_1": {{Help: "some help message", Type: "gauge"}},
+				"metric_name_2": {{Help: "some help message", Type: "counter"}},
+				"metric_name_3": {{Help: "some help message", Type: "gauge"}},
+				"metric_name_4": {{Help: "some help message", Type: "summary"}},
+				"metric_name_5": {{Help: "some help message", Type: "summary"}},
+				"metric_name_6": {{Help: "some help message", Type: "stateset"}},
+			},
+		}
+		gotStats := vmselect.PrometheusAPIV1Metadata(t, "", 0, apptest.QueryOpts{Tenant: tenantID})
+		if diff := cmp.Diff(expected, gotStats); diff != "" {
+			t.Errorf("unexpected response (-want, +got):\n%s", diff)
+		}
+	}
+
+	assertMetadataIngestOn(t, vminsert1, "2:2")
+	assertMetadataIngestOn(t, vminsert2, "3:3")
+	assertMetadataIngestOn(t, vminsertGlobal, "5:5")
+
+	// check query metric name filter
+	tc.Assert(&apptest.AssertOptions{
+		Msg: "unexpected /api/v1/metadata response",
+		Got: func() any {
+			return vmselect.PrometheusAPIV1Metadata(t, "metric_name_4", 0, apptest.QueryOpts{Tenant: "multitenant"})
+		},
+		Want: &apptest.PrometheusAPIV1Metadata{
+			Status: "success",
+			Data: map[string][]apptest.MetadataEntry{
+				"metric_name_4": {{Help: "some help message", Type: "summary"}},
+			},
+		},
+	})
+
+	// check query limit filter
+	tc.Assert(&apptest.AssertOptions{
+		Msg: "unexpected /api/v1/metadata response",
+		Got: func() any {
+			return vmselect.PrometheusAPIV1Metadata(t, "", 3, apptest.QueryOpts{Tenant: "5:5"})
+		},
+		Want: &apptest.PrometheusAPIV1Metadata{
+			Status: "success",
+			Data: map[string][]apptest.MetadataEntry{
+				"metric_name_1": {{Help: "some help message", Type: "gauge"}},
+				"metric_name_2": {{Help: "some help message", Type: "counter"}},
+				"metric_name_3": {{Help: "some help message", Type: "gauge"}},
+			},
+		},
+	})
+}

apptest/tests/metricsql_test.go

@@ -47,14 +47,16 @@ func TestClusterInstantQuery(t *testing.T) {
 }
 
 func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQuerier) {
-	data := []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{
-				{Name: "__name__", Value: "3fooµ¥"},
-				{Name: "3👋tfにちは", Value: "漢©®€£"},
-			},
-			Samples: []prompb.Sample{
-				{Value: 1, Timestamp: millis("2024-01-01T00:01:00Z")},
+	data := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "3fooµ¥"},
+					{Name: "3👋tfにちは", Value: "漢©®€£"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: millis("2024-01-01T00:01:00Z")},
+				},
 			},
 		},
 	}
@@ -89,23 +91,25 @@ func testInstantQueryWithUTFNames(t *testing.T, sut apptest.PrometheusWriteQueri
 	fn(`{"3👋tfにちは"="漢©®€£"}`)
 }
 
-var staleNaNsData = func() []prompb.TimeSeries {
-	return []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{
-				{
-					Name:  "__name__",
-					Value: "metric",
+var staleNaNsData = func() prompb.WriteRequest {
+	return prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{
+						Name:  "__name__",
+						Value: "metric",
+					},
 				},
-			},
-			Samples: []prompb.Sample{
-				{
-					Value:     1,
-					Timestamp: millis("2024-01-01T00:01:00Z"),
-				},
-				{
-					Value:     decimal.StaleNaN,
-					Timestamp: millis("2024-01-01T00:02:00Z"),
+				Samples: []prompb.Sample{
+					{
+						Value:     1,
+						Timestamp: millis("2024-01-01T00:01:00Z"),
+					},
+					{
+						Value:     decimal.StaleNaN,
+						Timestamp: millis("2024-01-01T00:02:00Z"),
+					},
 				},
 			},
 		},
@@ -185,21 +189,23 @@ func testInstantQueryDoesNotReturnStaleNaNs(t *testing.T, sut apptest.Prometheus
 // However, conversion of math.NaN to int64 could behave differently depending on platform and Go version.
 // Hence, this test could succeed for some platforms even if fix is rolled back.
 func testQueryRangeWithAtModifier(t *testing.T, sut apptest.PrometheusWriteQuerier) {
-	data := []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{
-				{Name: "__name__", Value: "up"},
+	data := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "up"},
+				},
+				Samples: []prompb.Sample{
+					{Value: 1, Timestamp: millis("2025-01-01T00:01:00Z")},
+				},
 			},
-			Samples: []prompb.Sample{
-				{Value: 1, Timestamp: millis("2025-01-01T00:01:00Z")},
-			},
-		},
-		{
-			Labels: []prompb.Label{
-				{Name: "__name__", Value: "metricNaN"},
-			},
-			Samples: []prompb.Sample{
-				{Value: decimal.StaleNaN, Timestamp: millis("2025-01-01T00:01:00Z")},
+			{
+				Labels: []prompb.Label{
+					{Name: "__name__", Value: "metricNaN"},
+				},
+				Samples: []prompb.Sample{
+					{Value: decimal.StaleNaN, Timestamp: millis("2025-01-01T00:01:00Z")},
+				},
 			},
 		},
 	}

apptest/tests/relabeling_test.go

@@ -139,41 +139,43 @@ func TestSingleIngestionWithRelabeling(t *testing.T) {
 		},
 	})
 
-	pbData := []prompb.TimeSeries{
-		{
-			Labels: []prompb.Label{
-				{
-					Name:  "__name__",
-					Value: "prometheusrw_series",
+	pbData := prompb.WriteRequest{
+		Timeseries: []prompb.TimeSeries{
+			{
+				Labels: []prompb.Label{
+					{
+						Name:  "__name__",
+						Value: "prometheusrw_series",
+					},
+					{
+						Name:  "label",
+						Value: "foo2",
+					},
 				},
-				{
-					Name:  "label",
-					Value: "foo2",
-				},
-			},
-			Samples: []prompb.Sample{
-				{
-					Value:     10,
-					Timestamp: 1707123456700, // 2024-02-05T08:57:36.700Z
+				Samples: []prompb.Sample{
+					{
+						Value:     10,
+						Timestamp: 1707123456700, // 2024-02-05T08:57:36.700Z
 
+					},
 				},
 			},
-		},
-		{
-			Labels: []prompb.Label{
-				{
-					Name:  "__name__",
-					Value: "must_drop_series",
+			{
+				Labels: []prompb.Label{
+					{
+						Name:  "__name__",
+						Value: "must_drop_series",
+					},
+					{
+						Name:  "label",
+						Value: "foo2",
+					},
 				},
-				{
-					Name:  "label",
-					Value: "foo2",
-				},
-			},
-			Samples: []prompb.Sample{
-				{
-					Value:     20,
-					Timestamp: 1707123456800, // 2024-02-05T08:57:36.800Z
+				Samples: []prompb.Sample{
+					{
+						Value:     20,
+						Timestamp: 1707123456800, // 2024-02-05T08:57:36.800Z
+					},
 				},
 			},
 		},

apptest/tests/replication_test.go

@@ -973,7 +973,7 @@ func testGroupSkipSlowReplicas(tc *apptest.TestCase, opts *testGroupReplicationO
 
 	// The data is replicated across N groups of M nodes. Replication factor is
 	// globalRF. There is no replication across the nodes within each group or
-	//it is unknown it there is one.
+	// it is unknown it there is one.
 	//
 	// Max number of nodes to skip is M*(globalRF-1). This corresponds to the
 	// case when N-globalRF+1 groups have received the response from all of