app/vmauth: wrap readDurationTrackingBody if r.Body != nil

### Describe Your Changes

Updated project dependencies to the latest versions.

### Checklist

The following checks are **mandatory**:

- [x] My change adheres to [VictoriaMetrics contributing
guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).
- [x] My change adheres to [VictoriaMetrics development
goals](https://docs.victoriametrics.com/victoriametrics/goals/).

Signed-off-by: Yury Molodov <yurymolodov@gmail.com>

.github/ISSUE_TEMPLATE/question.yml

@@ -5,7 +5,7 @@ body:
   - type: textarea
     id: describe-the-component
     attributes:
-      label: Is your question request related to a specific component?
+      label: Is your question related to a specific component?
       placeholder: |
         VictoriaMetrics, vmagent, vmalert, vmui, etc...
     validations:

.github/workflows/changelog-linter.yml

@@ -9,7 +9,7 @@ jobs:
   tip-lint:
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: 'actions/checkout@v4'
+      - uses: 'actions/checkout@v6'
         with:
           # needed for proper diff
           fetch-depth: 0

Makefile

@@ -17,7 +17,7 @@ EXTRA_GO_BUILD_TAGS ?=
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.4.0
+GOLANGCI_LINT_VERSION := 2.7.2
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -435,7 +435,7 @@ release-vmutils-windows-goarch: \
 		vmctl-windows-$(GOARCH)-prod.exe
 
 pprof-cpu:
-	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics $(PPROF_FILE)
 
 fmt:
 	gofmt -l -w -s ./lib
@@ -471,7 +471,23 @@ integration-test:
 
 apptest:
 	$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
-	go test ./apptest/... -skip="^TestCluster.*"
+	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
+
+integration-test-legacy: victoria-metrics vmbackup vmrestore
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.132.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
+	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
+	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
 	GOEXPERIMENT=synctest go test -bench=. ./lib/...

app/victoria-metrics/self_scraper.go

@@ -10,9 +10,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 )
 
@@ -48,6 +50,7 @@ func selfScraper(scrapeInterval time.Duration) {
 
 	var bb bytesutil.ByteBuffer
 	var rows prometheus.Rows
+	var metadataRows prometheus.MetadataRows
 	var mrs []storage.MetricRow
 	var labels []prompb.Label
 	t := time.NewTicker(scrapeInterval)
@@ -57,8 +60,12 @@ func selfScraper(scrapeInterval time.Duration) {
 		appmetrics.WritePrometheusMetrics(&bb)
 		s := bytesutil.ToUnsafeString(bb.B)
 		rows.Reset()
-		// VictoriaMetrics components don't expose metadata yet, only need to parse samples
-		rows.UnmarshalWithErrLogger(s, nil)
+		// Parse metrics and optionally metadata when enabled
+		if prommetadata.IsEnabled() {
+			rows, metadataRows = prometheus.UnmarshalWithMetadata(rows, metadataRows, s, nil)
+		} else {
+			rows.UnmarshalWithErrLogger(s, nil)
+		}
 		mrs = mrs[:0]
 		for i := range rows.Rows {
 			r := &rows.Rows[i]
@@ -91,6 +98,19 @@ func selfScraper(scrapeInterval time.Duration) {
 		if err := vmstorage.AddRows(mrs); err != nil {
 			logger.Errorf("cannot store self-scraped metrics: %s", err)
 		}
+		if len(metadataRows.Rows) > 0 {
+			mms := make([]metricsmetadata.Row, 0, len(metadataRows.Rows))
+			for _, mm := range metadataRows.Rows {
+				mms = append(mms, metricsmetadata.Row{
+					MetricFamilyName: bytesutil.ToUnsafeBytes(mm.Metric),
+					Help:             bytesutil.ToUnsafeBytes(mm.Help),
+					Type:             mm.Type,
+				})
+			}
+			if err := vmstorage.AddMetadataRows(mms); err != nil {
+				logger.Errorf("cannot store self-scraped metrics metadata: %s", err)
+			}
+		}
 	}
 	for {
 		select {

app/vmagent/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -350,6 +351,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "/zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(nil, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -644,6 +656,17 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(at, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -765,6 +788,9 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 

app/vmagent/newrelic/request_handler.go

@@ -78,7 +78,7 @@ func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label)
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
-	rowsInserted.Add(len(rows))
+	rowsInserted.Add(samplesCount)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(samplesCount)
 	}

app/vmagent/opentelemetry/request_handler.go

@@ -25,7 +25,7 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
-// InsertHandler processes metrics from given reader.
+// InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
 		return insertRows(at, tss, mms, nil)

app/vmagent/remotewrite/client.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -554,9 +553,9 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 // For more details, see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9417
 func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
 	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
+	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
 	if err != nil {
-		return nil, fmt.Errorf("zstd: decompress: %s", err)
+		return nil, err
 	}
 
 	return snappy.Encode(nil, plainBlock), nil

app/vmagent/remotewrite/relabel.go

@@ -9,14 +9,14 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/VictoriaMetrics/metrics"
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"go.yaml.in/yaml/v3"
-
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -139,6 +139,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		remoteWriteRelabelConfigData.Store(&rawCfg)
 		rcs.global = global
 	}
+
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
@@ -176,19 +177,9 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
+// isSet indicates whether (global or per-URL) command-line flags is set
 func (rcs *relabelConfigs) isSet() bool {
-	if rcs == nil {
-		return false
-	}
-	if rcs.global.Len() > 0 {
-		return true
-	}
-	for _, pc := range rcs.perURL {
-		if pc.Len() > 0 {
-			return true
-		}
-	}
-	return false
+	return *relabelConfigPathGlobal != "" || len(*relabelConfigPaths) > 0
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.

app/vmagent/zabbixconnector/request_handler.go

@@ -0,0 +1,80 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="zabbixconnector"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="zabbixconnector"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := len(rows)
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompb.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		labels = append(labels, extraLabels...)
+
+		samplesLen := len(samples)
+		samples = append(samples, prompb.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+
+		tssDst = append(tssDst, prompb.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/notifier/alert.go

@@ -80,14 +80,15 @@ func (as AlertState) String() string {
 
 // AlertTplData is used to execute templating
 type AlertTplData struct {
-	Type     string
-	Labels   map[string]string
-	Value    float64
-	Expr     string
-	AlertID  uint64
-	GroupID  uint64
-	ActiveAt time.Time
-	For      time.Duration
+	Type      string
+	Labels    map[string]string
+	Value     float64
+	Expr      string
+	AlertID   uint64
+	GroupID   uint64
+	ActiveAt  time.Time
+	For       time.Duration
+	IsPartial bool
 }
 
 var tplHeaders = []string{
@@ -101,6 +102,7 @@ var tplHeaders = []string{
 	"{{ $groupID := .GroupID }}",
 	"{{ $activeAt := .ActiveAt }}",
 	"{{ $for := .For }}",
+	"{{ $isPartial := .IsPartial }}",
 }
 
 // ExecTemplate executes the Alert template for given

app/vmalert/rule/alerting.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"math"
@@ -345,6 +346,8 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		ls.processed[l.Name] = l.Value
 	}
 
+	// labels only support limited templating variables,
+	// including `labels`, `value` and `expr`, to avoid breaking alert states or causing cardinality issue with results
 	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
@@ -386,11 +389,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, err
 		}
 		alertID := hash(ls.processed)
-		as, err := ar.expandAnnotationTemplates(s, qFn, time.Time{}, ls)
-		if err != nil {
-			return nil, err
-		}
-		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
+		a := ar.newAlert(s, time.Time{}, ls.processed, nil) // initial alert
 
 		prevT := time.Time{}
 		for i := range s.Values {
@@ -406,8 +405,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
 				a.ActiveAt = at
-				// re-template the annotations as active timestamp is changed
-				a.Annotations, _ = ar.expandAnnotationTemplates(s, qFn, at, ls)
 				a.Start = time.Time{}
 			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
 				a.State = notifier.StateFiring
@@ -453,7 +450,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 
 	defer func() {
 		ar.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			ar.metrics.errors.Inc()
 		}
 	}()
@@ -462,7 +459,8 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
-	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
+	isPartial := isPartialResponse(res)
+	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartial)
 	qFn := func(query string) ([]datasource.Metric, error) {
 		res, _, err := ar.q.Query(ctx, query, ts)
 		return res.Data, err
@@ -488,7 +486,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				at = a.ActiveAt
 			}
 		}
-		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
+		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls, isPartial)
 		if err != nil {
 			// only set error in current state, but do not break alert processing
 			curState.Err = err
@@ -606,16 +604,17 @@ func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.
 	return ls, nil
 }
 
-func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet) (map[string]string, error) {
+func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet, isPartial bool) (map[string]string, error) {
 	tplData := notifier.AlertTplData{
-		Value:    m.Values[0],
-		Type:     ar.Type.String(),
-		Labels:   ls.origin,
-		Expr:     ar.Expr,
-		AlertID:  hash(ls.processed),
-		GroupID:  ar.GroupID,
-		ActiveAt: activeAt,
-		For:      ar.For,
+		Value:     m.Values[0],
+		Type:      ar.Type.String(),
+		Labels:    ls.origin,
+		Expr:      ar.Expr,
+		AlertID:   hash(ls.processed),
+		GroupID:   ar.GroupID,
+		ActiveAt:  activeAt,
+		For:       ar.For,
+		IsPartial: isPartial,
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {

app/vmalert/rule/alerting_test.go

@@ -664,7 +664,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -684,7 +684,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-firing"},
-			Annotations: map[string]string{"activeAt": "1000"},
+			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
 			ActiveAt:    time.Unix(1, 0),
 			Start:       time.Unix(5, 0),
@@ -705,7 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -1120,7 +1120,7 @@ func TestAlertingRuleLimit_Success(t *testing.T) {
 }
 
 func TestAlertingRule_Template(t *testing.T) {
-	f := func(rule *AlertingRule, metrics []datasource.Metric, alertsExpected map[uint64]*notifier.Alert) {
+	f := func(rule *AlertingRule, metrics []datasource.Metric, isResponsePartial bool, alertsExpected map[uint64]*notifier.Alert) {
 		t.Helper()
 
 		fakeGroup := Group{
@@ -1133,6 +1133,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			entries: make([]StateEntry, 10),
 		}
 		fq.Add(metrics...)
+		fq.SetPartialResponse(isResponsePartial)
 
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -1163,7 +1164,7 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 1, "instance", "foo"),
 		metricWithValueAndLabels(t, 1, "instance", "bar"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
 				"summary": `common: Too high connection number for "foo"`,
@@ -1192,14 +1193,14 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}".{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 			"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
 		metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
 			Labels: map[string]string{
 				alertNameLabel:       "override label",
@@ -1207,7 +1208,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "foo",
 			},
 			Annotations: map[string]string{
-				"summary":     `first: Too high connection number for "foo"`,
+				"summary":     `first: Too high connection number for "foo".`,
 				"description": `override: It is 2 connections for "foo"`,
 			},
 		},
@@ -1218,7 +1219,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "bar",
 			},
 			Annotations: map[string]string{
-				"summary":     `second: Too high connection number for "bar"`,
+				"summary":     `second: Too high connection number for "bar".`,
 				"description": `override: It is 10 connections for "bar"`,
 			},
 		},
@@ -1231,7 +1232,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}.{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1239,7 +1240,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			alertNameLabel, "originAlertname",
 			alertGroupNameLabel, "originGroupname",
 			"instance", "foo"),
-	}, map[uint64]*notifier.Alert{
+	}, true, map[uint64]*notifier.Alert{
 		hash(map[string]string{
 			alertNameLabel:        "OriginLabels",
 			"exported_alertname":  "originAlertname",
@@ -1255,7 +1256,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":            "foo",
 			},
 			Annotations: map[string]string{
-				"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
+				"summary": `Alert "originAlertname(originGroupname)" for instance foo. WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.`,
 			},
 		},
 	})
@@ -1385,7 +1386,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"group":         "vmalert",
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
-		"invalid_label": `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1395,7 +1396,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)

app/vmalert/rule/recording.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -197,7 +198,7 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 
 	defer func() {
 		rr.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			rr.metrics.errors.Inc()
 		}
 	}()

app/vmauth/auth_config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"errors"
 	"flag"
 	"fmt"
 	"math"
@@ -94,6 +95,8 @@ type UserInfo struct {
 	rt http.RoundTripper
 
 	requests         *metrics.Counter
+	requestErrors    *metrics.Counter
+	backendRequests  *metrics.Counter
 	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
@@ -105,13 +108,29 @@ type HeadersConf struct {
 	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
 }
 
-func (ui *UserInfo) beginConcurrencyLimit() error {
+func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	select {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
 		ui.concurrencyLimitReached.Inc()
-		return fmt.Errorf("cannot handle more than %d concurrent requests from user %s", ui.getMaxConcurrentRequests(), ui.name())
+
+		// The per-user limit for the number of concurrent requests is reached.
+		// Wait until the currently executed requests are finished, so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case ui.concurrencyLimitCh <- struct{}{}:
+			return nil
+		case <-ctx.Done():
+			err := ctx.Err()
+			if errors.Is(err, context.DeadlineExceeded) {
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
+					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
+			}
+
+			return fmt.Errorf("cannot start executing the request because %d concurrent requests from the user %s are executed: %w",
+				ui.getMaxConcurrentRequests(), ui.name(), err)
+		}
 	}
 }
 
@@ -127,6 +146,18 @@ func (ui *UserInfo) getMaxConcurrentRequests() int {
 	return mcr
 }
 
+func (ui *UserInfo) stopHealthChecks() {
+	if ui == nil {
+		return
+	}
+	if ui.URLPrefix == nil {
+		return
+	}
+
+	bus := ui.URLPrefix.bus.Load()
+	bus.stopHealthChecks()
+}
+
 // Header is `Name: Value` http header, which must be added to the proxied request.
 type Header struct {
 	Name  string
@@ -262,7 +293,7 @@ type URLPrefix struct {
 	// the list of backend urls
 	//
 	// the list can be dynamically updated if `discover_backend_ips` option is set.
-	bus atomic.Pointer[[]*backendURL]
+	bus atomic.Pointer[backendURLs]
 
 	// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
 	discoverBackendIPs bool
@@ -286,21 +317,93 @@ func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
 	}
 }
 
+type backendURLs struct {
+	healthChecksContext context.Context
+	healthChecksCancel  func()
+	healthChecksWG      sync.WaitGroup
+
+	bus []*backendURL
+}
+
+func newBackendURLs() *backendURLs {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &backendURLs{
+		healthChecksContext: ctx,
+		healthChecksCancel:  cancel,
+	}
+}
+
+func (bus *backendURLs) add(u *url.URL) {
+	bus.bus = append(bus.bus, &backendURL{
+		url:                u,
+		healthCheckContext: bus.healthChecksContext,
+		healthCheckWG:      &bus.healthChecksWG,
+	})
+}
+
+func (bus *backendURLs) stopHealthChecks() {
+	bus.healthChecksCancel()
+	bus.healthChecksWG.Wait()
+}
+
 type backendURL struct {
-	brokenDeadline     atomic.Uint64
+	broken atomic.Bool
+
+	healthCheckContext context.Context
+	healthCheckWG      *sync.WaitGroup
+
 	concurrentRequests atomic.Int32
 
 	url *url.URL
 }
 
 func (bu *backendURL) isBroken() bool {
-	ct := fasttime.UnixTimestamp()
-	return ct < bu.brokenDeadline.Load()
+	return bu.broken.Load()
 }
 
 func (bu *backendURL) setBroken() {
-	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
-	bu.brokenDeadline.Store(deadline)
+	if bu.broken.CompareAndSwap(false, true) {
+		bu.healthCheckWG.Add(1)
+		go func() {
+			defer bu.healthCheckWG.Done()
+			bu.runHealthCheck()
+			bu.broken.Store(false)
+		}()
+	}
+}
+
+func (bu *backendURL) runHealthCheck() {
+	port := bu.url.Port()
+	if port == "" {
+		port = "80"
+	}
+	addr := net.JoinHostPort(bu.url.Hostname(), port)
+
+	t := time.NewTicker(*failTimeout)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			// Verify network connectivity via TCP dial before marking backend healthy.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9997
+			ctx, cancel := context.WithTimeout(bu.healthCheckContext, time.Second)
+			c, err := netutil.Dialer.DialContext(ctx, "tcp", addr)
+			cancel()
+			if err != nil {
+				if errors.Is(bu.healthCheckContext.Err(), context.Canceled) {
+					return
+				}
+				logger.Warnf("ignoring the backend at %s for %s because of dial error: %s", addr, *failTimeout, err)
+				continue
+			}
+
+			_ = c.Close()
+			return
+		case <-bu.healthCheckContext.Done():
+			return
+		}
+	}
 }
 
 func (bu *backendURL) get() {
@@ -312,8 +415,8 @@ func (bu *backendURL) put() {
 }
 
 func (up *URLPrefix) getBackendsCount() int {
-	pbus := up.bus.Load()
-	return len(*pbus)
+	bus := up.bus.Load()
+	return len(bus.bus)
 }
 
 // getBackendURL returns the backendURL depending on the load balance policy.
@@ -324,16 +427,15 @@ func (up *URLPrefix) getBackendsCount() int {
 func (up *URLPrefix) getBackendURL() *backendURL {
 	up.discoverBackendAddrsIfNeeded()
 
-	pbus := up.bus.Load()
-	bus := *pbus
-	if len(bus) == 0 {
+	bus := up.bus.Load()
+	if len(bus.bus) == 0 {
 		return nil
 	}
 
 	if up.loadBalancingPolicy == "first_available" {
-		return getFirstAvailableBackendURL(bus)
+		return getFirstAvailableBackendURL(bus.bus)
 	}
-	return getLeastLoadedBackendURL(bus, &up.n)
+	return getLeastLoadedBackendURL(bus.bus, &up.n)
 }
 
 func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
@@ -407,25 +509,24 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	cancel()
 
 	// generate new backendURLs for the resolved IPs
-	var busNew []*backendURL
+	busNew := newBackendURLs()
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
 			buCopy.Host = addr
-			busNew = append(busNew, &backendURL{
-				url: &buCopy,
-			})
+			busNew.add(&buCopy)
 		}
 	}
 
-	pbus := up.bus.Load()
-	if areEqualBackendURLs(*pbus, busNew) {
+	bus := up.bus.Load()
+	if areEqualBackendURLs(bus.bus, busNew.bus) {
 		return
 	}
 
 	// Store new backend urls
-	up.bus.Store(&busNew)
+	up.bus.Store(busNew)
+	bus.stopHealthChecks()
 }
 
 func areEqualBackendURLs(a, b []*backendURL) bool {
@@ -456,20 +557,23 @@ func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 	for i := 1; i < len(bus); i++ {
 		if !bus[i].isBroken() {
 			bu = bus[i]
-			break
+			bu.get()
+			return bu
 		}
 	}
-	bu.get()
-	return bu
+	return nil
 }
 
-// getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
+// getLeastLoadedBackendURL returns a non-broken backendURL with the lowest number of concurrent requests.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
 	if len(bus) == 1 {
 		// Fast path - return the only backend url.
 		bu := bus[0]
+		if bu.isBroken() {
+			return nil
+		}
 		bu.get()
 		return bu
 	}
@@ -494,7 +598,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
 	buMinIdx := n % uint32(len(bus))
 	minRequests := bus[buMinIdx].concurrentRequests.Load()
-	for i := uint32(0); i < uint32(len(bus)); i++ {
+	for i := uint32(1); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
 		bu := bus[idx]
 		if bu.isBroken() {
@@ -508,6 +612,9 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		}
 	}
 	buMin := bus[buMinIdx]
+	if buMin.isBroken() {
+		return nil
+	}
 	buMin.get()
 	atomicCounter.CompareAndSwap(n+1, buMinIdx+1)
 	return buMin
@@ -702,7 +809,7 @@ func reloadAuthConfig() (bool, error) {
 
 	ok, err := reloadAuthConfigData(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
 	}
 	if !ok {
 		return false, nil
@@ -732,6 +839,11 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 
 	acPrev := authConfig.Load()
 	if acPrev != nil {
+		acPrev.UnauthorizedUser.stopHealthChecks()
+		for i := range acPrev.Users {
+			acPrev.Users[i].stopHealthChecks()
+		}
+
 		metrics.UnregisterSet(acPrev.ms, true)
 	}
 	metrics.RegisterSet(ac.ms)
@@ -778,6 +890,8 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
 		}
 		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
@@ -826,6 +940,8 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
 		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
@@ -1060,13 +1176,11 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 	}
 
 	// Initialize up.bus
-	bus := make([]*backendURL, len(up.busOriginal))
-	for i, bu := range up.busOriginal {
-		bus[i] = &backendURL{
-			url: bu,
-		}
+	bus := newBackendURLs()
+	for _, bu := range up.busOriginal {
+		bus.add(bu)
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 
 	return nil
 }

app/vmauth/auth_config_test.go

@@ -753,7 +753,7 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	up.loadBalancingPolicy = "least_loaded"
 
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	fn := func(ns ...int) {
 		t.Helper()
@@ -825,7 +825,7 @@ func TestBrokenBackend(t *testing.T) {
 	})
 	up.loadBalancingPolicy = "least_loaded"
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	// explicitly mark one of the backends as broken
 	bus[1].setBroken()
@@ -848,7 +848,7 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 
 		up.discoverBackendAddrsIfNeeded()
 		pbus := up.bus.Load()
-		bus := *pbus
+		bus := pbus.bus
 
 		if len(bus) != 1 {
 			t.Fatalf("expected url list to be of size 1; got %d instead", len(bus))
@@ -942,16 +942,14 @@ func mustParseURL(u string) *URLPrefix {
 }
 
 func mustParseURLs(us []string) *URLPrefix {
-	bus := make([]*backendURL, len(us))
+	bus := newBackendURLs()
 	urls := make([]*url.URL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
 			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
 		}
-		bus[i] = &backendURL{
-			url: pu,
-		}
+		bus.add(pu)
 		urls[i] = pu
 	}
 	up := &URLPrefix{}
@@ -960,7 +958,7 @@ func mustParseURLs(us []string) *URLPrefix {
 	} else {
 		up.vOriginal = us
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 	up.busOriginal = urls
 	return up
 }

app/vmauth/main.go

@@ -44,12 +44,17 @@ var (
 		"See also -maxConcurrentRequests")
 	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
-	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
+		"'429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
-		"in per-user config")
+		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options "+
+		"and max_concurrent_requests option in per-user config")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration the request waits for execution when the number of concurrently executed "+
+		"requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. "+
+		"This allows graceful handling of short spikes in the number of concurrent requests")
+
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
@@ -151,6 +156,9 @@ func requestHandlerWithInternalRoutes(w http.ResponseWriter, r *http.Request) bo
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	if r.Body != nil {
+		r.Body = &readDurationTrackingBody{r: r.Body}
+	}
 
 	ats := getAuthTokensFromRequest(r)
 	if len(ats) == 0 {
@@ -208,20 +216,45 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 	ui.requests.Inc()
 
+	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
+	defer cancel()
+
 	// Limit the concurrency of requests to backends
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		if err := ui.beginConcurrencyLimit(); err != nil {
+		if err := ui.beginConcurrencyLimit(ctx); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
 			return
 		}
 	default:
-		concurrentRequestsLimitReached.Inc()
-		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
-		handleConcurrencyLimitError(w, r, err)
-		return
+		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
+		// so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case concurrencyLimitCh <- struct{}{}:
+			if err := ui.beginConcurrencyLimit(ctx); err != nil {
+				handleConcurrencyLimitError(w, r, err)
+				<-concurrencyLimitCh
+				return
+			}
+		case <-ctx.Done():
+			err := ctx.Err()
+
+			concurrentRequestsLimitReached.Inc()
+
+			if errors.Is(err, context.DeadlineExceeded) {
+				err = fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+					*maxQueueDuration, cap(concurrencyLimitCh))
+				handleConcurrencyLimitError(w, r, err)
+				return
+			}
+
+			err = fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
+			handleConcurrencyLimitError(w, r, err)
+			return
+		}
 	}
 	processRequest(w, r, ui)
 	ui.endConcurrencyLimit()
@@ -285,16 +318,18 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			return
 		}
 		bu.setBroken()
+		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
 		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)
-	ui.backendErrors.Inc()
+	ui.requestErrors.Inc()
 }
 
 func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
 	req.URL = targetURL
@@ -318,15 +353,37 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		err = ctxErr
 	}
 	if err != nil {
-		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-			// Do not retry canceled or timed out requests
+		if errors.Is(err, errReadTimeout) {
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
-			if errors.Is(err, context.DeadlineExceeded) {
-				// Timed out request must be counted as errors, since this usually means that the backend is slow.
-				logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
-				ui.backendErrors.Inc()
+
+			logger.Warnf("remoteAddr: %s; requestURI: %s; client %s request exceeded single read timeout -readTimeout=%s, closing connection", remoteAddr, requestURI, ui.name(), *readTimeout)
+
+			rejectSlowClientRequests.Inc()
+			if w1, ok := w.(http.Hijacker); ok {
+				conn, _, connErr := w1.Hijack()
+				if connErr != nil {
+					logger.Errorf("cannot hijack connection for slow read timeout handling for %s: %s", targetURL, connErr)
+					return true, false
+				}
+				_ = conn.Close()
+				return true, false
 			}
+
+			return true, false
+		}
+
+		// Do not retry canceled
+		if errors.Is(err, context.Canceled) {
+			clientCanceledRequests.Inc()
+			return true, false
+		}
+		// Do not retry timed out requests
+		if errors.Is(err, context.DeadlineExceeded) {
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			requestURI := httpserver.GetRequestURI(r)
+			// Timed out request must be counted as errors, since this usually means that the backend is slow.
+			logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
 			return false, false
 		}
 		if !rtbOK || !rtb.canRetry() {
@@ -337,6 +394,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
@@ -344,11 +402,11 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			return false, true
 		}
 
-		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
+		// Request body wasn't read yet, this usually means that the backend isn't reachable; retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, req.URL, targetURL, err)
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
@@ -357,12 +415,13 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
 			return true, false
 		}
 		// Retry requests at other backends if it matches retryStatusCodes.
@@ -370,7 +429,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
 			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
 		return false, false
 	}
@@ -381,11 +440,15 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-	if err != nil && !netutil.IsTrivialNetworkError(err) && !errors.Is(err, context.Canceled) {
+	if errors.Is(err, context.Canceled) {
+		clientCanceledRequests.Inc()
+		return true, false
+	} else if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
 
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+		ui.requestErrors.Inc()
 		return true, false
 	}
 	return true, false
@@ -513,6 +576,8 @@ var (
 	configReloadRequests     = metrics.NewCounter(`vmauth_http_requests_total{path="/-/reload"}`)
 	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
+	clientCanceledRequests   = metrics.NewCounter(`vmauth_http_request_errors_total{reason="client_canceled"}`)
+	rejectSlowClientRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="reject_slow_client"}`)
 )
 
 func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
@@ -596,6 +661,14 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
+	ctx := r.Context()
+	if errors.Is(ctx.Err(), context.Canceled) {
+		// Do not return any response for the request canceled by the client,
+		// since the connection to the client is already closed.
+		clientCanceledRequests.Inc()
+		return
+	}
+
 	w.Header().Add("Retry-After", "10")
 	err = &httpserver.ErrorWithStatusCode{
 		Err:        err,
@@ -652,6 +725,7 @@ type zeroReader struct{}
 func (r *zeroReader) Read(_ []byte) (int, error) {
 	return 0, io.EOF
 }
+
 func (r *zeroReader) Close() error {
 	return nil
 }
@@ -730,3 +804,34 @@ func debugInfo(u *url.URL, r *http.Request) string {
 	fmt.Fprint(s, ")")
 	return s.String()
 }
+
+var slowReadDuration = metrics.NewSummary(`vmauth_request_slow_read_duration_seconds`)
+
+var readTimeout = flag.Duration("readTimeout", 0, "The maximum duration for a single read call when exceeded the connection is closed. Zero disables request read timeout. "+
+	"See also -writeTimeout")
+
+var errReadTimeout = fmt.Errorf("request read timeout")
+
+type readDurationTrackingBody struct {
+	r io.ReadCloser
+}
+
+func (r *readDurationTrackingBody) Read(p []byte) (n int, err error) {
+	start := time.Now()
+	n, err = r.r.Read(p)
+	dur := time.Since(start)
+
+	// Record slow read durations only to avoid overhead for fast reads.
+	if dur > time.Millisecond {
+		slowReadDuration.Update(dur.Seconds())
+	}
+	if err == nil && *readTimeout > 0 && dur > *readTimeout {
+		return n, errReadTimeout
+	}
+
+	return n, err
+}
+
+func (r *readDurationTrackingBody) Close() error {
+	return r.r.Close()
+}

app/vmctl/flags.go

@@ -468,7 +468,7 @@ var (
 			Name: vmNativeFilterMatch,
 			Usage: "Time series selector to match series for export. For example, select {instance!=\"localhost\"} will " +
 				"match all series with \"instance\" label different to \"localhost\".\n" +
-				" See more details here https://github.com/VictoriaMetrics/VictoriaMetrics#how-to-export-data-in-native-format",
+				" See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format",
 			Value: `{__name__!=""}`,
 		},
 		&cli.StringFlag{

app/vmctl/influx.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"log"
@@ -37,7 +38,7 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st
 	}
 }
 
-func (ip *influxProcessor) run() error {
+func (ip *influxProcessor) run(ctx context.Context) error {
 	series, err := ip.ic.Explore()
 	if err != nil {
 		return fmt.Errorf("explore query failed: %s", err)
@@ -47,7 +48,7 @@ func (ip *influxProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d timeseries to import. Continue?", len(series))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/main.go

@@ -103,7 +103,7 @@ func main() {
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency), c.Bool(globalVerbose))
-					return otsdbProcessor.run()
+					return otsdbProcessor.run(ctx)
 				},
 			},
 			{
@@ -164,7 +164,7 @@ func main() {
 						c.Bool(influxSkipDatabaseLabel),
 						c.Bool(influxPrometheusMode),
 						c.Bool(globalVerbose))
-					return processor.run()
+					return processor.run(ctx)
 				},
 			},
 			{
@@ -279,7 +279,7 @@ func main() {
 						cc:        c.Int(promConcurrency),
 						isVerbose: c.Bool(globalVerbose),
 					}
-					return pp.run()
+					return pp.run(ctx)
 				},
 			},
 			{

app/vmctl/opentsdb.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
@@ -37,7 +38,7 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, verbos
 	}
 }
 
-func (op *otsdbProcessor) run() error {
+func (op *otsdbProcessor) run(ctx context.Context) error {
 	log.Println("Loading all metrics from OpenTSDB for filters: ", op.oc.Filters)
 	var metrics []string
 	for _, filter := range op.oc.Filters {
@@ -53,7 +54,7 @@ func (op *otsdbProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d metrics to import. Continue?", len(metrics))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 	op.im.ResetStats()

app/vmctl/prometheus.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
@@ -30,7 +31,7 @@ type prometheusProcessor struct {
 	isVerbose bool
 }
 
-func (pp *prometheusProcessor) run() error {
+func (pp *prometheusProcessor) run(ctx context.Context) error {
 	blocks, err := pp.cl.Explore()
 	if err != nil {
 		return fmt.Errorf("explore failed: %s", err)
@@ -39,7 +40,7 @@ func (pp *prometheusProcessor) run() error {
 		return fmt.Errorf("found no blocks to import")
 	}
 	question := fmt.Sprintf("Found %d blocks to import. Continue?", len(blocks))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/remoteread.go

@@ -47,7 +47,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 
 	question := fmt.Sprintf("Selected time range %q - %q will be split into %d ranges according to %q step. Continue?",
 		rrp.filter.timeStart.String(), rrp.filter.timeEnd.String(), len(ranges), rrp.filter.chunk)
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/utils.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -15,7 +16,7 @@ const barTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "
 // isSilent should be inited in main
 var isSilent bool
 
-func prompt(question string) bool {
+func prompt(ctx context.Context, question string) bool {
 	if isSilent {
 		return true
 	}
@@ -25,15 +26,32 @@ func prompt(question string) bool {
 	}
 	reader := bufio.NewReader(os.Stdin)
 	fmt.Print(question, " [Y/n] ")
-	answer, err := reader.ReadString('\n')
-	if err != nil {
+
+	answerCh := make(chan string, 1)
+	errCh := make(chan error, 1)
+
+	go func() {
+		answer, err := reader.ReadString('\n')
+		if err != nil {
+			errCh <- err
+			return
+		}
+		answerCh <- answer
+	}()
+
+	select {
+	case <-ctx.Done():
+		fmt.Println("\nCanceled.")
+		return false
+	case err := <-errCh:
 		panic(err)
+	case answer := <-answerCh:
+		answer = strings.TrimSpace(strings.ToLower(answer))
+		if answer == "" || answer == "yes" || answer == "y" {
+			return true
+		}
+		return false
 	}
-	answer = strings.TrimSpace(strings.ToLower(answer))
-	if answer == "" || answer == "yes" || answer == "y" {
-		return true
-	}
-	return false
 }
 
 func wrapErr(vmErr *vm.ImportError, verbose bool) error {

app/vmctl/vm_native.go

@@ -79,7 +79,7 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 			return fmt.Errorf("failed to get tenants: %w", err)
 		}
 		question := fmt.Sprintf("The following tenants were discovered: %s.\n Continue?", tenants)
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	}
@@ -233,7 +233,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		// do not prompt for intercluster because there could be many tenants,
 		// and we don't want to interrupt the process when moving to the next tenant.
 		question := foundSeriesMsg + ". Continue?"
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	} else {

app/vminsert/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -231,6 +232,17 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "/zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -423,6 +435,9 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vm_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 

app/vminsert/prompush/push.go

@@ -4,13 +4,15 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
-	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+	rowsInserted         = metrics.NewCounter(`vm_rows_inserted_total{type="promscrape"}`)
+	rowsPerInsert        = metrics.NewHistogram(`vm_rows_per_insert{type="promscrape"}`)
+	metadataRowsInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="promscrape"}`)
 )
 
 const maxRowsPerBlock = 10000
@@ -41,6 +43,13 @@ func Push(wr *prompb.WriteRequest) {
 		}
 		push(ctx, tssBlock)
 	}
+	if prommetadata.IsEnabled() {
+		if err := ctx.WriteMetadata(wr.Metadata); err != nil {
+			logger.Errorf("cannot write promscrape metrics metadata to storage: %s", err)
+		} else {
+			metadataRowsInserted.Add(len(wr.Metadata))
+		}
+	}
 }
 
 func push(ctx *common.InsertCtx, tss []prompb.TimeSeries) {

app/vminsert/zabbixconnector/request_handler.go

@@ -0,0 +1,67 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+)
+
+var (
+	rowsInserted  = metrics.NewCounter(`vm_rows_inserted_total{type="zabbixconnector"}`)
+	rowsPerInsert = metrics.NewHistogram(`vm_rows_per_insert{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(rows, extraLabels)
+	})
+}
+
+func insertRows(rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := common.GetInsertCtx()
+	defer common.PutInsertCtx(ctx)
+
+	rowsTotal := len(rows)
+	ctx.Reset(rowsTotal)
+	hasRelabeling := relabel.HasRelabeling()
+	for i := range rows {
+		r := &rows[i]
+		ctx.Labels = ctx.Labels[:0]
+		for k := range r.Tags {
+			t := &r.Tags[k]
+			ctx.AddLabelBytes(t.Key, t.Value)
+		}
+		for k := range extraLabels {
+			label := &extraLabels[k]
+			ctx.AddLabel(label.Name, label.Value)
+		}
+		if hasRelabeling {
+			ctx.ApplyRelabeling()
+		}
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		ctx.SortLabelsIfNeeded()
+		if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+	}
+
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	return ctx.FlushBufs()
+}

app/vmselect/main.go

@@ -520,7 +520,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 			fmt.Fprintf(w, "%s", `{"status":"error","msg":"for accessing vmalert flag '-vmalert.proxyURL' must be configured"}`)
 			return true
 		}
-		proxyVMAlertRequests(w, r)
+		proxyVMAlertRequests(w, r, path)
 		return true
 	}
 
@@ -558,7 +558,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/rules", "/rules":
 		rulesRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
@@ -568,7 +568,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/alerts", "/alerts":
 		alertsRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
@@ -578,7 +578,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/notifiers", "/notifiers":
 		notifiersRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r)
+			proxyVMAlertRequests(w, r, path)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -725,7 +725,7 @@ var (
 	metricNamesStatsResetErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
 )
 
-func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
+func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request, path string) {
 	defer func() {
 		err := recover()
 		if err == nil || err == http.ErrAbortHandler {
@@ -736,8 +736,10 @@ func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	r.Host = vmalertProxyHost
-	vmalertProxy.ServeHTTP(w, r)
+	req := r.Clone(r.Context())
+	req.URL.Path = strings.TrimPrefix(path, "prometheus")
+	req.Host = vmalertProxyHost
+	vmalertProxy.ServeHTTP(w, req)
 }
 
 var (

app/vmselect/prometheus/metadata_response.qtpl

@@ -1,6 +1,5 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
-  "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
@@ -18,7 +17,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-m
 		{% for _, row := range result %}
 			"{%s string(row.MetricFamilyName) %}": [
 					{
-						"type": {%q= prompb.MetricMetadataTypeToString(row.Type) %},
+						"type": {%q= row.Type.String() %},
             {% if len(row.Unit) > 0 -%}
 						"unit": {%q= string(row.Unit) %},
             {% endif -%}

app/vmselect/prometheus/metadata_response.qtpl.go

@@ -6,104 +6,103 @@ package prometheus
 
 //line app/vmselect/prometheus/metadata_response.qtpl:1
 import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 )
 
 // MetadataResponse generates response for /api/v1/metadataSee https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
 
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 func StreamMetadataResponse(qw422016 *qt422016.Writer, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 	qw422016.N().S(`{"status":"success","data": {`)
-//line app/vmselect/prometheus/metadata_response.qtpl:15
+//line app/vmselect/prometheus/metadata_response.qtpl:14
 	mapItems := len(result)
 	currentItem := 0
 
-//line app/vmselect/prometheus/metadata_response.qtpl:18
+//line app/vmselect/prometheus/metadata_response.qtpl:17
 	for _, row := range result {
-//line app/vmselect/prometheus/metadata_response.qtpl:18
+//line app/vmselect/prometheus/metadata_response.qtpl:17
 		qw422016.N().S(`"`)
-//line app/vmselect/prometheus/metadata_response.qtpl:19
+//line app/vmselect/prometheus/metadata_response.qtpl:18
 		qw422016.E().S(string(row.MetricFamilyName))
-//line app/vmselect/prometheus/metadata_response.qtpl:19
+//line app/vmselect/prometheus/metadata_response.qtpl:18
 		qw422016.N().S(`": [{"type":`)
-//line app/vmselect/prometheus/metadata_response.qtpl:21
-		qw422016.N().Q(prompb.MetricMetadataTypeToString(row.Type))
-//line app/vmselect/prometheus/metadata_response.qtpl:21
+//line app/vmselect/prometheus/metadata_response.qtpl:20
+		qw422016.N().Q(row.Type.String())
+//line app/vmselect/prometheus/metadata_response.qtpl:20
 		qw422016.N().S(`,`)
-//line app/vmselect/prometheus/metadata_response.qtpl:22
+//line app/vmselect/prometheus/metadata_response.qtpl:21
 		if len(row.Unit) > 0 {
-//line app/vmselect/prometheus/metadata_response.qtpl:22
+//line app/vmselect/prometheus/metadata_response.qtpl:21
 			qw422016.N().S(`"unit":`)
-//line app/vmselect/prometheus/metadata_response.qtpl:23
+//line app/vmselect/prometheus/metadata_response.qtpl:22
 			qw422016.N().Q(string(row.Unit))
+//line app/vmselect/prometheus/metadata_response.qtpl:22
+			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/metadata_response.qtpl:23
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/metadata_response.qtpl:24
 		}
-//line app/vmselect/prometheus/metadata_response.qtpl:24
+//line app/vmselect/prometheus/metadata_response.qtpl:23
 		qw422016.N().S(`"help":`)
-//line app/vmselect/prometheus/metadata_response.qtpl:25
+//line app/vmselect/prometheus/metadata_response.qtpl:24
 		qw422016.N().Q(string(row.Help))
-//line app/vmselect/prometheus/metadata_response.qtpl:25
+//line app/vmselect/prometheus/metadata_response.qtpl:24
 		qw422016.N().S(`}]`)
-//line app/vmselect/prometheus/metadata_response.qtpl:28
+//line app/vmselect/prometheus/metadata_response.qtpl:27
 		if currentItem != mapItems-1 {
-//line app/vmselect/prometheus/metadata_response.qtpl:28
+//line app/vmselect/prometheus/metadata_response.qtpl:27
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/metadata_response.qtpl:28
+//line app/vmselect/prometheus/metadata_response.qtpl:27
 		}
-//line app/vmselect/prometheus/metadata_response.qtpl:29
+//line app/vmselect/prometheus/metadata_response.qtpl:28
 		currentItem++
 
-//line app/vmselect/prometheus/metadata_response.qtpl:30
+//line app/vmselect/prometheus/metadata_response.qtpl:29
 	}
-//line app/vmselect/prometheus/metadata_response.qtpl:30
+//line app/vmselect/prometheus/metadata_response.qtpl:29
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/metadata_response.qtpl:32
+//line app/vmselect/prometheus/metadata_response.qtpl:31
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/metadata_response.qtpl:32
+//line app/vmselect/prometheus/metadata_response.qtpl:31
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 }
 
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 func WriteMetadataResponse(qq422016 qtio422016.Writer, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	StreamMetadataResponse(qw422016, result, qt)
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 }
 
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 func MetadataResponse(result []*metricsmetadata.Row, qt *querytracer.Tracer) string {
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	WriteMetadataResponse(qb422016, result, qt)
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 	return qs422016
-//line app/vmselect/prometheus/metadata_response.qtpl:34
+//line app/vmselect/prometheus/metadata_response.qtpl:33
 }

app/vmselect/promql/eval.go

@@ -785,7 +785,8 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 // - rollupFunc(m) if iafc is nil
 // - aggrFunc(rollupFunc(m)) if iafc isn't nil
 func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr,
-	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	if re.At == nil {
 		return evalRollupFuncWithoutAt(qt, ec, funcName, rf, expr, re, iafc)
 	}
@@ -835,7 +836,8 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 }
 
 func evalRollupFuncWithoutAt(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
@@ -1058,7 +1060,8 @@ func removeNanValues(dstValues []float64, dstTimestamps []int64, values []float6
 
 // evalInstantRollup evaluates instant rollup where ec.Start == ec.End.
 func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64,
+) ([]*timeseries, error) {
 	if ec.Start != ec.End {
 		logger.Panicf("BUG: evalInstantRollup cannot be called on non-empty time range; got %s", ec.timeRangeString())
 	}
@@ -1083,10 +1086,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		rollupResultCacheV.DeleteInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 	}
 	getCachedSeries := func(qt *querytracer.Tracer) ([]*timeseries, int64, error) {
+		rollupResultCacheV.rollupResultCacheRequests.Inc()
 	again:
 		offset := int64(0)
 		tssCached := rollupResultCacheV.GetInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 		if len(tssCached) == 0 {
+			rollupResultCacheV.rollupResultCacheMisses.Inc()
 			// Cache miss. Re-populate the missing data.
 			start := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds()
 			offset = timestamp - start
@@ -1129,6 +1134,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			deleteCachedSeries(qt)
 			goto again
 		}
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 		ec.QueryStats.addSeriesFetched(len(tssCached))
 		return tssCached, offset, nil
 	}
@@ -1169,60 +1175,6 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			},
 		}
 		return evalExpr(qt, ec, be)
-	case "rate":
-		if iafc != nil {
-			if !strings.EqualFold(iafc.ae.Name, "sum") {
-				qt.Printf("do not apply instant rollup optimization for incremental aggregate %s()", iafc.ae.Name)
-				return evalAt(qt, timestamp, window)
-			}
-			qt.Printf("optimized calculation for sum(rate(m[d])) as (sum(increase(m[d])) / d)")
-			afe := expr.(*metricsql.AggrFuncExpr)
-			fe := afe.Args[0].(*metricsql.FuncExpr)
-			feIncrease := *fe
-			feIncrease.Name = "increase"
-			// copy RollupExpr to drop possible offset,
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-			newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-			newArg.Offset = nil
-			feIncrease.Args = []metricsql.Expr{newArg}
-			d := newArg.Window.Duration(ec.Step)
-			if d == 0 {
-				d = ec.Step
-			}
-			afeIncrease := *afe
-			afeIncrease.Args = []metricsql.Expr{&feIncrease}
-			be := &metricsql.BinaryOpExpr{
-				Op:              "/",
-				KeepMetricNames: true,
-				Left:            &afeIncrease,
-				Right: &metricsql.NumberExpr{
-					N: float64(d) / 1000,
-				},
-			}
-			return evalExpr(qt, ec, be)
-		}
-		qt.Printf("optimized calculation for instant rollup rate(m[d]) as (increase(m[d]) / d)")
-		fe := expr.(*metricsql.FuncExpr)
-		feIncrease := *fe
-		feIncrease.Name = "increase"
-		// copy RollupExpr to drop possible offset,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-		newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-		newArg.Offset = nil
-		feIncrease.Args = []metricsql.Expr{newArg}
-		d := newArg.Window.Duration(ec.Step)
-		if d == 0 {
-			d = ec.Step
-		}
-		be := &metricsql.BinaryOpExpr{
-			Op:              "/",
-			KeepMetricNames: fe.KeepMetricNames,
-			Left:            &feIncrease,
-			Right: &metricsql.NumberExpr{
-				N: float64(d) / 1000,
-			},
-		}
-		return evalExpr(qt, ec, be)
 	case "max_over_time":
 		if iafc != nil {
 			if !strings.EqualFold(iafc.ae.Name, "max") {
@@ -1591,16 +1543,11 @@ func assertInstantValues(tss []*timeseries) {
 	}
 }
 
-var (
-	rollupResultCacheFullHits    = metrics.NewCounter(`vm_rollup_result_cache_full_hits_total`)
-	rollupResultCachePartialHits = metrics.NewCounter(`vm_rollup_result_cache_partial_hits_total`)
-	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
-
-	memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
-)
+var memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
 
 func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr,
+) ([]*timeseries, error) {
 	window, err := windowExpr.NonNegativeDuration(ec.Step)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", expr.AppendString(nil), err)
@@ -1636,19 +1583,20 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	}
 
 	// Search for cached results.
+	rollupResultCacheV.rollupResultCacheRequests.Inc()
 	tssCached, start := rollupResultCacheV.GetSeries(qt, ec, expr, window)
 	ec.QueryStats.addSeriesFetched(len(tssCached))
 	if start > ec.End {
 		qt.Printf("the result is fully cached")
-		rollupResultCacheFullHits.Inc()
+		rollupResultCacheV.rollupResultCacheFullHits.Inc()
 		return tssCached, nil
 	}
 	if start > ec.Start {
 		qt.Printf("partial cache hit")
-		rollupResultCachePartialHits.Inc()
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 	} else {
 		qt.Printf("cache miss")
-		rollupResultCacheMiss.Inc()
+		rollupResultCacheV.rollupResultCacheMisses.Inc()
 	}
 
 	// Fetch missing results, which aren't cached yet.
@@ -1684,7 +1632,8 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 //
 // pointsPerSeries is used only for estimating the needed memory for query processing
 func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64,
+) ([]*timeseries, error) {
 	if qt.Enabled() {
 		qt = qt.NewChild("rollup %s: timeRange=%s, step=%d, window=%d", expr.AppendString(nil), ec.timeRangeString(), ec.Step, window)
 		defer qt.Done()
@@ -1807,7 +1756,8 @@ func maxSilenceInterval() int64 {
 
 func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
 	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs)
 	defer qt.Done()
 	var samplesScannedTotal atomic.Uint64
@@ -1846,7 +1796,8 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 }
 
 func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
 
@@ -1886,7 +1837,8 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 }
 
 func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName,
-	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) uint64 {
+	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64,
+) uint64 {
 	tsDst.MetricName.CopyFrom(mnSrc)
 	if len(rc.TagValue) > 0 {
 		tsDst.MetricName.AddTag("rollup", rc.TagValue)

app/vmselect/promql/rollup_result_cache.go

@@ -83,9 +83,11 @@ func checkRollupResultCacheReset() {
 
 const checkRollupResultCacheResetInterval = 5 * time.Second
 
-var needRollupResultCacheReset atomic.Bool
-var checkRollupResultCacheResetOnce sync.Once
-var rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
+var (
+	needRollupResultCacheReset       atomic.Bool
+	checkRollupResultCacheResetOnce  sync.Once
+	rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
+)
 
 var rollupResultCacheV = &rollupResultCache{
 	c: workingsetcache.New(1024 * 1024), // This is a cache for testing.
@@ -132,7 +134,7 @@ func InitRollupResultCache(cachePath string) {
 		c = workingsetcache.New(cacheSize)
 		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 	}
-	if *disableCache {
+	if *disableCache && len(rollupResultCachePath) > 0 && !*resetRollupResultCacheOnStartup {
 		c.Reset()
 	}
 
@@ -178,6 +180,12 @@ func InitRollupResultCache(cachePath string) {
 
 	rollupResultCacheV = &rollupResultCache{
 		c: c,
+
+		rollupResultCacheRequests:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_requests_total`),
+		rollupResultCacheFullHits:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_full_hits_total`),
+		rollupResultCachePartialHits: metrics.GetOrCreateCounter(`vm_rollup_result_cache_partial_hits_total`),
+		rollupResultCacheMisses:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_miss_total`),
+		rollupResultCacheResets:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_resets_total`),
 	}
 }
 
@@ -193,13 +201,18 @@ func StopRollupResultCache() {
 
 type rollupResultCache struct {
 	c *workingsetcache.Cache
-}
 
-var rollupResultCacheResets = metrics.NewCounter(`vm_cache_resets_total{type="promql/rollupResult"}`)
+	rollupResultCacheRequests    *metrics.Counter
+	rollupResultCacheFullHits    *metrics.Counter
+	rollupResultCachePartialHits *metrics.Counter
+	rollupResultCacheMisses      *metrics.Counter
+
+	rollupResultCacheResets *metrics.Counter
+}
 
 // ResetRollupResultCache resets rollup result cache.
 func ResetRollupResultCache() {
-	rollupResultCacheResets.Inc()
+	rollupResultCacheV.rollupResultCacheResets.Inc()
 	rollupResultCacheKeyPrefix.Add(1)
 	logger.Infof("rollupResult cache has been cleared")
 }

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-C4E6lDpP.js"></script>
+  <script type="module" crossorigin src="./assets/index-Clpj_g75.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-D5YL0cqB.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-DACH7WjD.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-jEWkrqzO.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -29,7 +29,8 @@ import (
 )
 
 var (
-	retentionPeriod   = flagutil.NewRetentionDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
+	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -118,7 +119,7 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	}
 
 	resetResponseCacheIfNeeded = resetCacheIfNeeded
-	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
+	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
@@ -388,11 +389,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotPath := Storage.MustCreateSnapshot()
+		snapshotName := Storage.MustCreateSnapshot()
+
+		// Verify whether the client already closed the connection.
+		// In this case it is better to drop the created snapshot, since the client isn't interested in it.
+		if err := r.Context().Err(); err != nil {
+			logger.Infof("deleting already created snapshot at %s because the client canceled the request", snapshotName)
+			if err := deleteSnapshot(snapshotName); err != nil {
+				logger.Infof("cannot delete just created snapshot: %s", err)
+				return true
+			}
+			return true
+		}
+
 		if prometheusCompatibleResponse {
-			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotPath))
+			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotName))
 		} else {
-			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotPath))
+			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotName))
 		}
 		return true
 	case "/list":
@@ -412,23 +425,12 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		snapshotsDeleteTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
-
-		snapshots := Storage.MustListSnapshots()
-		for _, snName := range snapshots {
-			if snName == snapshotName {
-				if err := Storage.DeleteSnapshot(snName); err != nil {
-					err = fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
-					jsonResponseError(w, err)
-					snapshotsDeleteErrorsTotal.Inc()
-					return true
-				}
-				fmt.Fprintf(w, `{"status":"ok"}`)
-				return true
-			}
+		if err := deleteSnapshot(snapshotName); err != nil {
+			jsonResponseError(w, err)
+			snapshotsDeleteErrorsTotal.Inc()
+			return true
 		}
-
-		err := fmt.Errorf("cannot find snapshot %q", snapshotName)
-		jsonResponseError(w, err)
+		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
@@ -449,6 +451,19 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
+func deleteSnapshot(snapshotName string) error {
+	snapshots := Storage.MustListSnapshots()
+	for _, snName := range snapshots {
+		if snName == snapshotName {
+			if err := Storage.DeleteSnapshot(snName); err != nil {
+				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("cannot find snapshot %q", snapshotName)
+}
+
 func initStaleSnapshotsRemover(strg *storage.Storage) {
 	staleSnapshotsRemoverCh = make(chan struct{})
 	if snapshotsMaxAge.Duration() <= 0 {
@@ -500,7 +515,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
 	tm := &m.TableMetrics
-	idbm := &m.IndexDBMetrics
+	idbm := &m.TableMetrics.IndexDBMetrics
 
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *DataPath), fs.MustGetFreeSpace(*DataPath))
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *DataPath), uint64(minFreeDiskSpaceBytes.N))
@@ -628,77 +643,82 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 
 	metrics.WriteCounterUint64(w, `vm_missing_metric_names_for_metric_id_total`, idbm.MissingMetricNamesForMetricID)
 
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_syncs_total`, idbm.DateMetricIDCacheSyncsCount)
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_resets_total`, idbm.DateMetricIDCacheResetsCount)
-
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/tsid"}`, m.TSIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricIDs"}`, m.MetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricName"}`, m.MetricNameCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/metricID"}`, idbm.MetricIDCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/tsid"}`, m.TSIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, storage.RegexpCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/metricID"}`, idbm.MetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, storage.RegexpCacheSizeBytes())
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheSizeBytes())
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/tsid"}`, m.TSIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, storage.RegexpCacheMaxSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMaxSizeBytes())
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, storage.RegexpCacheMaxSizeBytes())
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMaxSizeBytes())
 
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/tsid"}`, m.TSIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricIDs"}`, m.MetricIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricName"}`, m.MetricNameCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/tsid"}`, m.TSIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricIDs"}`, m.MetricIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricName"}`, m.MetricNameCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 
 	metrics.WriteCounterUint64(w, `vm_cache_resets_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheResets)
 
-	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
-
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/tsid"}`, m.TSIDCacheCollisions)
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/metricName"}`, m.MetricNameCacheCollisions)
 
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/metricID"}`, idbm.MetricIDCacheSyncsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSyncsCount)
+
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/metricID"}`, idbm.MetricIDCacheRotationsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheRotationsCount)
+
+	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
+
 	metrics.WriteGaugeUint64(w, `vm_next_retention_seconds`, m.NextRetentionSeconds)
 
 	if *trackMetricNamesStats {

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.4 AS build-web-stage
+FROM golang:1.25.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/eslint.config.js

@@ -46,7 +46,7 @@ export default [...compat.extends(
   settings: {
     react: {
       pragma: "React",
-      version: "detect",
+      version: "19.0",
     },
 
     linkComponents: ["Hyperlink", {
@@ -69,10 +69,11 @@ export default [...compat.extends(
       "varsIgnorePattern": "^_",
       "ignoreRestSiblings": true
     }],
-    
+
     "unused-imports/no-unused-imports": "error",
 
     "react/jsx-closing-bracket-location": [1, "line-aligned"],
+    "object-curly-spacing": [2, "always"],
 
     "react/jsx-max-props-per-line": [1, {
       maximum: 1,
@@ -81,13 +82,23 @@ export default [...compat.extends(
     "react/jsx-first-prop-new-line": [1, "multiline"],
 
     // Disable core indent rule due to recursion issues in ESLint 9; use JSX-specific rules instead
-    indent: "off",
+    indent: ["error", 2, {
+      SwitchCase: 1,
+      ignoredNodes: [
+        "JSXElement",
+        "JSXElement *",
+        "JSXFragment",
+        "JSXFragment *",
+      ],
+    }],
     "react/jsx-indent": ["error", 2],
     "react/jsx-indent-props": ["error", 2],
 
+    "linebreak-style": ["error", "unix"],
+    quotes: ["error", "double"],
+    semi: ["error", "always"],
     // Formatting rules moved out of ESLint core; omit here to avoid deprecation noise
     "react/prop-types": 0,
     "react/react-in-jsx-scope": "off",
-
   },
 }];

app/vmui/packages/vmui/package.json

@@ -18,47 +18,48 @@
     "preview": "vite preview",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:dev": "vitest"
+    "test:dev": "vitest",
+    "precommit": "npm run lint:local && npm run typecheck && npm run test"
   },
   "dependencies": {
     "classnames": "^2.5.1",
-    "dayjs": "^1.11.13",
+    "dayjs": "^1.11.19",
     "lodash.debounce": "^4.0.8",
-    "marked": "^16.0.0",
-    "preact": "^10.26.9",
-    "qs": "^6.14.0",
+    "marked": "^17.0.1",
+    "preact": "^10.28.2",
+    "qs": "^6.14.1",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.6.3",
+    "react-router-dom": "^7.12.0",
     "uplot": "^1.6.32",
-    "vite": "^7.1.11",
-    "web-vitals": "^5.0.3"
+    "vite": "^7.3.1",
+    "web-vitals": "^5.1.0"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.1",
-    "@eslint/js": "^9.30.1",
+    "@eslint/eslintrc": "^3.3.3",
+    "@eslint/js": "^9.39.2",
     "@preact/preset-vite": "^2.10.2",
-    "@testing-library/jest-dom": "^6.6.3",
+    "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^24.0.12",
+    "@types/node": "^25.0.8",
     "@types/qs": "^6.14.0",
-    "@types/react": "^19.1.8",
+    "@types/react": "^19.2.8",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.36.0",
-    "@typescript-eslint/parser": "^8.36.0",
-    "cross-env": "^7.0.3",
-    "eslint": "^9.30.1",
+    "@typescript-eslint/eslint-plugin": "^8.53.0",
+    "@typescript-eslint/parser": "^8.53.0",
+    "cross-env": "^10.1.0",
+    "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
-    "eslint-plugin-unused-imports": "^4.1.4",
-    "globals": "^16.3.0",
+    "eslint-plugin-unused-imports": "^4.3.0",
+    "globals": "^17.0.0",
     "http-proxy-middleware": "^3.0.5",
-    "jsdom": "^26.1.0",
+    "jsdom": "^27.4.0",
     "postcss": "^8.5.6",
-    "rollup-plugin-visualizer": "^6.0.3",
-    "sass-embedded": "^1.89.2",
-    "typescript": "^5.8.3",
-    "vitest": "^3.2.4"
+    "rollup-plugin-visualizer": "^6.0.5",
+    "sass-embedded": "^1.97.2",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.17"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -8,7 +8,8 @@ import { useHideDuplicateFields } from "./hooks/useHideDuplicateFields";
 import Accordion from "../../../Main/Accordion/Accordion";
 import { useLegendGroup } from "./hooks/useLegendGroup";
 import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
-import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
+import { LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import { getFromStorage } from "../../../../utils/storage";
 
 export type LegendProps = {
   labels: LegendItemType[];
@@ -38,17 +39,26 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
 
   const Content = isTableView ? LegendTable : LegendLines;
 
+  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false";
+  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT;
+
+  const expandedWarning = (
+    <span className="vm-legend-group-header__warning">
+      Legend collapsed by default ({sortedLabels.length} series) — click to expand.
+    </span>
+  );
+
   return (
     <div
       className="vm-legend-group"
       key={group}
     >
       <Accordion
-        defaultExpanded={sortedLabels.length < DEFAULT_MAX_SERIES.chart}
+        defaultExpanded={defaultExpanded}
         title={(
           <div className="vm-legend-group-header">
             <div className="vm-legend-group-header-title">
-              Group by{groupByLabel ? "" : " query"}: <b>{group}</b>
+              Group by{groupByLabel ? "" : " query"}: <b>{group}</b> {!defaultExpanded && expandedWarning}
             </div>
             {!!duplicateFields.length && (
               <div className="vm-legend-group-header-labels">

app/vmui/packages/vmui/src/components/Chart/Line/Legend/style.scss

@@ -32,6 +32,14 @@
         }
       }
 
+      &__warning {
+        flex-grow: 1;
+        text-align: right;
+        padding-right: calc($padding-large * 2);
+        font-size: $font-size-small;
+        color: $color-warning;
+      }
+
       &-labels {
         display: flex;
         flex-wrap: wrap;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -1,15 +1,17 @@
-import { forwardRef, useCallback, useImperativeHandle, useState } from "preact/compat";
+import { forwardRef, useCallback, useEffect, useImperativeHandle, useState } from "preact/compat";
 import { DisplayType, ErrorTypes } from "../../../../types";
 import TextField from "../../../Main/TextField/TextField";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import { InfoIcon, RestartIcon } from "../../../Main/Icons";
 import Button from "../../../Main/Button/Button";
-import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
+import { DEFAULT_MAX_SERIES, LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../../state/customPanel/CustomPanelStateContext";
+import Switch from "../../../Main/Switch/Switch";
+import { getFromStorage, saveToStorage } from "../../../../utils/storage";
 
 interface ServerConfiguratorProps {
   onClose: () => void
@@ -27,6 +29,9 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   const { seriesLimits } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
+  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
+  const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
+
   const [limits, setLimits] = useState(seriesLimits);
   const [error, setError] = useState({
     table: "",
@@ -52,6 +57,10 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     onClose();
   }, [limits]);
 
+  useEffect(() => {
+    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
+  }, [legendCollapse]);
+
   useImperativeHandle(ref, () => ({ handleApply }), [handleApply]);
 
   return (
@@ -97,6 +106,19 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
           </div>
         ))}
       </div>
+
+      <div className="vm-graph-settings-row">
+        <span className="vm-graph-settings-row__label">Auto-collapse legend</span>
+        <Switch
+          value={legendCollapse}
+          onChange={setLegendCollapse}
+          label={legendCollapse ? "Enabled" : "Disabled"}
+          fullWidth={isMobile}
+        />
+        <span className="vm-legend-configs-item__info">
+          Collapses the legend when series count exceeds {LEGEND_COLLAPSE_SERIES_LIMIT} to reduce UI load.
+        </span>
+      </div>
     </div>
   );
 });

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/style.scss

@@ -18,6 +18,7 @@
     align-items: center;
     justify-content: space-between;
     gap: $padding-global;
+    margin-bottom: $padding-global;
 
     &_mobile {
       gap: $padding-small;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/ServerConfigurator/ServerConfigurator.tsx

@@ -9,7 +9,6 @@ import { getFromStorage, removeFromStorage, saveToStorage } from "../../../../ut
 import useBoolean from "../../../../hooks/useBoolean";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
-import { getTenantIdFromUrl } from "../../../../utils/tenants";
 
 interface ServerConfiguratorProps {
   onClose: () => void;
@@ -39,10 +38,6 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   };
 
   const handleApply = useCallback(() => {
-    const tenantIdFromUrl = getTenantIdFromUrl(serverUrl);
-    if (tenantIdFromUrl !== "") {
-      dispatch({ type: "SET_TENANT_ID", payload: tenantIdFromUrl });
-    }
     dispatch({ type: "SET_SERVER", payload: serverUrl });
     onClose();
   }, [serverUrl]);
@@ -60,12 +55,6 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     }
   }, [enabledStorage]);
 
-  useEffect(() => {
-    if (enabledStorage) {
-      saveToStorage("SERVER_URL", serverUrl);
-    }
-  }, [serverUrl]);
-
   useEffect(() => {
     // the tenant selector can change the serverUrl
     if (stateServerUrl === serverUrl) return;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration.tsx

@@ -1,4 +1,4 @@
-import { FC, useState, useRef, useEffect, useMemo } from "preact/compat";
+import { FC, useState, useRef, useMemo } from "preact/compat";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { ArrowDownIcon, StorageIcon } from "../../../Main/Icons";
@@ -10,14 +10,14 @@ import { getAppModeEnable } from "../../../../utils/app-mode";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import TextField from "../../../Main/TextField/TextField";
-import { getTenantIdFromUrl, replaceTenantId } from "../../../../utils/tenants";
+import { replaceTenantId } from "../../../../utils/tenants";
 import useBoolean from "../../../../hooks/useBoolean";
 
 const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   const appModeEnable = getAppModeEnable();
   const { isMobile } = useDeviceDetect();
 
-  const { tenantId: tenantIdState, serverUrl } = useAppState();
+  const { tenantId, serverUrl } = useAppState();
   const dispatch = useAppDispatch();
   const timeDispatch = useTimeDispatch();
 
@@ -48,10 +48,8 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   }, [accountIds]);
 
   const createHandlerChange = (value: string) => () => {
-    const tenant = value;
-    dispatch({ type: "SET_TENANT_ID", payload: tenant });
     if (serverUrl) {
-      const updateServerUrl = replaceTenantId(serverUrl, tenant);
+      const updateServerUrl = replaceTenantId(serverUrl, value);
       if (updateServerUrl === serverUrl) return;
       dispatch({ type: "SET_SERVER", payload: updateServerUrl });
       timeDispatch({ type: "RUN_QUERY" });
@@ -59,16 +57,6 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
     handleCloseOptions();
   };
 
-  useEffect(() => {
-    const id = getTenantIdFromUrl(serverUrl);
-
-    if (tenantIdState && tenantIdState !== id) {
-      createHandlerChange(tenantIdState)();
-    } else {
-      createHandlerChange(id)();
-    }
-  }, [serverUrl]);
-
   if (!showTenantSelector) return null;
 
   return (
@@ -83,7 +71,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               <span className="vm-mobile-option__icon"><StorageIcon/></span>
               <div className="vm-mobile-option-text">
                 <span className="vm-mobile-option-text__label">Tenant ID</span>
-                <span className="vm-mobile-option-text__value">{tenantIdState}</span>
+                <span className="vm-mobile-option-text__value">{tenantId}</span>
               </div>
               <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
             </div>
@@ -106,7 +94,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               )}
               onClick={toggleOpenOptions}
             >
-              {tenantIdState}
+              {tenantId}
             </Button>
           )}
         </div>
@@ -138,7 +126,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               className={classNames({
                 "vm-list-item": true,
                 "vm-list-item_mobile": isMobile,
-                "vm-list-item_active": id === tenantIdState
+                "vm-list-item_active": id === tenantId
               })}
               key={id}
               onClick={createHandlerChange(id)}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/hooks/useFetchAccountIds.ts

@@ -3,19 +3,18 @@ import { useEffect, useMemo, useState } from "preact/compat";
 import { ErrorTypes } from "../../../../../types";
 import { getAccountIds } from "../../../../../api/accountId";
 import { getAppModeEnable, getAppModeParams } from "../../../../../utils/app-mode";
-import { getTenantIdFromUrl } from "../../../../../utils/tenants";
 
 export const useFetchAccountIds = () => {
   const { useTenantID } = getAppModeParams();
   const appModeEnable = getAppModeEnable();
-  const { serverUrl } = useAppState();
+  const { tenantId, serverUrl } = useAppState();
 
   const [isLoading, setIsLoading] = useState(false);
   const [error, setError] = useState<ErrorTypes | string>();
   const [accountIds, setAccountIds] = useState<string[]>([]);
 
   const fetchUrl = useMemo(() => getAccountIds(serverUrl), [serverUrl]);
-  const isServerUrlWithTenant = useMemo(() => !!getTenantIdFromUrl(serverUrl), [serverUrl]);
+  const isServerUrlWithTenant = useMemo(() => !!tenantId, [tenantId]);
   const preventFetch = appModeEnable ? !useTenantID : !isServerUrlWithTenant;
 
   useEffect(() => {

app/vmui/packages/vmui/src/components/ExploreAlerts/helpers.ts

@@ -17,4 +17,4 @@ export const formatDuration = (raw: number) => {
 export const formatEventTime = (raw: string) => {
   const t = dayjs(raw);
   return t.year() <= 1 ? "Never" : t.format("DD MMM YYYY HH:mm:ss");
-}
+};

app/vmui/packages/vmui/src/components/Main/Button/style.scss

@@ -1,4 +1,5 @@
 @use "src/styles/variables" as *;
+@use 'sass:meta';
 
 $button-radius: 6px;
 
@@ -42,6 +43,8 @@ $button-radius: 6px;
 
     svg {
       width: 14px;
+      min-width: 14px;
+      max-width: 14px;
     }
   }
 
@@ -51,6 +54,8 @@ $button-radius: 6px;
 
     svg {
       width: 16px;
+      min-width: 16px;
+      max-width: 16px;
     }
   }
 
@@ -60,6 +65,8 @@ $button-radius: 6px;
 
     svg {
       width: 18px;
+      min-width: 18px;
+      max-width: 18px;
       line-height: 16px;
     }
   }
@@ -128,8 +135,14 @@ $button-radius: 6px;
   );
 
   @each $name, $color in $button-colors {
-    @include contained-button($name, $color, if($name == white, $color-black, $color-white));
-    @include outlined-button($name, $color, if($name == white, $color-white, $color));
-    @include text-button($name, if($name == white, $color-white, $color));
+    @if $name == white {
+      @include contained-button($name, $color, $color-black);
+      @include outlined-button($name, $color, $color-white);
+      @include text-button($name, $color-white);
+    } @else {
+      @include contained-button($name, $color, $color-white);
+      @include outlined-button($name, $color, $color);
+      @include text-button($name, $color);
+    }
   }
 }

app/vmui/packages/vmui/src/components/QueryHistory/utils.ts

@@ -1,4 +1,8 @@
-import { getFromStorage, removeFromStorage, saveToStorage, StorageKeys } from "../../utils/storage";
+import {
+  getFromStorage,
+  saveToStorage,
+  StorageKeys,
+} from "../../utils/storage";
 import { QueryHistoryType } from "../../state/query/reducer";
 import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../constants/graph";
 
@@ -73,17 +77,3 @@ export const getUpdatedHistory = (query: string, queryHistory?: QueryHistoryType
     values: newValues
   };
 };
-
-const migrateMetricsQueryHistoryToHistoryByKey = () => {
-  const migrateHistory = (type: HistoryType) => {
-    const queryList = getFromStorage(type) as string;
-    if (queryList) {
-      const queryHistory: string[][] = JSON.parse(queryList);
-      saveHistoryToStorage("METRICS_QUERY_HISTORY", type, queryHistory);
-      removeFromStorage([type]);
-    }
-  };
-  migrateHistory("QUERY_HISTORY");
-  migrateHistory("QUERY_FAVORITES");
-};
-migrateMetricsQueryHistoryToHistoryByKey();

app/vmui/packages/vmui/src/components/WebStorageCheck/WebStorageCheck.tsx

@@ -0,0 +1,69 @@
+import { useEffect } from "react";
+import { StorageErrorCode } from "./types";
+import { useSnack } from "../../contexts/Snackbar";
+import { storageErrorInfo } from "./storageErrors";
+import "./style.scss";
+
+const classifyStorageException = (e: unknown): StorageErrorCode => {
+  if (!(e instanceof DOMException)) return StorageErrorCode.UNKNOWN;
+
+  switch (e.name) {
+    case "QuotaExceededError":
+      return StorageErrorCode.QUOTA_EXCEEDED;
+    case "SecurityError":
+      return StorageErrorCode.SECURITY_ERROR;
+    default:
+      return StorageErrorCode.UNKNOWN;
+  }
+};
+
+const getStorageError = (storage: Storage | null | undefined): StorageErrorCode | null => {
+  if (!storage) {
+    return StorageErrorCode.NO_STORAGE;
+  }
+
+  try {
+    const key = "__vmui_test__";
+    storage.setItem(key, "1");
+    storage.removeItem(key);
+    return null;
+  } catch (e) {
+    return classifyStorageException(e);
+  }
+};
+
+const WebStorageCheck = () => {
+  const { showInfoMessage } = useSnack();
+
+  useEffect(() => {
+    const error = getStorageError(window.localStorage);
+
+    if (error) {
+      const { title, description, fix } = storageErrorInfo[error];
+
+      const text = (
+        <div className="vm-storage-check">
+          <h3>{title}</h3>
+          <p>{description}</p>
+
+          {!!fix?.length && (
+            <div className="vm-storage-check__fix">
+              <div>Try this:</div>
+              <ul>
+                {fix.map((step, i) => (
+                  <li key={`${i}-${step}`}>{step}</li>
+                ))}
+              </ul>
+            </div>
+          )}
+        </div>
+      );
+
+      showInfoMessage({ text: text, type: "error", timeout: 600000 });
+    }
+  }, []);
+
+  return null;
+};
+
+export default WebStorageCheck;

app/vmui/packages/vmui/src/components/WebStorageCheck/storageErrors.ts

@@ -0,0 +1,47 @@
+import { StorageError, StorageErrorCode } from "./types";
+
+export const storageErrorInfo: Record<StorageErrorCode, StorageError> = {
+  [StorageErrorCode.NO_STORAGE]: {
+    title: "Storage unavailable",
+    description:
+      "Browser storage is not available for this website.",
+    fix: [
+      "Disable Private/Incognito mode and reload the page.",
+      "Disable privacy or ad-blocking extensions for this site and reload.",
+      "Open the site in another browser.",
+    ],
+  },
+
+  [StorageErrorCode.SECURITY_ERROR]: {
+    title: "Storage access blocked",
+    description:
+      "Browser settings or an extension are blocking access to browser storage.",
+    fix: [
+      "Disable Private/Incognito mode and reload the page.",
+      "Disable privacy or ad-blocking extensions for this site and reload.",
+      "Open the site in a regular browser tab (not embedded).",
+    ],
+  },
+
+  [StorageErrorCode.QUOTA_EXCEEDED]: {
+    title: "Storage quota exceeded",
+    description:
+      "The storage limit for this website has been reached.",
+    fix: [
+      "Clear this website’s stored data and reload the page.",
+      "Close other tabs for this website and try again.",
+      "Use another browser or browser profile.",
+    ],
+  },
+
+  [StorageErrorCode.UNKNOWN]: {
+    title: "Storage error",
+    description:
+      "An unexpected error occurred while accessing browser storage.",
+    fix: [
+      "Reload the page.",
+      "Update the browser and try again.",
+      "Disable browser extensions and reload.",
+    ],
+  },
+};

app/vmui/packages/vmui/src/components/WebStorageCheck/style.scss

@@ -0,0 +1,11 @@
+@use "src/styles/variables" as *;
+
+.vm-storage-check {
+  h3 {
+    font-weight: bold;
+  }
+
+  p {
+    margin-bottom: $padding-global
+  }
+}

app/vmui/packages/vmui/src/components/WebStorageCheck/types.ts

@@ -0,0 +1,13 @@
+
+export enum StorageErrorCode {
+  NO_STORAGE = "NO_STORAGE",
+  SECURITY_ERROR = "SECURITY_ERROR",
+  QUOTA_EXCEEDED = "QUOTA_EXCEEDED",
+  UNKNOWN = "UNKNOWN",
+}
+
+export type StorageError = {
+  title: string;
+  description: string;
+  fix: string[]
+}

app/vmui/packages/vmui/src/constants/graph.ts

@@ -8,6 +8,8 @@ export const DEFAULT_MAX_SERIES = {
   code: 1000,
 };
 
+export const LEGEND_COLLAPSE_SERIES_LIMIT = 100;
+
 export const GRAPH_SIZES: GraphSize[] = [
   {
     id: "small",

app/vmui/packages/vmui/src/layouts/MainLayout/MainLayout.tsx

@@ -12,6 +12,8 @@ import useDeviceDetect from "../../hooks/useDeviceDetect";
 import ControlsMainLayout from "./ControlsMainLayout";
 import useFetchDefaultTimezone from "../../hooks/useFetchDefaultTimezone";
 import useFetchAppConfig from "../../hooks/useFetchAppConfig";
+import WebStorageCheck from "../../components/WebStorageCheck/WebStorageCheck";
+import { migrateStorageToPrefixedKeys } from "../../utils/storage";
 
 const MainLayout: FC = () => {
   const appModeEnable = getAppModeEnable();
@@ -45,6 +47,13 @@ const MainLayout: FC = () => {
   useEffect(setDocumentTitle, [pathname]);
   useEffect(redirectSearchToHashParams, []);
 
+  useEffect(() => {
+    const migrateStorage = migrateStorageToPrefixedKeys();
+    if (migrateStorage.removed.length || migrateStorage.migrated.length) {
+      console.info(migrateStorage);
+    }
+  }, []);
+
   return <section className="vm-container">
     <Header controlsComponent={ControlsMainLayout}/>
     <div
@@ -57,6 +66,8 @@ const MainLayout: FC = () => {
       <Outlet/>
     </div>
     {!appModeEnable && <Footer/>}
+
+    <WebStorageCheck/>
   </section>;
 };
 

app/vmui/packages/vmui/src/pages/CardinalityPanel/hooks/useCardinalityFetch.ts

@@ -7,7 +7,6 @@ import AppConfigurator from "../appConfigurator";
 import { useSearchParams } from "react-router-dom";
 import dayjs from "dayjs";
 import { DATE_FORMAT } from "../../../constants/date";
-import { getTenantIdFromUrl } from "../../../utils/tenants";
 import usePrevious from "../../../hooks/usePrevious";
 
 export const useFetchQuery = (): {
@@ -27,7 +26,7 @@ export const useFetchQuery = (): {
   const prevDate = usePrevious(date);
   const prevTotal = useRef<{ data: TSDBStatus }>();
 
-  const { serverUrl } = useAppState();
+  const { tenantId, serverUrl } = useAppState();
   const [isLoading, setIsLoading] = useState(false);
   const [error, setError] = useState<ErrorTypes | string>();
   const [tsdbStatus, setTSDBStatus] = useState<TSDBStatus>(appConfigurator.defaultTSDBStatus);
@@ -158,9 +157,8 @@ export const useFetchQuery = (): {
   }, [error]);
 
   useEffect(() => {
-    const id = getTenantIdFromUrl(serverUrl);
-    setIsCluster(!!id);
-  }, [serverUrl]);
+    setIsCluster(!!tenantId);
+  }, [tenantId]);
 
 
   appConfigurator.tsdbStatusData = tsdbStatus;

app/vmui/packages/vmui/src/pages/CustomPanel/hooks/useSetQueryParams.ts

@@ -1,7 +1,6 @@
 import { useEffect, useState } from "react";
 import { useTimeDispatch, useTimeState } from "../../../state/time/TimeStateContext";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
-import { useAppDispatch, useAppState } from "../../../state/common/StateContext";
 import { useQueryDispatch, useQueryState } from "../../../state/query/QueryStateContext";
 import { displayTypeTabs } from "../DisplayTypeSwitch";
 import { useGraphDispatch, useGraphState } from "../../../state/graph/GraphStateContext";
@@ -15,14 +14,12 @@ import { arrayEquals } from "../../../utils/array";
 import { isEqualURLSearchParams } from "../../../utils/url";
 
 export const useSetQueryParams = () => {
-  const { tenantId } = useAppState();
   const { displayType } = useCustomPanelState();
   const { query } = useQueryState();
   const { duration, relativeTime, period: { date, step } } = useTimeState();
   const { customStep } = useGraphState();
   const [searchParams, setSearchParams] = useSearchParams();
 
-  const dispatch = useAppDispatch();
   const timeDispatch = useTimeDispatch();
   const graphDispatch = useGraphDispatch();
   const queryDispatch = useQueryDispatch();
@@ -72,10 +69,6 @@ export const useSetQueryParams = () => {
       if (searchParams.get(`${group}.tab`) !== displayTypeCode) {
         newSearchParams.set(`${group}.tab`, `${displayTypeCode}`);
       }
-
-      if (searchParams.get(`${group}.tenantID`) !== tenantId && tenantId) {
-        newSearchParams.set(`${group}.tenantID`, tenantId);
-      }
     });
 
     // Remove extra parameters that exceed the request size
@@ -89,7 +82,7 @@ export const useSetQueryParams = () => {
 
     if (isEqualURLSearchParams(newSearchParams, searchParams) || !newSearchParams.size) return;
     setSearchParams(newSearchParams);
-  }, [tenantId, displayType, query, duration, relativeTime, date, step, customStep]);
+  }, [displayType, query, duration, relativeTime, date, step, customStep]);
 
   useEffect(() => {
     const timer = setTimeout(setterSearchParams, 200);
@@ -114,11 +107,6 @@ export const useSetQueryParams = () => {
       customPanelDispatch({ type: "SET_DISPLAY_TYPE", payload: displayTypeFromUrl });
     }
 
-    const tenantIdFromUrl = searchParams.get("g0.tenantID") || "";
-    if (tenantIdFromUrl !== tenantId) {
-      dispatch({ type: "SET_TENANT_ID", payload: tenantIdFromUrl });
-    }
-
     const queryFromUrl = getQueryArray();
     if (!arrayEquals(queryFromUrl, query)) {
       queryDispatch({ type: "SET_QUERY", payload: queryFromUrl });

app/vmui/packages/vmui/src/state/common/StateContext.tsx

@@ -1,7 +1,8 @@
-import { createContext, FC, useContext, useMemo, useReducer } from "preact/compat";
+import { createContext, FC, useContext, useEffect, useMemo, useReducer } from "preact/compat";
 import { Action, AppState, initialState, reducer } from "./reducer";
 import { getQueryStringValue } from "../../utils/query-string";
 import { Dispatch } from "react";
+import { getFromStorage, removeFromStorage, saveToStorage } from "../../utils/storage";
 
 type StateContextType = { state: AppState, dispatch: Dispatch<Action> };
 
@@ -23,6 +24,17 @@ export const AppStateProvider: FC = ({ children }) => {
     return { state, dispatch };
   }, [state, dispatch]);
 
+  useEffect(() => {
+    if (!state.serverUrl) return;
+    const enabledStorage = !!getFromStorage("SERVER_URL");
+
+    if (enabledStorage) {
+      saveToStorage("SERVER_URL", state.serverUrl);
+    } else {
+      removeFromStorage(["SERVER_URL"]);
+    }
+  }, [state.serverUrl]);
+
   return <StateContext.Provider value={contextValue}>
     {children}
   </StateContext.Provider>;

app/vmui/packages/vmui/src/state/common/reducer.ts

@@ -1,9 +1,9 @@
 import { getDefaultServer } from "../../utils/default-server-url";
-import { getQueryStringValue } from "../../utils/query-string";
 import { getFromStorage, saveToStorage } from "../../utils/storage";
 import { AppConfig, Theme } from "../../types";
 import { isDarkTheme } from "../../utils/theme";
 import { removeTrailingSlash } from "../../utils/url";
+import { getTenantIdFromUrl } from "../../utils/tenants";
 
 export interface AppState {
   serverUrl: string;
@@ -16,15 +16,14 @@ export interface AppState {
 export type Action =
   | { type: "SET_SERVER", payload: string }
   | { type: "SET_THEME", payload: Theme }
-  | { type: "SET_TENANT_ID", payload: string }
   | { type: "SET_APP_CONFIG", payload: AppConfig }
   | { type: "SET_DARK_THEME" }
 
-const tenantId = getQueryStringValue("g0.tenantID", "") as string;
+const serverUrl = removeTrailingSlash(getDefaultServer());
 
 export const initialState: AppState = {
-  serverUrl: removeTrailingSlash(getDefaultServer(tenantId)),
-  tenantId,
+  serverUrl,
+  tenantId: getTenantIdFromUrl(serverUrl),
   theme: (getFromStorage("THEME") || Theme.system) as Theme,
   isDarkTheme: null,
   appConfig: {}
@@ -35,13 +34,9 @@ export function reducer(state: AppState, action: Action): AppState {
     case "SET_SERVER":
       return {
         ...state,
+        tenantId: getTenantIdFromUrl(action.payload),
         serverUrl: removeTrailingSlash(action.payload)
       };
-    case "SET_TENANT_ID":
-      return {
-        ...state,
-        tenantId: action.payload
-      };
     case "SET_THEME":
       saveToStorage("THEME", action.payload);
       return {

app/vmui/packages/vmui/src/utils/default-server-url.test.ts

@@ -20,7 +20,7 @@ describe("test server urls", () => {
 
     it("https://play.vm.com/#/rules?q=test", () => {
       const result = getDefaultURL("https://play.vm.com/#/rules?q=test");
-      expect(result).toBe("https://play.vm.com");
+      expect(result).toBe("https://play.vm.com/prometheus");
     });
   });
 });

app/vmui/packages/vmui/src/utils/default-server-url.ts

@@ -1,13 +1,12 @@
 import { getAppModeParams } from "./app-mode";
-import { replaceTenantId } from "./tenants";
 import { APP_TYPE, AppType } from "../constants/appType";
 import { getFromStorage } from "./storage";
 
 export const getDefaultURL = (u: string) => {
-  return u.replace(/(\/(?:prometheus\/)?(?:graph|vmui)\/.*|\/#\/.*)/, "").replace(/(\/select\/[^/]+)$/, "$1/prometheus");
+  return u.replace(/(\/(?:prometheus\/)?(?:graph|vmui)\/.*|\/#\/.*)/, "/prometheus");
 };
 
-export const getDefaultServer = (tenantId?: string): string => {
+export const getDefaultServer = (): string => {
   const { serverURL } = getAppModeParams();
   const storageURL = getFromStorage("SERVER_URL") as string;
   const anomalyURL = `${window.location.origin}${window.location.pathname.replace(/^\/vmui/, "")}`;
@@ -18,6 +17,6 @@ export const getDefaultServer = (tenantId?: string): string => {
     case AppType.vmanomaly:
       return storageURL || anomalyURL;
     default:
-      return tenantId ? replaceTenantId(url, tenantId) : url;
+      return url;
   }
 };

app/vmui/packages/vmui/src/utils/storage.ts

@@ -1,47 +1,105 @@
-/**
- * Do not use this type in local storage type
- * @deprecated
- * */
-type DeprecatedStorageKeys = "QUERY_HISTORY" | "QUERY_FAVORITES";
+const STORAGE_PREFIX = "VMUI:" as const;
 
-export type StorageKeys = "AUTOCOMPLETE"
-  | "NO_CACHE"
-  | "QUERY_TRACING"
-  | "SERIES_LIMITS"
-  | "TABLE_COMPACT"
-  | "TIMEZONE"
-  | "DISABLED_DEFAULT_TIMEZONE"
-  | "THEME"
-  | "EXPLORE_METRICS_TIPS"
-  | "METRICS_QUERY_HISTORY"
-  | "SERVER_URL"
-  | "RAW_JSON_LIVE_VIEW"
-  | "POINTS_SHOW_ALL"
-  | DeprecatedStorageKeys;
+export const ALL_STORAGE_KEYS = [
+  "AUTOCOMPLETE",
+  "NO_CACHE",
+  "QUERY_TRACING",
+  "SERIES_LIMITS",
+  "LEGEND_AUTO_COLLAPSE",
+  "TABLE_COMPACT",
+  "TIMEZONE",
+  "DISABLED_DEFAULT_TIMEZONE",
+  "THEME",
+  "EXPLORE_METRICS_TIPS",
+  "METRICS_QUERY_HISTORY",
+  "SERVER_URL",
+  "POINTS_SHOW_ALL",
+] as const;
 
+export type StorageKeys = (typeof ALL_STORAGE_KEYS)[number];
 
-export const saveToStorage = (key: StorageKeys, value: string | boolean | Record<string, unknown>): void => {
-  if (value) {
-    // keeping object in storage so that keeping the string is not different from keeping
-    window.localStorage.setItem(key, JSON.stringify({ value }));
-  } else {
-    removeFromStorage([key]);
-  }
-  window.dispatchEvent(new Event("storage"));
+type PrefixedStorageKeys = `${typeof STORAGE_PREFIX}${StorageKeys}`;
+
+const toPrefixedKey = (key: StorageKeys): PrefixedStorageKeys => {
+  return `${STORAGE_PREFIX}${key}`;
 };
 
-// TODO: make this aware of data type that is stored
-export const getFromStorage = (key: StorageKeys): undefined | boolean | string | Record<string, unknown> => {
-  const valueObj = window.localStorage.getItem(key);
-  if (valueObj === null) {
-    return undefined;
-  } else {
-    try {
-      return JSON.parse(valueObj)?.value; // see comment in "saveToStorage"
-    } catch (e) {
-      return valueObj; // fallback for corrupted json
+type StorageValue = string | boolean | Record<string, unknown>;
+
+export const saveToStorage = (key: StorageKeys, value: StorageValue, withPrefix = true): void => {
+  try {
+    const storageKey = withPrefix ? toPrefixedKey(key) : key;
+
+    if (value) {
+      // keeping object in storage so that keeping the string is not different from keeping
+      window.localStorage.setItem(storageKey, JSON.stringify({ value }));
+    } else {
+      window.localStorage.removeItem(storageKey);
     }
+    window.dispatchEvent(new Event("storage"));
+  } catch (e) {
+    console.error(e);
   }
 };
 
-export const removeFromStorage = (keys: StorageKeys[]): void => keys.forEach(k => window.localStorage.removeItem(k));
+export const getFromStorage = (key: StorageKeys, withPrefix = true): undefined | StorageValue => {
+  const storageKey = withPrefix ? toPrefixedKey(key) : key;
+  const valueObj = window.localStorage.getItem(storageKey);
+
+  if (valueObj === null) return undefined;
+
+  try {
+    return JSON.parse(valueObj)?.value; // see comment in "saveToStorage"
+  } catch (e) {
+    return valueObj; // fallback for corrupted json
+  }
+};
+
+export const removeFromStorage = (keys: StorageKeys[], withPrefix = true): void => {
+  const storageKeys = withPrefix ? keys.map(toPrefixedKey) : keys;
+  storageKeys.forEach(k => window.localStorage.removeItem(k));
+};
+
+/**
+ * Migrates legacy (unprefixed) localStorage keys to the new prefixed format (`${STORAGE_PREFIX}*`).
+ * Keeps the prefixed value if it already exists, then removes the legacy key.
+ */
+
+type StorageMigrationResult = {
+  migrated: StorageKeys[];
+  removed: StorageKeys[];
+  skipped: StorageKeys[];
+};
+
+export const migrateStorageToPrefixedKeys = (): StorageMigrationResult => {
+  const res: StorageMigrationResult = {
+    migrated: [],
+    removed: [],
+    skipped: [],
+  };
+
+  for (const key of ALL_STORAGE_KEYS) {
+    const legacyKey = key as StorageKeys; // unprefixed
+    const legacyValue = getFromStorage(legacyKey, false);
+    const prefixedValue = getFromStorage(legacyKey, true);
+
+    if (legacyValue === undefined) {
+      res.skipped.push(legacyKey);
+      continue;
+    }
+
+    // prefixed exists -> keep it, just remove legacy
+    if (prefixedValue !== undefined) {
+      removeFromStorage([legacyKey], false);
+      res.removed.push(legacyKey);
+      continue;
+    }
+
+    // prefixed missing -> copy legacy -> prefixed, then remove legacy
+    saveToStorage(legacyKey, legacyValue, true);
+    removeFromStorage([legacyKey], false);
+    res.migrated.push(legacyKey);
+  }
+
+  return res;
+};

app/vmui/packages/vmui/src/utils/tenants.test.ts

@@ -0,0 +1,89 @@
+import { describe, it, expect } from "vitest";
+
+import {
+  replaceTenantId,
+  getTenantIdFromUrl,
+  getUrlWithoutTenant,
+} from "./tenants";
+
+describe("tenant url helpers", () => {
+  describe("getTenantIdFromUrl", () => {
+    it("returns accountID", () => {
+      expect(getTenantIdFromUrl("http://vmselect:8481/select/0/vmui/")).toBe("0");
+    });
+
+    it("returns accountID:projectID", () => {
+      expect(getTenantIdFromUrl("http://vmselect:8481/select/12:7/vmui/")).toBe("12:7");
+    });
+
+    it("returns empty string if tenant is missing", () => {
+      expect(getTenantIdFromUrl("http://vmselect:8481/select/vmui/")).toBe("");
+    });
+
+    it("returns empty string for unrelated paths", () => {
+      expect(getTenantIdFromUrl("http://vmselect:8481/foo/bar")).toBe("");
+    });
+
+    it("returns accountID when url ends right after tenant", () => {
+      expect(getTenantIdFromUrl("http://vmselect:8481/select/0")).toBe("0");
+    });
+  });
+
+  describe("replaceTenantId", () => {
+    it("replaces accountID with another accountID", () => {
+      expect(
+        replaceTenantId("http://vmselect:8481/select/0/vmui/", "2")
+      ).toBe("http://vmselect:8481/select/2/vmui/");
+    });
+
+    it("replaces accountID with accountID:projectID", () => {
+      expect(
+        replaceTenantId("http://vmselect:8481/select/0/prometheus/", "1:9")
+      ).toBe("http://vmselect:8481/select/1:9/prometheus/");
+    });
+
+    it("keeps the rest of the path intact", () => {
+      expect(
+        replaceTenantId("http://vmselect:8481/select/3:4/prometheus/api/v1/query", "7")
+      ).toBe("http://vmselect:8481/select/7/prometheus/api/v1/query");
+    });
+
+    it("does not change url if it doesn't match expected pattern", () => {
+      expect(
+        replaceTenantId("http://vmselect:8481/foo/bar", "2")
+      ).toBe("http://vmselect:8481/foo/bar");
+    });
+  });
+
+  describe("getUrlWithoutTenant", () => {
+    it("removes /select/<tenant>/... and returns base url", () => {
+      expect(
+        getUrlWithoutTenant("http://vmselect:8481/select/0/vmui/")
+      ).toBe("http://vmselect:8481");
+    });
+
+    it("removes /select/<tenant>/... for accountID:projectID and returns base url", () => {
+      expect(
+        getUrlWithoutTenant("http://vmselect:8481/select/5:6/prometheus/")
+      ).toBe("http://vmselect:8481");
+    });
+
+    it("works with deep paths and returns base url", () => {
+      expect(
+        getUrlWithoutTenant("http://vmselect:8481/select/1:2/prometheus/api/v1/query")
+      ).toBe("http://vmselect:8481");
+    });
+
+    it("does not change url if it doesn't match expected pattern", () => {
+      expect(
+        getUrlWithoutTenant("http://vmselect:8481/foo/bar")
+      ).toBe("http://vmselect:8481/foo/bar");
+    });
+
+    it("removes url ending right after tenant", () => {
+      expect(
+        getUrlWithoutTenant("http://vmselect:8481/select/0")
+      ).toBe("http://vmselect:8481");
+    });
+  });
+});

app/vmui/packages/vmui/src/utils/tenants.ts

@@ -1,13 +1,21 @@
-const regexp = /(\/select\/)([^/])(\/)(.+)/;
+const TENANT_REGEXP = /(\/select\/)(\d+(?::\d+)?)(\/.*)?$/;
 
 export const replaceTenantId = (serverUrl: string, tenantId: string) => {
-  return serverUrl.replace(regexp, `$1${tenantId}/$4`);
+  return serverUrl.replace(TENANT_REGEXP, `$1${tenantId}$3`);
 };
 
 export const getTenantIdFromUrl = (url: string): string => {
-  return url.match(regexp)?.[2] || "";
+  return url.match(TENANT_REGEXP)?.[2] ?? "";
 };
 
 export const getUrlWithoutTenant = (url: string): string => {
-  return url.replace(regexp, "");
+  return url.replace(TENANT_REGEXP, "");
+};
+
+export const updateBrowserUrlTenant = (tenantId: string) => {
+  const base = `${window.location.origin}${window.location.pathname}${window.location.search}`;
+  const nextBase = replaceTenantId(base, tenantId);
+
+  const nextUrl = `${nextBase}${window.location.hash}`;
+  window.history.replaceState(null, "", nextUrl);
 };

app/vmui/packages/vmui/src/utils/time.ts

@@ -31,7 +31,7 @@ const shortDurations = supportedDurations.map(d => d.short);
 
 export const sameTs = (a: number, b: number) => {
   return roundToThousandths(a) === roundToThousandths(b);
-}
+};
 
 export const humanizeSeconds = (num: number): string => {
   return getDurationFromMilliseconds(dayjs.duration(num, "seconds").asMilliseconds());

app/vmui/packages/vmui/src/utils/uplot/axes.ts

@@ -44,7 +44,7 @@ export const getTimeSeries = (
   const tStart = roundToThousandths(period.start);
   const tEnd = roundToThousandths(period.end);
   const baseStep = getSecondsFromDuration(stepDuration) || 0.001;
-  const step = Math.max(0.001, roundToThousandths(baseStep))
+  const step = Math.max(0.001, roundToThousandths(baseStep));
 
   const anchor = roundToThousandths(tsAnchor ?? tStart);
 

app/vmui/packages/vmui/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "target": "ESNext",
-    "types": ["vite/client", "vitest/globals"],
+    "types": ["vite/client", "vitest/globals", "node"],
     "lib": [
       "dom",
       "dom.iterable",

apptest/tests/ingestprotocols_test.go

@@ -265,6 +265,36 @@ func TestSingleIngestionProtocols(t *testing.T) {
 			{Timestamp: 1707123456800, Value: 20}, // 2024-02-05T08:57:36.700Z
 		},
 	})
+
+	// zabbixconnector format
+	sut.ZabbixConnectorHistory(t,
+		[]string{
+			`{"host":{"host":"h1","name":"n1"},"item_tags":[], "itemid":1,"name":"zabbixconnector_series","clock":1707123456,"ns":700000000,"value":10,"type":0}`,
+			`{"host":{"host":"h2","name":"n2"},"item_tags":[{"tag":"foo2","value":"value1"}], "itemid":1,"name":"zabbixconnector_series2","clock":1707123456,"ns":800000000,"value":20,"type":0}`,
+		},
+		apptest.QueryOpts{})
+	sut.ForceFlush(t)
+	f(sut, &opts{
+		query: `{__name__=~"zabbixconnector.+"}`,
+		wantMetrics: []map[string]string{
+			{
+				"__name__": "zabbixconnector_series",
+				"host":     "h1",
+				"hostname": "n1",
+			},
+			{
+				"__name__": "zabbixconnector_series2",
+				"host":     "h2",
+				"hostname": "n2",
+				"tag_foo2": "value1",
+			},
+		},
+		wantSamples: []*apptest.Sample{
+			{Timestamp: 1707123456700, Value: 10}, // 2024-02-05T08:57:36.700Z
+			{Timestamp: 1707123456800, Value: 20}, // 2024-02-05T08:57:36.700Z
+		},
+	})
+
 }
 
 func TestClusterIngestionProtocols(t *testing.T) {
@@ -531,4 +561,33 @@ func TestClusterIngestionProtocols(t *testing.T) {
 			{Timestamp: 1707123456800, Value: 20}, // 2024-02-05T08:57:36.700Z
 		},
 	})
+	// zabbixconnector format
+	vminsert.ZabbixConnectorHistory(t,
+		[]string{
+			`{"host":{"host":"h1","name":"n1"},"item_tags":[], "itemid":1,"name":"zabbixconnector_series","clock":1707123456,"ns":700000000,"value":10,"type":0}`,
+			`{"host":{"host":"h2","name":"n2"},"item_tags":[{"tag":"foo2","value":"value1"}], "itemid":1,"name":"zabbixconnector_series2","clock":1707123456,"ns":800000000,"value":20,"type":0}`,
+		},
+		apptest.QueryOpts{})
+	vmstorage.ForceFlush(t)
+	f(&opts{
+		query: `{__name__=~"zabbixconnector.+"}`,
+		wantMetrics: []map[string]string{
+			{
+				"__name__": "zabbixconnector_series",
+				"host":     "h1",
+				"hostname": "n1",
+			},
+			{
+				"__name__": "zabbixconnector_series2",
+				"host":     "h2",
+				"hostname": "n2",
+				"tag_foo2": "value1",
+			},
+		},
+		wantSamples: []*apptest.Sample{
+			{Timestamp: 1707123456700, Value: 10}, // 2024-02-05T08:57:36.700Z
+			{Timestamp: 1707123456800, Value: 20}, // 2024-02-05T08:57:36.700Z
+		},
+	})
+
 }

apptest/tests/legacy_indexdb_test.go

@@ -0,0 +1,898 @@
+package tests
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+var (
+	legacyVmsinglePath  = os.Getenv("VM_LEGACY_VMSINGLE_PATH")
+	legacyVmstoragePath = os.Getenv("VM_LEGACY_VMSTORAGE_PATH")
+)
+
+type testLegacyDeleteSeriesOpts struct {
+	startLegacySUT func() at.PrometheusWriteQuerier
+	startNewSUT    func() at.PrometheusWriteQuerier
+	stopLegacySUT  func()
+	stopNewSUT     func()
+}
+
+func TestLegacySingleDeleteSeries(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testLegacyDeleteSeriesOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle-new", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vmsingle-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vmsingle-new")
+		},
+	}
+
+	testLegacyDeleteSeries(tc, opts)
+}
+
+func TestLegacyClusterDeleteSeries(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testLegacyDeleteSeriesOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-new",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-new",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-legacy")
+			tc.StopApp("vmstorage2-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-new")
+			tc.StopApp("vmstorage2-new")
+		},
+	}
+
+	testLegacyDeleteSeries(tc, opts)
+}
+
+func testLegacyDeleteSeries(tc *at.TestCase, opts testLegacyDeleteSeriesOpts) {
+	t := tc.T()
+
+	type want struct {
+		series       []map[string]string
+		queryResults []*at.QueryResult
+	}
+
+	genData := func(prefix string, start, end, step int64, value float64) (recs []string, w *want) {
+		count := (end - start) / step
+		recs = make([]string, count)
+		w = &want{
+			series:       make([]map[string]string, count),
+			queryResults: make([]*at.QueryResult, count),
+		}
+		for i := range count {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			timestamp := start + int64(i)*step
+
+			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
+			w.series[i] = map[string]string{"__name__": name}
+			w.queryResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, w
+	}
+
+	assertSearchResults := func(app at.PrometheusQuerier, query string, start, end int64, step string, want *want) {
+		t.Helper()
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want.series,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  step,
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     want.queryResults,
+				},
+			},
+			FailNow: true,
+		})
+	}
+
+	// - start legacy vmsingle
+	// - insert data1
+	// - confirm that metric names and samples are searcheable
+	// - stop legacy vmsingle
+	const step = 24 * 3600 * 1000 // 24h
+	start1 := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end1 := time.Date(2000, 1, 10, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data1, want1 := genData("metric", start1, end1, step, 1)
+	legacySUT := opts.startLegacySUT()
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, data1, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertSearchResults(legacySUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
+	opts.stopLegacySUT()
+
+	// - start new vmsingle
+	// - confirm that data1 metric names and samples are searcheable
+	// - delete data1
+	// - confirm that data1 metric names and samples are not searcheable anymore
+	// - insert data2 (same metric names, different dates)
+	// - confirm that metric names become searcheable again
+	// - confirm that data1 samples are not searchable and data2 samples are searcheable
+
+	newSUT := opts.startNewSUT()
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", want1)
+
+	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	wantNoResults := &want{
+		series:       []map[string]string{},
+		queryResults: []*at.QueryResult{},
+	}
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end1, "1d", wantNoResults)
+
+	start2 := time.Date(2000, 1, 11, 0, 0, 0, 0, time.UTC).UnixMilli()
+	end2 := time.Date(2000, 1, 20, 0, 0, 0, 0, time.UTC).UnixMilli()
+	data2, want2 := genData("metric", start2, end2, step, 2)
+	newSUT.PrometheusAPIV1ImportPrometheus(t, data2, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end2, "1d", want2)
+
+	// - restart new vmsingle
+	// - confirm that metric names still searchable, data1 samples are not
+	//   searchable, and data2 samples are searcheable
+
+	opts.stopNewSUT()
+	newSUT = opts.startNewSUT()
+	assertSearchResults(newSUT, `{__name__=~".*"}`, start1, end2, "1d", want2)
+	opts.stopNewSUT()
+}
+
+type testLegacyBackupRestoreOpts struct {
+	startLegacySUT     func() at.PrometheusWriteQuerier
+	startNewSUT        func() at.PrometheusWriteQuerier
+	stopLegacySUT      func()
+	stopNewSUT         func()
+	storageDataPaths   []string
+	snapshotCreateURLs func(at.PrometheusWriteQuerier) []string
+}
+
+func TestLegacySingleBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testLegacyBackupRestoreOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle-new", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vmsingle-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vmsingle-new")
+		},
+		storageDataPaths: []string{
+			storageDataPath,
+		},
+		snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
+			return []string{
+				sut.(*at.Vmsingle).SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testLegacyBackupRestore(tc, opts)
+}
+
+func TestLegacyClusterBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testLegacyBackupRestoreOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-new",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-new",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-legacy")
+			tc.StopApp("vmstorage2-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-new")
+			tc.StopApp("vmstorage2-new")
+		},
+		storageDataPaths: []string{
+			storage1DataPath,
+			storage2DataPath,
+		},
+		snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
+			c := sut.(*at.Vmcluster)
+			return []string{
+				c.Vmstorages[0].SnapshotCreateURL(),
+				c.Vmstorages[1].SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testLegacyBackupRestore(tc, opts)
+}
+
+func testLegacyBackupRestore(tc *at.TestCase, opts testLegacyBackupRestoreOpts) {
+	t := tc.T()
+
+	const msecPerMinute = 60 * 1000
+	// Use the same number of metrics and time range for all the data ingestions
+	// below.
+	const numMetrics = 1000
+	start := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).Add(-numMetrics * time.Minute).UnixMilli()
+	end := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
+	genData := func(prefix string) (recs []string, wantSeries []map[string]string, wantQueryResults []*at.QueryResult) {
+		recs = make([]string, numMetrics)
+		wantSeries = make([]map[string]string, numMetrics)
+		wantQueryResults = make([]*at.QueryResult, numMetrics)
+		for i := range numMetrics {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			value := float64(i)
+			timestamp := start + int64(i)*msecPerMinute
+
+			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
+			wantSeries[i] = map[string]string{"__name__": name}
+			wantQueryResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, wantSeries, wantQueryResults
+	}
+
+	backupBaseDir, err := filepath.Abs(filepath.Join(tc.Dir(), "backups"))
+	if err != nil {
+		t.Fatalf("could not get absolute path for the backup base dir")
+	}
+
+	// assertSeries issues various queries to the app and compares the query
+	// results with the expected ones.
+	assertQueries := func(app at.PrometheusQuerier, query string, wantSeries []map[string]string, wantQueryResults []*at.QueryResult) {
+		t.Helper()
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   wantSeries,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  "60s",
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     wantQueryResults,
+				},
+			},
+			Retries: 300,
+			FailNow: true,
+		})
+	}
+
+	createBackup := func(sut at.PrometheusWriteQuerier, name string) {
+		t.Helper()
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmbackup-%s-%s", name, replica)
+			snapshotCreateURL := opts.snapshotCreateURLs(sut)[i]
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, backupPath)
+		}
+	}
+
+	restoreFromBackup := func(name string) {
+		t.Helper()
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmrestore-%s-%s", name, replica)
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmrestore(instance, backupPath, storageDataPath)
+		}
+	}
+
+	legacy1Data, wantLegacy1Series, wantLegacy1QueryResults := genData("legacy1")
+	legacy2Data, wantLegacy2Series, wantLegacy2QueryResults := genData("legacy2")
+	new1Data, wantNew1Series, wantNew1QueryResults := genData("new1")
+	new2Data, wantNew2Series, wantNew2QueryResults := genData("new2")
+	wantLegacy12Series := slices.Concat(wantLegacy1Series, wantLegacy2Series)
+	wantLegacy12QueryResults := slices.Concat(wantLegacy1QueryResults, wantLegacy2QueryResults)
+	wantLegacy1New1Series := slices.Concat(wantLegacy1Series, wantNew1Series)
+	wantLegacy1New1QueryResults := slices.Concat(wantLegacy1QueryResults, wantNew1QueryResults)
+	wantLegacy1New12Series := slices.Concat(wantLegacy1New1Series, wantNew2Series)
+	wantLegacy1New12QueryResults := slices.Concat(wantLegacy1New1QueryResults, wantNew2QueryResults)
+	var legacySUT, newSUT at.PrometheusWriteQuerier
+
+	// Verify backup/restore with legacy SUT.
+
+	// Start legacy SUT with empty storage data dir.
+	legacySUT = opts.startLegacySUT()
+
+	// Ingest legacy1 records, ensure the queries return legacy1, and create
+	// legacy1 backup.
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy1Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+	createBackup(legacySUT, "legacy1")
+
+	// Ingest legacy2 records, ensure the queries return legacy1+legacy2, and
+	// create legacy1+legacy2 backup.
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy2Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertQueries(legacySUT, `{__name__=~"legacy.*"}`, wantLegacy12Series, wantLegacy12QueryResults)
+	createBackup(legacySUT, "legacy12")
+
+	// Stop legacy SUT and restore legacy1 data.
+	// Start legacy SUT and ensure the queries return legacy1.
+	opts.stopLegacySUT()
+	restoreFromBackup("legacy1")
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+
+	opts.stopLegacySUT()
+
+	// Verify backup/restore with new SUT.
+
+	// Start new SUT (with partition indexDBs) with storage containing legacy1
+	// data and Ensure that queries return legacy1 data.
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+
+	// Ingest new1 records, ensure that queries now return legacy1+new1, and
+	// create the legacy1+new1 backup.
+	newSUT.PrometheusAPIV1ImportPrometheus(t, new1Data, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertQueries(newSUT, `{__name__=~"(legacy|new).*"}`, wantLegacy1New1Series, wantLegacy1New1QueryResults)
+	createBackup(newSUT, "legacy1-new1")
+
+	// Ingest new2 records, ensure that queries now return legacy1+new1+new2,
+	// and create the legacy1+new1+new2 backup.
+	newSUT.PrometheusAPIV1ImportPrometheus(t, new2Data, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertQueries(newSUT, `{__name__=~"(legacy|new1|new2).*"}`, wantLegacy1New12Series, wantLegacy1New12QueryResults)
+	createBackup(newSUT, "legacy1-new12")
+
+	// Stop new SUT and restore legacy1+new1 data.
+	// Start new SUT and ensure queries return legacy1+new1 data.
+	opts.stopNewSUT()
+	restoreFromBackup("legacy1-new1")
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy1New1Series, wantLegacy1New1QueryResults)
+
+	opts.stopNewSUT()
+
+	// Verify backup/restore with legacy SUT again.
+
+	// Start legacy SUT with storage containing legacy1+new1 data.
+	//
+	// Ensure that the /series and /query_range queries return legacy1 data only.
+	// new1 data is not returned because legacy vmsingle does not know about
+	// partition indexDBs.
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1Series, wantLegacy1QueryResults)
+
+	// Stop legacy SUT and restore legacy1+legacy2 data.
+	// Start legacy SUT and ensure that queries now return legacy1+legacy2 data.
+	opts.stopLegacySUT()
+	restoreFromBackup("legacy12")
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy12Series, wantLegacy12QueryResults)
+
+	opts.stopLegacySUT()
+
+	// Verify backup/restore with new vmsingle again.
+
+	// Start new vmsingle with storage containing legacy1+legacy2 data and
+	// ensure that queries return legacy1+legacy2 data.
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy12Series, wantLegacy12QueryResults)
+
+	// Stop new SUT and restore legacy1+new1+new2 data.
+	// Start new SUT and ensure that queries return legacy1+new1+new2 data.
+	opts.stopNewSUT()
+	restoreFromBackup("legacy1-new12")
+	newSUT = opts.startNewSUT()
+	assertQueries(newSUT, `{__name__=~"(legacy|new).*"}`, wantLegacy1New12Series, wantLegacy1New12QueryResults)
+
+	opts.stopNewSUT()
+}
+
+type testLegacyDowngradeOpts struct {
+	startLegacySUT func() at.PrometheusWriteQuerier
+	startNewSUT    func() at.PrometheusWriteQuerier
+	stopLegacySUT  func()
+	stopNewSUT     func()
+}
+
+func TestLegacySingleDowngrade(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testLegacyDowngradeOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingleAt("vmsingle-legacy", legacyVmsinglePath, []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle-new", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.disableCache=true",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vmsingle-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vmsingle-new")
+		},
+	}
+
+	testLegacyDowngrade(tc, opts)
+}
+
+func TestLegacyClusterDowngrade(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testLegacyDowngradeOpts{
+		startLegacySUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-legacy",
+				Vmstorage1Binary:   legacyVmstoragePath,
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-legacy",
+				Vmstorage2Binary:   legacyVmstoragePath,
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		startNewSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1-new",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2-new",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.disableCache=true",
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopLegacySUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-legacy")
+			tc.StopApp("vmstorage2-legacy")
+		},
+		stopNewSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1-new")
+			tc.StopApp("vmstorage2-new")
+		},
+	}
+
+	testLegacyDowngrade(tc, opts)
+}
+
+func testLegacyDowngrade(tc *at.TestCase, opts testLegacyDowngradeOpts) {
+	t := tc.T()
+
+	type want struct {
+		series            []map[string]string
+		labels            []string
+		labelValues       []string
+		queryResults      []*at.QueryResult
+		queryRangeResults []*at.QueryResult
+	}
+
+	uniq := func(s []string) []string {
+		slices.Sort(s)
+		return slices.Compact(s)
+	}
+
+	mergeWant := func(want1, want2 want) want {
+		var result want
+		result.series = slices.Concat(want1.series, want2.series)
+		result.labels = uniq(slices.Concat(want1.labels, want2.labels))
+		result.labelValues = slices.Concat(want1.labelValues, want2.labelValues)
+		result.queryResults = slices.Concat(want1.queryResults, want2.queryResults)
+		result.queryRangeResults = slices.Concat(want1.queryRangeResults, want2.queryRangeResults)
+
+		return result
+	}
+
+	// Use the same number of metrics and time range for all the data batches below.
+	const numMetrics = 1000
+	const labelName = "prefix"
+	start := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
+	end := start
+	genData := func(prefix string) (recs []string, want want) {
+		labelValue := prefix
+		recs = make([]string, numMetrics)
+		want.series = make([]map[string]string, numMetrics)
+		want.labels = []string{"__name__", labelName}
+		want.labelValues = []string{labelValue}
+		want.queryResults = make([]*at.QueryResult, numMetrics)
+		want.queryRangeResults = make([]*at.QueryResult, numMetrics)
+		for i := range numMetrics {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			value := float64(i)
+			timestamp := start
+
+			recs[i] = fmt.Sprintf("%s{%s=\"%s\"} %f %d", name, labelName, labelValue, value, timestamp)
+			want.series[i] = map[string]string{"__name__": name, labelName: labelValue}
+			want.queryResults[i] = &at.QueryResult{
+				Metric: map[string]string{"__name__": name, labelName: labelValue},
+				Sample: &at.Sample{Timestamp: timestamp, Value: value},
+			}
+			want.queryRangeResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name, labelName: labelValue},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, want
+	}
+
+	// assertSeries issues various queries to the app and compares the query
+	// results with the expected ones.
+	assertQueries := func(app at.PrometheusQuerier, query string, want want, wantSeriesCount uint64) {
+		t.Helper()
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want.series,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series/count response",
+			Got: func() any {
+				return app.PrometheusAPIV1SeriesCount(t, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				})
+			},
+			Want: &at.PrometheusAPIV1SeriesCountResponse{
+				Status: "success",
+				Data:   []uint64{wantSeriesCount},
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/labels response",
+			Got: func() any {
+				return app.PrometheusAPIV1Labels(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				})
+			},
+			Want: &at.PrometheusAPIV1LabelsResponse{
+				Status: "success",
+				Data:   want.labels,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/label/../values response",
+			Got: func() any {
+				return app.PrometheusAPIV1LabelValues(t, labelName, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				})
+			},
+			Want: &at.PrometheusAPIV1LabelValuesResponse{
+				Status: "success",
+				Data:   want.labelValues,
+			},
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query response",
+			Got: func() any {
+				return app.PrometheusAPIV1Query(t, query, at.QueryOpts{
+					Time: fmt.Sprintf("%d", start),
+					Step: "10m",
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "vector",
+					Result:     want.queryResults,
+				},
+			},
+			Retries: 300,
+			FailNow: true,
+		})
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  "60s",
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     want.queryRangeResults,
+				},
+			},
+			Retries: 300,
+			FailNow: true,
+		})
+	}
+
+	wantEmpty := want{
+		series:            []map[string]string{},
+		labels:            []string{"__name__"},
+		labelValues:       []string{},
+		queryResults:      []*at.QueryResult{},
+		queryRangeResults: []*at.QueryResult{},
+	}
+
+	legacy1Data, wantLegacy1 := genData("legacy1")
+	legacy2Data, wantLegacy2 := genData("legacy2")
+	new1Data, wantNew1 := genData("new1")
+	wantLegacy1New1 := mergeWant(wantLegacy1, wantNew1)
+	wantLegacy2New1 := mergeWant(wantLegacy2, wantNew1)
+	var legacySUT, newSUT at.PrometheusWriteQuerier
+
+	// Start legacy SUT with empty storage data dir.
+	// Ingest legacy1 records, ensure the queries return legacy1
+	legacySUT = opts.startLegacySUT()
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy1Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1, numMetrics)
+	opts.stopLegacySUT()
+
+	// Start new SUT (with partition indexDBs) with storage containing legacy1
+	// data and ensure that queries return new1 and legacy1 data.
+	newSUT = opts.startNewSUT()
+	newSUT.PrometheusAPIV1ImportPrometheus(t, new1Data, at.QueryOpts{})
+	newSUT.ForceFlush(t)
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy1New1, 2*numMetrics)
+	opts.stopNewSUT()
+
+	// Downgrade to legacy SUT, ensure the queries return only legacy1.
+	// Delete all series, ensure that queries return no series.
+	// Ingest legacy2 records, ensure the queries return only legacy2.
+	legacySUT = opts.startLegacySUT()
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy1, numMetrics)
+	legacySUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantEmpty, numMetrics)
+	legacySUT.PrometheusAPIV1ImportPrometheus(t, legacy2Data, at.QueryOpts{})
+	legacySUT.ForceFlush(t)
+	// series count includes deleted metrics
+	assertQueries(legacySUT, `{__name__=~".*"}`, wantLegacy2, 2*numMetrics)
+	opts.stopLegacySUT()
+
+	// Upgrade to new SUT, ensure the queries return recently ingested legacy2 and new1
+	// since legacy SUT cannot delete them.
+	// Delete all series, ensure that queries return no series.
+	newSUT = opts.startNewSUT()
+	// series count includes deleted metrics
+	assertQueries(newSUT, `{__name__=~".*"}`, wantLegacy2New1, 3*numMetrics)
+	newSUT.APIV1AdminTSDBDeleteSeries(t, `{__name__=~".*"}`, at.QueryOpts{})
+	// series count includes deleted metrics
+	assertQueries(newSUT, `{__name__=~".*"}`, wantEmpty, 3*numMetrics)
+	opts.stopNewSUT()
+}

apptest/tests/metricsmetadata_test.go

@@ -48,9 +48,9 @@ func TestSingleMetricsMetadata(t *testing.T) {
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_6"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 		},
 		Metadata: []prompb.MetricMetadata{
-			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: uint32(prompb.MetricMetadataSUMMARY)},
-			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: uint32(prompb.MetricMetadataSUMMARY)},
-			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: uint32(prompb.MetricMetadataSTATESET)},
+			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 
@@ -160,9 +160,9 @@ func TestClusterMetricsMetadata(t *testing.T) {
 			{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_6"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
 		},
 		Metadata: []prompb.MetricMetadata{
-			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: uint32(prompb.MetricMetadataSUMMARY)},
-			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: uint32(prompb.MetricMetadataSUMMARY)},
-			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: uint32(prompb.MetricMetadataSTATESET)},
+			{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
+			{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
 		},
 	}
 

apptest/tests/special_query_regression_test.go

@@ -53,6 +53,7 @@ func testSpecialQueryRegression(tc *apptest.TestCase, sut apptest.PrometheusWrit
 	testMatchSeries(tc, sut)
 	testNegativeIncrease(tc, sut)
 	testInstantQueryWithOffsetUsingCache(tc, sut)
+	testQueryRangeEndAtFirstMillisecondOfDate(tc, sut)
 
 	// graphite
 	testComparisonNotInfNotNan(tc, sut)
@@ -76,10 +77,12 @@ func testCaseSensitiveRegex(tc *apptest.TestCase, sut apptest.PrometheusWriteQue
 	tc.Assert(&apptest.AssertOptions{
 		Msg: "unexpected /api/v1/export response",
 		Got: func() any {
-			return sut.PrometheusAPIV1Export(t, `{label=~'(?i)sensitiveregex'}`, apptest.QueryOpts{
+			resp := sut.PrometheusAPIV1Export(t, `{label=~'(?i)sensitiveregex'}`, apptest.QueryOpts{
 				Start: "2024-02-05T08:50:00.700Z",
 				End:   "2024-02-05T09:00:00.700Z",
 			})
+			resp.Sort()
+			return resp
 		},
 		Want: &apptest.PrometheusAPIV1QueryResponse{
 			Status: "success",
@@ -332,6 +335,48 @@ func testInstantQueryWithOffsetUsingCache(tc *apptest.TestCase, sut apptest.Prom
 	})
 }
 
+func testQueryRangeEndAtFirstMillisecondOfDate(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier) {
+	t := tc.T()
+
+	// unexpected /api/v1/query_range response
+	// when the sample is at the last millisecond of a day, e.g. `2025-12-12 00:00:00`
+	// query_range with `End` at the last millisecond of that day may cause the time point to be missed.
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9804
+
+	// `End` should be inclusive according to https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
+
+	sut.PrometheusAPIV1ImportPrometheus(t, []string{
+		`foo_bar 7 1765497600000`, // 2025-12-12 00:00:00
+	}, apptest.QueryOpts{})
+	sut.ForceFlush(t)
+
+	tc.Assert(&apptest.AssertOptions{
+		Msg:        "unexpected /api/v1/query response",
+		DoNotRetry: true,
+		Got: func() any {
+			return sut.PrometheusAPIV1QueryRange(t, `foo_bar`, apptest.QueryOpts{
+				Start: "2025-12-11T20:00:00.000Z",
+				End:   "2025-12-12T00:00:00.000Z",
+				Step:  "1h",
+			})
+		},
+		Want: &apptest.PrometheusAPIV1QueryResponse{
+			Status: "success",
+			Data: &apptest.QueryData{
+				ResultType: "matrix",
+				Result: []*apptest.QueryResult{
+					{
+						Metric: map[string]string{"__name__": "foo_bar"},
+						Samples: []*apptest.Sample{
+							{Timestamp: 1765497600000, Value: 7},
+						},
+					},
+				},
+			},
+		},
+	})
+}
+
 func testComparisonNotInfNotNan(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier) {
 	t := tc.T()
 

apptest/vminsert.go

@@ -255,6 +255,28 @@ func (app *Vminsert) PrometheusAPIV1ImportPrometheus(t *testing.T, records []str
 	})
 }
 
+// ZabbixConnectorHistory is a test helper function that inserts a
+// collection of records in zabbixconnector  format by sending a HTTP
+// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
+func (app *Vminsert) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/insert/%s/zabbixconnector/api/v1/history", app.httpListenAddr, opts.getTenant())
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	app.sendBlocking(t, len(records), func() {
+		_, statusCode := app.cli.Post(t, url, "application/json", data)
+		if statusCode != http.StatusOK {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+		}
+	})
+
+}
+
 // String returns the string representation of the vminsert app state.
 func (app *Vminsert) String() string {
 	return fmt.Sprintf("{app: %s httpListenAddr: %q}", app.app, app.httpListenAddr)

apptest/vmsingle.go

@@ -597,8 +597,27 @@ func (app *Vmsingle) APIV1StatusTSDB(t *testing.T, matchQuery string, date strin
 	return status
 }
 
-// HTTPAddr returns the address at which the vmstorage process is listening
-// for http connections.
+// ZabbixConnectorHistory is a test helper function that inserts a
+// collection of records in zabbixconnector  format by sending a HTTP
+// POST request to /zabbixconnector/api/v1/history vmsingle endpoint.
+func (app *Vmsingle) ZabbixConnectorHistory(t *testing.T, records []string, opts QueryOpts) {
+	t.Helper()
+
+	url := fmt.Sprintf("http://%s/zabbixconnector/api/v1/history", app.httpListenAddr)
+	uv := opts.asURLValues()
+	uvs := uv.Encode()
+	if len(uvs) > 0 {
+		url += "?" + uvs
+	}
+	data := []byte(strings.Join(records, "\n"))
+	_, statusCode := app.cli.Post(t, url, "application/json", data)
+	if statusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
+	}
+}
+
+// HTTPAddr returns the address at which the vminsert process is
+// listening for incoming HTTP requests.
 func (app *Vmsingle) HTTPAddr() string {
 	return app.httpListenAddr
 }

dashboards/alert-statistics.json

@@ -65,7 +65,36 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": null,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/victoriametrics/vmalert/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    }
+  ],
   "panels": [
     {
       "datasource": {

dashboards/backupmanager.json

@@ -66,7 +66,36 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": null,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    }
+  ],
   "liveNow": false,
   "panels": [
     {
@@ -423,7 +452,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(increase(vm_backup_errors_total{job=~\"$job\", instance=~\"$instance\"}[1h]))",
+          "expr": "sum(increase(vm_backup_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__range]))",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -576,7 +605,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(increase(vm_retention_errors_total{job=~\"$job\", instance=~\"$instance\"}[1h]))",
+          "expr": "sum(increase(vm_retention_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__range]))",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"

dashboards/clusterbytenant.json

@@ -26,7 +26,36 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
   "id": 3,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/victoriametrics/pertenantstatistic/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    }
+  ],
   "panels": [
     {
       "collapsed": false,

dashboards/operator.json

@@ -26,7 +26,44 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": 0,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/operator/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Releases",
+      "type": "link",
+      "url": "https://github.com/VictoriaMetrics/operator/releases"
+    }
+  ],
   "panels": [
     {
       "collapsed": false,

dashboards/query-stats.json

@@ -19,7 +19,36 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": 861,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/victoriametrics/query-stats/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    }
+  ],
   "panels": [
     {
       "collapsed": false,

dashboards/vm/backupmanager.json

@@ -67,7 +67,36 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": null,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    }
+  ],
   "liveNow": false,
   "panels": [
     {
@@ -424,7 +453,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(increase(vm_backup_errors_total{job=~\"$job\", instance=~\"$instance\"}[1h]))",
+          "expr": "sum(increase(vm_backup_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__range]))",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -577,7 +606,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(increase(vm_retention_errors_total{job=~\"$job\", instance=~\"$instance\"}[1h]))",
+          "expr": "sum(increase(vm_retention_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__range]))",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"

dashboards/vm/clusterbytenant.json

@@ -27,7 +27,36 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
   "id": 3,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/victoriametrics/pertenantstatistic/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    }
+  ],
   "panels": [
     {
       "collapsed": false,

dashboards/vm/operator.json

@@ -27,7 +27,44 @@
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": 0,
-  "links": [],
+  "links": [
+    {
+      "icon": "doc",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Docs",
+      "type": "link",
+      "url": "https://docs.victoriametrics.com/operator/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
+    },
+    {
+      "asDropdown": false,
+      "icon": "bolt",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [],
+      "targetBlank": true,
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Releases",
+      "type": "link",
+      "url": "https://github.com/VictoriaMetrics/operator/releases"
+    }
+  ],
   "panels": [
     {
       "collapsed": false,

dashboards/vm/vmalert.json

@@ -54,40 +54,52 @@
   "id": 3,
   "links": [
     {
-      "asDropdown": false,
-      "icon": "external link",
-      "includeVars": false,
-      "keepTime": false,
+      "icon": "doc",
       "tags": [],
       "targetBlank": true,
-      "title": "vmalert docs",
-      "tooltip": "",
+      "title": "Docs",
       "type": "link",
       "url": "https://docs.victoriametrics.com/victoriametrics/vmalert/"
     },
     {
       "asDropdown": false,
-      "icon": "external link",
+      "icon": "question",
       "includeVars": false,
       "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "Found a bug?",
+      "title": "Troubleshooting",
       "tooltip": "",
       "type": "link",
-      "url": " https://github.com/VictoriaMetrics/VictoriaMetrics/issues"
+      "url": "https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
     },
     {
       "asDropdown": false,
-      "icon": "external link",
+      "icon": "bolt",
       "includeVars": false,
       "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "New releases",
+      "title": "Enterprise support",
       "tooltip": "",
       "type": "link",
-      "url": " https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Releases",
+      "type": "link",
+      "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
     }
   ],
   "panels": [

dashboards/vm/vmauth.json

@@ -63,38 +63,38 @@
   "id": null,
   "links": [
     {
-      "asDropdown": false,
       "icon": "doc",
-      "includeVars": false,
-      "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "vmauth docs",
-      "tooltip": "vmauth docs",
+      "title": "Docs",
       "type": "link",
       "url": "https://docs.victoriametrics.com/victoriametrics/vmauth/"
     },
     {
-      "asDropdown": false,
-      "icon": "question",
-      "includeVars": false,
-      "keepTime": false,
+      "icon": "info",
       "tags": [],
       "targetBlank": true,
-      "title": "Found a bug?",
-      "tooltip": "Found a bug?",
+      "title": "Community support",
       "type": "link",
-      "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues"
+      "url": "https://victoriametrics.com/support/community-support/"
     },
     {
       "asDropdown": false,
-      "icon": "info",
+      "icon": "bolt",
       "includeVars": false,
       "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "New releases",
-      "tooltip": "New releases",
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Releases",
       "type": "link",
       "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
     }
@@ -604,224 +604,13 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 9
+        "y": 8
       },
       "id": 13,
       "panels": [],
       "title": "Overview",
       "type": "row"
     },
-    {
-      "datasource": {
-        "type": "victoriametrics-metrics-datasource",
-        "uid": "$ds"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 10
-      },
-      "id": 11,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "victoriametrics-metrics-datasource",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username)",
-          "hide": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "victoriametrics-metrics-datasource",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
-          "hide": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Requests rate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "victoriametrics-metrics-datasource",
-        "uid": "$ds"
-      },
-      "description": "Shows percent utilization of  per concurrent requests capacity.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "bars",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "dashed"
-            }
-          },
-          "mappings": [],
-          "max": 1,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 0.9
-              }
-            ]
-          },
-          "unit": "percentunit"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 10
-      },
-      "id": 14,
-      "options": {
-        "legend": {
-          "calcs": [
-            "max",
-            "mean"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true,
-          "sortBy": "Mean",
-          "sortDesc": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "victoriametrics-metrics-datasource",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "expr": "max(\nmax_over_time(vmauth_user_concurrent_requests_current{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])\n/ \nvmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}\n) by(username) > 0\n",
-          "hide": false,
-          "interval": "5m",
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "User concurrent requests usage",
-      "type": "timeseries"
-    },
     {
       "datasource": {
         "type": "victoriametrics-metrics-datasource",
@@ -840,6 +629,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -856,6 +646,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -871,7 +662,7 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
@@ -886,7 +677,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 19
+        "y": 9
       },
       "id": 16,
       "options": {
@@ -897,10 +688,12 @@
           "showLegend": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
+      "pluginVersion": "12.3.0",
       "targets": [
         {
           "datasource": {
@@ -936,6 +729,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "bars",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -952,6 +746,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -967,7 +762,7 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
@@ -982,7 +777,7 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 19
+        "y": 9
       },
       "id": 10,
       "options": {
@@ -993,10 +788,12 @@
           "showLegend": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
+      "pluginVersion": "12.3.0",
       "targets": [
         {
           "datasource": {
@@ -1043,7 +840,6 @@
         "type": "victoriametrics-metrics-datasource",
         "uid": "$ds"
       },
-      "description": "Shows duration in seconds of user requests by quantile.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1056,6 +852,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -1072,6 +869,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1087,7 +885,338 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username)",
+          "hide": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+          "hide": false,
+          "legendFormat": "unauthorized_user",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "User requests rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "victoriametrics-metrics-datasource",
+        "uid": "$ds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 37,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_user_request_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
+          "hide": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_unauthorized_user_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
+          "hide": false,
+          "legendFormat": "unauthorized_user",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "User requests error rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "victoriametrics-metrics-datasource",
+        "uid": "$ds"
+      },
+      "description": "Shows percent utilization of  per concurrent requests capacity.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "bars",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "dashed"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 0.9
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 27
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "max",
+            "mean"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "sortBy": "Mean",
+          "sortDesc": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "max(\nmax_over_time(vmauth_user_concurrent_requests_current{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])\n/ \nvmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}\n) by(username) > 0\n",
+          "hide": false,
+          "interval": "5m",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "User concurrent requests usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "victoriametrics-metrics-datasource",
+        "uid": "$ds"
+      },
+      "description": "Shows duration in seconds of user requests by quantile.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1102,7 +1231,7 @@
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 27
       },
       "id": 19,
@@ -1119,10 +1248,12 @@
           "sortDesc": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
+      "pluginVersion": "12.3.0",
       "targets": [
         {
           "datasource": {

dashboards/vmalert.json

@@ -53,40 +53,52 @@
   "id": 3,
   "links": [
     {
-      "asDropdown": false,
-      "icon": "external link",
-      "includeVars": false,
-      "keepTime": false,
+      "icon": "doc",
       "tags": [],
       "targetBlank": true,
-      "title": "vmalert docs",
-      "tooltip": "",
+      "title": "Docs",
       "type": "link",
       "url": "https://docs.victoriametrics.com/victoriametrics/vmalert/"
     },
     {
       "asDropdown": false,
-      "icon": "external link",
+      "icon": "question",
       "includeVars": false,
       "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "Found a bug?",
+      "title": "Troubleshooting",
       "tooltip": "",
       "type": "link",
-      "url": " https://github.com/VictoriaMetrics/VictoriaMetrics/issues"
+      "url": "https://docs.victoriametrics.com/victoriametrics/vmalert/#troubleshooting"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Community support",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/community-support/"
     },
     {
       "asDropdown": false,
-      "icon": "external link",
+      "icon": "bolt",
       "includeVars": false,
       "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "New releases",
+      "title": "Enterprise support",
       "tooltip": "",
       "type": "link",
-      "url": " https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Releases",
+      "type": "link",
+      "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
     }
   ],
   "panels": [

dashboards/vmauth.json

@@ -62,38 +62,38 @@
   "id": null,
   "links": [
     {
-      "asDropdown": false,
       "icon": "doc",
-      "includeVars": false,
-      "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "vmauth docs",
-      "tooltip": "vmauth docs",
+      "title": "Docs",
       "type": "link",
       "url": "https://docs.victoriametrics.com/victoriametrics/vmauth/"
     },
     {
-      "asDropdown": false,
-      "icon": "question",
-      "includeVars": false,
-      "keepTime": false,
+      "icon": "info",
       "tags": [],
       "targetBlank": true,
-      "title": "Found a bug?",
-      "tooltip": "Found a bug?",
+      "title": "Community support",
       "type": "link",
-      "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues"
+      "url": "https://victoriametrics.com/support/community-support/"
     },
     {
       "asDropdown": false,
-      "icon": "info",
+      "icon": "bolt",
       "includeVars": false,
       "keepTime": false,
       "tags": [],
       "targetBlank": true,
-      "title": "New releases",
-      "tooltip": "New releases",
+      "title": "Enterprise support",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://victoriametrics.com/support/enterprise-support/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Releases",
       "type": "link",
       "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases"
     }
@@ -603,224 +603,13 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 9
+        "y": 8
       },
       "id": 13,
       "panels": [],
       "title": "Overview",
       "type": "row"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 10
-      },
-      "id": 11,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username)",
-          "hide": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
-          "hide": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Requests rate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "$ds"
-      },
-      "description": "Shows percent utilization of  per concurrent requests capacity.",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "bars",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "dashed"
-            }
-          },
-          "mappings": [],
-          "max": 1,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 0.9
-              }
-            ]
-          },
-          "unit": "percentunit"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 10
-      },
-      "id": 14,
-      "options": {
-        "legend": {
-          "calcs": [
-            "max",
-            "mean"
-          ],
-          "displayMode": "table",
-          "placement": "bottom",
-          "showLegend": true,
-          "sortBy": "Mean",
-          "sortDesc": true
-        },
-        "tooltip": {
-          "mode": "multi",
-          "sort": "desc"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "$ds"
-          },
-          "editorMode": "code",
-          "expr": "max(\nmax_over_time(vmauth_user_concurrent_requests_current{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])\n/ \nvmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}\n) by(username) > 0\n",
-          "hide": false,
-          "interval": "5m",
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "User concurrent requests usage",
-      "type": "timeseries"
-    },
     {
       "datasource": {
         "type": "prometheus",
@@ -839,6 +628,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -855,6 +645,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -870,7 +661,7 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
@@ -885,7 +676,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 19
+        "y": 9
       },
       "id": 16,
       "options": {
@@ -896,10 +687,12 @@
           "showLegend": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
+      "pluginVersion": "12.3.0",
       "targets": [
         {
           "datasource": {
@@ -935,6 +728,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "bars",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -951,6 +745,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -966,7 +761,7 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
               },
               {
                 "color": "red",
@@ -981,7 +776,7 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 19
+        "y": 9
       },
       "id": 10,
       "options": {
@@ -992,10 +787,12 @@
           "showLegend": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
+      "pluginVersion": "12.3.0",
       "targets": [
         {
           "datasource": {
@@ -1042,7 +839,6 @@
         "type": "prometheus",
         "uid": "$ds"
       },
-      "description": "Shows duration in seconds of user requests by quantile.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -1055,6 +851,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -1071,6 +868,7 @@
               "type": "linear"
             },
             "showPoints": "auto",
+            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1086,7 +884,338 @@
             "steps": [
               {
                 "color": "green",
-                "value": null
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username)",
+          "hide": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
+          "hide": false,
+          "legendFormat": "unauthorized_user",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "User requests rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 37,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_user_request_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
+          "hide": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(vmauth_unauthorized_user_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
+          "hide": false,
+          "legendFormat": "unauthorized_user",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "User requests error rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds"
+      },
+      "description": "Shows percent utilization of  per concurrent requests capacity.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "bars",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "dashed"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              },
+              {
+                "color": "red",
+                "value": 0.9
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 27
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "max",
+            "mean"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true,
+          "sortBy": "Mean",
+          "sortDesc": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "12.3.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "editorMode": "code",
+          "expr": "max(\nmax_over_time(vmauth_user_concurrent_requests_current{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])\n/ \nvmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}\n) by(username) > 0\n",
+          "hide": false,
+          "interval": "5m",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "User concurrent requests usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "$ds"
+      },
+      "description": "Shows duration in seconds of user requests by quantile.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "showValues": false,
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
               },
               {
                 "color": "red",
@@ -1101,7 +1230,7 @@
       "gridPos": {
         "h": 9,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 27
       },
       "id": 19,
@@ -1118,10 +1247,12 @@
           "sortDesc": true
         },
         "tooltip": {
+          "hideZeros": false,
           "mode": "single",
           "sort": "none"
         }
       },
+      "pluginVersion": "12.3.0",
       "targets": [
         {
           "datasource": {

deployment/docker/Makefile

@@ -3,11 +3,11 @@
 DOCKER_REGISTRIES ?= docker.io quay.io
 DOCKER_NAMESPACE ?= victoriametrics
 
-ROOT_IMAGE ?= alpine:3.22.2
+ROOT_IMAGE ?= alpine:3.23.2
 ROOT_IMAGE_SCRATCH ?= scratch
-CERTS_IMAGE := alpine:3.22.2
+CERTS_IMAGE := alpine:3.23.2
 
-GO_BUILDER_IMAGE := golang:1.25.4
+GO_BUILDER_IMAGE := golang:1.25.5
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.131.0
+    image: victoriametrics/vmagent:v1.133.0
     depends_on:
       - "vmauth"
     ports:
@@ -37,14 +37,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.131.0-cluster
+    image: victoriametrics/vmstorage:v1.133.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.131.0-cluster
+    image: victoriametrics/vmstorage:v1.133.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -54,7 +54,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.131.0-cluster
+    image: victoriametrics/vminsert:v1.133.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -63,7 +63,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.131.0-cluster
+    image: victoriametrics/vminsert:v1.133.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -75,7 +75,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.131.0-cluster
+    image: victoriametrics/vmselect:v1.133.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -85,7 +85,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.131.0-cluster
+    image: victoriametrics/vmselect:v1.133.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -100,7 +100,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.131.0
+    image: victoriametrics/vmauth:v1.133.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -114,7 +114,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.131.0
+    image: victoriametrics/vmalert:v1.133.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.131.0
+    image: victoriametrics/vmagent:v1.133.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.131.0
+    image: victoriametrics/victoria-metrics:v1.133.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -54,7 +54,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.131.0
+    image: victoriametrics/vmalert:v1.133.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts-vmauth.yml

@@ -11,6 +11,7 @@ groups:
         labels:
           severity: warning
         annotations:
+          dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
           summary: "vmauth ({{ $labels.instance }}) reached concurrent requests limit"
           description: "Possible solutions: increase the limit with flag: -maxConcurrentRequests, 
           deploy additional vmauth replicas, check requests latency at backend service. 
@@ -21,7 +22,27 @@ groups:
         labels:
           severity: warning
         annotations:
+          dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
           summary: "vmauth has reached concurrent requests limit for username {{ $labels.username }}"
           description: "Possible solutions: increase limit with flag: -maxConcurrentPerUserRequests, 
           deploy additional vmauth replicas, check requests latency at backend service."
-
+      - alert: UnauthorizedUserRequestErrors
+        expr: increase(vmauth_unauthorized_user_request_errors_total[5m]) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=37&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for unauthorized user (instance {{ $labels.instance }})"
+          description: "Requests from unauthorized user are receiving errors.
+            Please check the vmauth logs to verify that the configuration is correct and clients are sending valid requests."
+      - alert: UserRequestErrors
+        expr: increase(vmauth_user_request_errors_total[5m]) > 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=37&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for user {{ $labels.username }} (instance {{ $labels.instance }})"
+          description: "Requests from user {{ $labels.username }} are receiving errors.
+            Please check the vmauth logs to verify that the configuration is correct and clients are sending valid requests."

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.131.0
+    image: victoriametrics/vmagent:v1.133.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.131.0
+    image: victoriametrics/victoria-metrics:v1.133.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.131.0
+    image: victoriametrics/vmalert:v1.133.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.28.1
+    image: victoriametrics/vmanomaly:v1.28.4
     depends_on:
       - "victoriametrics"
     ports: