chore: [cluster] add build version information to the home page for consistency with other projects

Add fallback to legacy indexDB for stats search

After introducing the new partition index
(f97f627f79), storage stopped returning
stats for date ranges outside the partition index. This made the
migration backward incompatible, as there was no way to retrieve stats
for dates prior to the migration.

This change adds a fallback to the legacy indexDB search when the status
search on the current partition index returns zero series.

This is an imperfect solution: due to tag filters, the TSDB status
search may legitimately return empty results. However, the additional
overhead is small and acceptable.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10315

.github/ISSUE_TEMPLATE/question.yml

@@ -5,7 +5,7 @@ body:
   - type: textarea
     id: describe-the-component
     attributes:
-      label: Is your question request related to a specific component?
+      label: Is your question related to a specific component?
       placeholder: |
         VictoriaMetrics, vmagent, vmalert, vmui, etc...
     validations:

.github/scripts/lint-changelog-tip.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env sh
+
+set -e
+
+CHANGELOG_FILE="docs/victoriametrics/changelog/CHANGELOG.md"
+
+GITHUB_BASE_REF=${GITHUB_BASE_REF:-"master"}
+GIT_REMOTE=${GIT_REMOTE:-"origin"}
+
+git diff "${GIT_REMOTE}/${GITHUB_BASE_REF}"...HEAD -- $CHANGELOG_FILE > diff.txt
+if ! grep -q "^+" diff.txt; then
+  echo "No additions in CHANGELOG.md"
+  exit 0
+fi
+
+ADDED_LINES=$(grep "^+\S" diff.txt | sed 's/^+//')
+
+START_TIP=$(grep -n "^## tip" "$CHANGELOG_FILE" | head -1 | cut -d: -f1)
+if [ -z "$START_TIP" ]; then
+  echo "ERROR: ${CHANGELOG_FILE} does not contain a ## tip section"
+  exit 1
+fi
+
+END_TIP=$(awk "NR>$START_TIP && /^## / {print NR; exit}" "${CHANGELOG_FILE}")
+if [ -z "$END_TIP" ]; then
+  END_TIP=$(wc -l < "$CHANGELOG_FILE")
+fi
+
+BAD=0
+while IFS= read -r line; do
+  # Grep exact line inside the file and get line numbers
+  MATCHES=$(grep -n -F "$line" "$CHANGELOG_FILE" | cut -d: -f1)
+  for m in $MATCHES; do
+    if [ "$m" -lt "$START_TIP" ] || [ "$m" -gt "$END_TIP" ]; then
+      echo "'$line' on line ${m} is outside ## tip section (lines ${START_TIP}-${END_TIP})"
+      BAD=1
+    fi
+  done
+done << EOF
+$ADDED_LINES
+EOF
+
+if [ "$BAD" -ne 0 ]; then
+  echo "CHANGELOG modifications must be placed inside the ## tip section."
+  exit 1
+fi
+
+echo "CHANGELOG modifications are valid."

.github/workflows/build.yml

@@ -55,7 +55,7 @@ jobs:
             arch: amd64
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go

.github/workflows/changelog-linter.yml

@@ -0,0 +1,19 @@
+name: 'changelog-linter'
+
+on:
+  pull_request:
+    paths:
+      - "docs/victoriametrics/changelog/CHANGELOG.md"
+
+jobs:
+  tip-lint:
+    runs-on: 'ubuntu-latest'
+    steps:
+      - uses: 'actions/checkout@v6'
+        with:
+          # needed for proper diff
+          fetch-depth: 0
+
+      - name: 'Validate that changelog changes are under ## tip'
+        run: |
+          GITHUB_BASE_REF=${{ github.base_ref }} ./.github/scripts/lint-changelog-tip.sh

.github/workflows/check-commit-signed.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0  # we need full history for commit verification
 

.github/workflows/codeql-analysis-go.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Set up Go
         id: go

.github/workflows/docs.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           path: __vm
 
       - name: Checkout private code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/test.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
@@ -71,7 +71,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go
@@ -97,7 +97,7 @@ jobs:
 
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Go
         id: go

.github/workflows/vmui.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup Node
         uses: actions/setup-node@v6

Makefile

@@ -17,7 +17,7 @@ EXTRA_GO_BUILD_TAGS ?=
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.4.0
+GOLANGCI_LINT_VERSION := 2.7.2
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -243,7 +243,7 @@ release-vmcluster-windows-goarch: \
 		vmstorage-windows-$(GOARCH)-prod.exe
 
 pprof-cpu:
-	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics@ $(PPROF_FILE)
+	go tool pprof -trim_path=github.com/VictoriaMetrics/VictoriaMetrics $(PPROF_FILE)
 
 fmt:
 	gofmt -l -w -s ./lib
@@ -279,7 +279,23 @@ integration-test:
 
 apptest:
 	$(MAKE) all vmctl vmbackup vmrestore
-	go test ./apptest/... -skip="^TestSingle.*"
+	go test ./apptest/... -skip="^Test(Single|Legacy).*"
+
+integration-test-legacy: all vmbackup vmrestore
+	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
+	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
+	VERSION=v1.132.0; \
+	VMSINGLE=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}.tar.gz; \
+	VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
+	URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
+	DIR=/tmp/$${VERSION}; \
+	test -d $${DIR} || (mkdir $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMSINGLE} && tar xzf /tmp/$${VMSINGLE} -C $${DIR} && \
+		curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
+	); \
+	VM_LEGACY_VMSINGLE_PATH=$${DIR}/victoria-metrics-prod \
+	VM_LEGACY_VMSTORAGE_PATH=$${DIR}/vmstorage-prod \
+	go test ./apptest/tests -run="^TestLegacyCluster.*"
 
 benchmark:
 	GOEXPERIMENT=synctest go test -bench=. ./lib/...
@@ -308,7 +324,8 @@ app-local-windows-goarch:
 	CGO_ENABLED=0 GOOS=windows GOARCH=$(GOARCH) go build $(RACE) -ldflags "$(GO_BUILDINFO)" -tags "$(EXTRA_GO_BUILD_TAGS)" -o bin/$(APP_NAME)-windows-$(GOARCH)$(RACE).exe $(PKG_PREFIX)/app/$(APP_NAME)
 
 quicktemplate-gen: install-qtc
-	qtc
+	qtc -dir=lib
+	qtc -dir=app
 
 install-qtc:
 	which qtc || go install github.com/valyala/quicktemplate/qtc@latest

app/vmagent/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -350,6 +351,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "/zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(nil, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -644,6 +656,17 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(at, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -765,6 +788,9 @@ var (
 	opentelemetryPushRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 	opentelemetryPushErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/opentelemetry/v1/metrics", protocol="opentelemetry"}`)
 
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+
 	newrelicWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 	newrelicWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/newrelic/infra/v2/metrics/events/bulk", protocol="newrelic"}`)
 

app/vmagent/newrelic/request_handler.go

@@ -78,7 +78,7 @@ func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompb.Label)
 	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
 		return remotewrite.ErrQueueFullHTTPRetry
 	}
-	rowsInserted.Add(len(rows))
+	rowsInserted.Add(samplesCount)
 	if at != nil {
 		rowsTenantInserted.Get(at).Add(samplesCount)
 	}

app/vmagent/opentelemetry/request_handler.go

@@ -25,7 +25,7 @@ var (
 	rowsPerInsert          = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
 )
 
-// InsertHandler processes metrics from given reader.
+// InsertHandlerForReader processes metrics from given reader.
 func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
 	return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {
 		return insertRows(at, tss, mms, nil)

app/vmagent/remotewrite/client.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -554,9 +553,9 @@ func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.D
 // For more details, see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9417
 func repackBlockFromZstdToSnappy(zstdBlock []byte) ([]byte, error) {
 	plainBlock := make([]byte, 0, len(zstdBlock)*2)
-	plainBlock, err := zstd.Decompress(plainBlock, zstdBlock)
+	plainBlock, err := encoding.DecompressZSTD(plainBlock, zstdBlock)
 	if err != nil {
-		return nil, fmt.Errorf("zstd: decompress: %s", err)
+		return nil, err
 	}
 
 	return snappy.Encode(nil, plainBlock), nil

app/vmagent/remotewrite/relabel.go

@@ -9,14 +9,14 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/VictoriaMetrics/metrics"
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
-	"go.yaml.in/yaml/v3"
-
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -139,6 +139,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		remoteWriteRelabelConfigData.Store(&rawCfg)
 		rcs.global = global
 	}
+
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
@@ -176,19 +177,9 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
+// isSet indicates whether (global or per-URL) command-line flags is set
 func (rcs *relabelConfigs) isSet() bool {
-	if rcs == nil {
-		return false
-	}
-	if rcs.global.Len() > 0 {
-		return true
-	}
-	for _, pc := range rcs.perURL {
-		if pc.Len() > 0 {
-			return true
-		}
-	}
-	return false
+	return *relabelConfigPathGlobal != "" || len(*relabelConfigPaths) > 0
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.

app/vmagent/zabbixconnector/request_handler.go

@@ -0,0 +1,80 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vmagent_rows_inserted_total{type="zabbixconnector"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="zabbixconnector"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vmagent_rows_per_insert{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := common.GetPushCtx()
+	defer common.PutPushCtx(ctx)
+
+	rowsTotal := len(rows)
+	tssDst := ctx.WriteRequest.Timeseries[:0]
+	labels := ctx.Labels[:0]
+	samples := ctx.Samples[:0]
+	for i := range rows {
+		r := &rows[i]
+
+		labelsLen := len(labels)
+		for j := range r.Tags {
+			tag := &r.Tags[j]
+			labels = append(labels, prompb.Label{
+				Name:  bytesutil.ToUnsafeString(tag.Key),
+				Value: bytesutil.ToUnsafeString(tag.Value),
+			})
+		}
+		labels = append(labels, extraLabels...)
+
+		samplesLen := len(samples)
+		samples = append(samples, prompb.Sample{
+			Value:     r.Value,
+			Timestamp: r.Timestamp,
+		})
+
+		tssDst = append(tssDst, prompb.TimeSeries{
+			Labels:  labels[labelsLen:],
+			Samples: samples[samplesLen:],
+		})
+	}
+	ctx.WriteRequest.Timeseries = tssDst
+	ctx.Labels = labels
+	ctx.Samples = samples
+	if !remotewrite.TryPush(at, &ctx.WriteRequest) {
+		return remotewrite.ErrQueueFullHTTPRetry
+	}
+	rowsInserted.Add(rowsTotal)
+	if at != nil {
+		rowsTenantInserted.Get(at).Add(rowsTotal)
+	}
+	rowsPerInsert.Update(float64(rowsTotal))
+	return nil
+}

app/vmalert/notifier/alert.go

@@ -80,14 +80,15 @@ func (as AlertState) String() string {
 
 // AlertTplData is used to execute templating
 type AlertTplData struct {
-	Type     string
-	Labels   map[string]string
-	Value    float64
-	Expr     string
-	AlertID  uint64
-	GroupID  uint64
-	ActiveAt time.Time
-	For      time.Duration
+	Type      string
+	Labels    map[string]string
+	Value     float64
+	Expr      string
+	AlertID   uint64
+	GroupID   uint64
+	ActiveAt  time.Time
+	For       time.Duration
+	IsPartial bool
 }
 
 var tplHeaders = []string{
@@ -101,6 +102,7 @@ var tplHeaders = []string{
 	"{{ $groupID := .GroupID }}",
 	"{{ $activeAt := .ActiveAt }}",
 	"{{ $for := .For }}",
+	"{{ $isPartial := .IsPartial }}",
 }
 
 // ExecTemplate executes the Alert template for given

app/vmalert/notifier/alertmanager.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -86,6 +87,11 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert, alertLabels []
 	err := am.send(ctx, alerts, alertLabels, headers)
 	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
+		// the context can be cancelled on graceful shutdown
+		// or on group update. So no need to handle the error as usual.
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
 		am.metrics.alertsSendErrors.Add(len(alerts))
 		am.lastError = err.Error()
 	} else {

app/vmalert/rule/alerting.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"math"
@@ -345,6 +346,8 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		ls.processed[l.Name] = l.Value
 	}
 
+	// labels only support limited templating variables,
+	// including `labels`, `value` and `expr`, to avoid breaking alert states or causing cardinality issue with results
 	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
@@ -386,11 +389,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, err
 		}
 		alertID := hash(ls.processed)
-		as, err := ar.expandAnnotationTemplates(s, qFn, time.Time{}, ls)
-		if err != nil {
-			return nil, err
-		}
-		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
+		a := ar.newAlert(s, time.Time{}, ls.processed, nil) // initial alert
 
 		prevT := time.Time{}
 		for i := range s.Values {
@@ -406,8 +405,6 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
 				a.ActiveAt = at
-				// re-template the annotations as active timestamp is changed
-				a.Annotations, _ = ar.expandAnnotationTemplates(s, qFn, at, ls)
 				a.Start = time.Time{}
 			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
 				a.State = notifier.StateFiring
@@ -453,7 +450,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 
 	defer func() {
 		ar.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			ar.metrics.errors.Inc()
 		}
 	}()
@@ -462,7 +459,8 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
-	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
+	isPartial := isPartialResponse(res)
+	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartial)
 	qFn := func(query string) ([]datasource.Metric, error) {
 		res, _, err := ar.q.Query(ctx, query, ts)
 		return res.Data, err
@@ -488,7 +486,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				at = a.ActiveAt
 			}
 		}
-		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
+		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls, isPartial)
 		if err != nil {
 			// only set error in current state, but do not break alert processing
 			curState.Err = err
@@ -606,16 +604,17 @@ func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.
 	return ls, nil
 }
 
-func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet) (map[string]string, error) {
+func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet, isPartial bool) (map[string]string, error) {
 	tplData := notifier.AlertTplData{
-		Value:    m.Values[0],
-		Type:     ar.Type.String(),
-		Labels:   ls.origin,
-		Expr:     ar.Expr,
-		AlertID:  hash(ls.processed),
-		GroupID:  ar.GroupID,
-		ActiveAt: activeAt,
-		For:      ar.For,
+		Value:     m.Values[0],
+		Type:      ar.Type.String(),
+		Labels:    ls.origin,
+		Expr:      ar.Expr,
+		AlertID:   hash(ls.processed),
+		GroupID:   ar.GroupID,
+		ActiveAt:  activeAt,
+		For:       ar.For,
+		IsPartial: isPartial,
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {

app/vmalert/rule/alerting_test.go

@@ -664,7 +664,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -684,7 +684,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-firing"},
-			Annotations: map[string]string{"activeAt": "1000"},
+			Annotations: map[string]string{},
 			State:       notifier.StateFiring,
 			ActiveAt:    time.Unix(1, 0),
 			Start:       time.Unix(5, 0),
@@ -705,7 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
-			Annotations: map[string]string{"activeAt": "5000"},
+			Annotations: map[string]string{},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -1120,7 +1120,7 @@ func TestAlertingRuleLimit_Success(t *testing.T) {
 }
 
 func TestAlertingRule_Template(t *testing.T) {
-	f := func(rule *AlertingRule, metrics []datasource.Metric, alertsExpected map[uint64]*notifier.Alert) {
+	f := func(rule *AlertingRule, metrics []datasource.Metric, isResponsePartial bool, alertsExpected map[uint64]*notifier.Alert) {
 		t.Helper()
 
 		fakeGroup := Group{
@@ -1133,6 +1133,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			entries: make([]StateEntry, 10),
 		}
 		fq.Add(metrics...)
+		fq.SetPartialResponse(isResponsePartial)
 
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -1163,7 +1164,7 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 1, "instance", "foo"),
 		metricWithValueAndLabels(t, 1, "instance", "bar"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
 				"summary": `common: Too high connection number for "foo"`,
@@ -1192,14 +1193,14 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
+			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}".{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 			"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
 		metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
-	}, map[uint64]*notifier.Alert{
+	}, false, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
 			Labels: map[string]string{
 				alertNameLabel:       "override label",
@@ -1207,7 +1208,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "foo",
 			},
 			Annotations: map[string]string{
-				"summary":     `first: Too high connection number for "foo"`,
+				"summary":     `first: Too high connection number for "foo".`,
 				"description": `override: It is 2 connections for "foo"`,
 			},
 		},
@@ -1218,7 +1219,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "bar",
 			},
 			Annotations: map[string]string{
-				"summary":     `second: Too high connection number for "bar"`,
+				"summary":     `second: Too high connection number for "bar".`,
 				"description": `override: It is 10 connections for "bar"`,
 			},
 		},
@@ -1231,7 +1232,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
+			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}.{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1239,7 +1240,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			alertNameLabel, "originAlertname",
 			alertGroupNameLabel, "originGroupname",
 			"instance", "foo"),
-	}, map[uint64]*notifier.Alert{
+	}, true, map[uint64]*notifier.Alert{
 		hash(map[string]string{
 			alertNameLabel:        "OriginLabels",
 			"exported_alertname":  "originAlertname",
@@ -1255,7 +1256,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":            "foo",
 			},
 			Annotations: map[string]string{
-				"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
+				"summary": `Alert "originAlertname(originGroupname)" for instance foo. WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.`,
 			},
 		},
 	})
@@ -1385,7 +1386,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"group":         "vmalert",
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
-		"invalid_label": `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1395,7 +1396,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)

app/vmalert/rule/recording.go

@@ -2,6 +2,7 @@ package rule
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -197,7 +198,7 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 
 	defer func() {
 		rr.state.add(curState)
-		if curState.Err != nil {
+		if curState.Err != nil && !errors.Is(curState.Err, context.Canceled) {
 			rr.metrics.errors.Inc()
 		}
 	}()

app/vmauth/auth_config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"errors"
 	"flag"
 	"fmt"
 	"math"
@@ -94,6 +95,8 @@ type UserInfo struct {
 	rt http.RoundTripper
 
 	requests         *metrics.Counter
+	requestErrors    *metrics.Counter
+	backendRequests  *metrics.Counter
 	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
@@ -105,13 +108,29 @@ type HeadersConf struct {
 	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
 }
 
-func (ui *UserInfo) beginConcurrencyLimit() error {
+func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	select {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
 		ui.concurrencyLimitReached.Inc()
-		return fmt.Errorf("cannot handle more than %d concurrent requests from user %s", ui.getMaxConcurrentRequests(), ui.name())
+
+		// The per-user limit for the number of concurrent requests is reached.
+		// Wait until the currently executed requests are finished, so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case ui.concurrencyLimitCh <- struct{}{}:
+			return nil
+		case <-ctx.Done():
+			err := ctx.Err()
+			if errors.Is(err, context.DeadlineExceeded) {
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
+					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
+			}
+
+			return fmt.Errorf("cannot start executing the request because %d concurrent requests from the user %s are executed: %w",
+				ui.getMaxConcurrentRequests(), ui.name(), err)
+		}
 	}
 }
 
@@ -127,6 +146,18 @@ func (ui *UserInfo) getMaxConcurrentRequests() int {
 	return mcr
 }
 
+func (ui *UserInfo) stopHealthChecks() {
+	if ui == nil {
+		return
+	}
+	if ui.URLPrefix == nil {
+		return
+	}
+
+	bus := ui.URLPrefix.bus.Load()
+	bus.stopHealthChecks()
+}
+
 // Header is `Name: Value` http header, which must be added to the proxied request.
 type Header struct {
 	Name  string
@@ -262,7 +293,7 @@ type URLPrefix struct {
 	// the list of backend urls
 	//
 	// the list can be dynamically updated if `discover_backend_ips` option is set.
-	bus atomic.Pointer[[]*backendURL]
+	bus atomic.Pointer[backendURLs]
 
 	// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
 	discoverBackendIPs bool
@@ -286,21 +317,93 @@ func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
 	}
 }
 
+type backendURLs struct {
+	healthChecksContext context.Context
+	healthChecksCancel  func()
+	healthChecksWG      sync.WaitGroup
+
+	bus []*backendURL
+}
+
+func newBackendURLs() *backendURLs {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &backendURLs{
+		healthChecksContext: ctx,
+		healthChecksCancel:  cancel,
+	}
+}
+
+func (bus *backendURLs) add(u *url.URL) {
+	bus.bus = append(bus.bus, &backendURL{
+		url:                u,
+		healthCheckContext: bus.healthChecksContext,
+		healthCheckWG:      &bus.healthChecksWG,
+	})
+}
+
+func (bus *backendURLs) stopHealthChecks() {
+	bus.healthChecksCancel()
+	bus.healthChecksWG.Wait()
+}
+
 type backendURL struct {
-	brokenDeadline     atomic.Uint64
+	broken atomic.Bool
+
+	healthCheckContext context.Context
+	healthCheckWG      *sync.WaitGroup
+
 	concurrentRequests atomic.Int32
 
 	url *url.URL
 }
 
 func (bu *backendURL) isBroken() bool {
-	ct := fasttime.UnixTimestamp()
-	return ct < bu.brokenDeadline.Load()
+	return bu.broken.Load()
 }
 
 func (bu *backendURL) setBroken() {
-	deadline := fasttime.UnixTimestamp() + uint64((*failTimeout).Seconds())
-	bu.brokenDeadline.Store(deadline)
+	if bu.broken.CompareAndSwap(false, true) {
+		bu.healthCheckWG.Add(1)
+		go func() {
+			defer bu.healthCheckWG.Done()
+			bu.runHealthCheck()
+			bu.broken.Store(false)
+		}()
+	}
+}
+
+func (bu *backendURL) runHealthCheck() {
+	port := bu.url.Port()
+	if port == "" {
+		port = "80"
+	}
+	addr := net.JoinHostPort(bu.url.Hostname(), port)
+
+	t := time.NewTicker(*failTimeout)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			// Verify network connectivity via TCP dial before marking backend healthy.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9997
+			ctx, cancel := context.WithTimeout(bu.healthCheckContext, time.Second)
+			c, err := netutil.Dialer.DialContext(ctx, "tcp", addr)
+			cancel()
+			if err != nil {
+				if errors.Is(bu.healthCheckContext.Err(), context.Canceled) {
+					return
+				}
+				logger.Warnf("ignoring the backend at %s for %s because of dial error: %s", addr, *failTimeout, err)
+				continue
+			}
+
+			_ = c.Close()
+			return
+		case <-bu.healthCheckContext.Done():
+			return
+		}
+	}
 }
 
 func (bu *backendURL) get() {
@@ -312,8 +415,8 @@ func (bu *backendURL) put() {
 }
 
 func (up *URLPrefix) getBackendsCount() int {
-	pbus := up.bus.Load()
-	return len(*pbus)
+	bus := up.bus.Load()
+	return len(bus.bus)
 }
 
 // getBackendURL returns the backendURL depending on the load balance policy.
@@ -324,16 +427,15 @@ func (up *URLPrefix) getBackendsCount() int {
 func (up *URLPrefix) getBackendURL() *backendURL {
 	up.discoverBackendAddrsIfNeeded()
 
-	pbus := up.bus.Load()
-	bus := *pbus
-	if len(bus) == 0 {
+	bus := up.bus.Load()
+	if len(bus.bus) == 0 {
 		return nil
 	}
 
 	if up.loadBalancingPolicy == "first_available" {
-		return getFirstAvailableBackendURL(bus)
+		return getFirstAvailableBackendURL(bus.bus)
 	}
-	return getLeastLoadedBackendURL(bus, &up.n)
+	return getLeastLoadedBackendURL(bus.bus, &up.n)
 }
 
 func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
@@ -407,25 +509,24 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	cancel()
 
 	// generate new backendURLs for the resolved IPs
-	var busNew []*backendURL
+	busNew := newBackendURLs()
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
 			buCopy.Host = addr
-			busNew = append(busNew, &backendURL{
-				url: &buCopy,
-			})
+			busNew.add(&buCopy)
 		}
 	}
 
-	pbus := up.bus.Load()
-	if areEqualBackendURLs(*pbus, busNew) {
+	bus := up.bus.Load()
+	if areEqualBackendURLs(bus.bus, busNew.bus) {
 		return
 	}
 
 	// Store new backend urls
-	up.bus.Store(&busNew)
+	up.bus.Store(busNew)
+	bus.stopHealthChecks()
 }
 
 func areEqualBackendURLs(a, b []*backendURL) bool {
@@ -456,20 +557,23 @@ func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
 	for i := 1; i < len(bus); i++ {
 		if !bus[i].isBroken() {
 			bu = bus[i]
-			break
+			bu.get()
+			return bu
 		}
 	}
-	bu.get()
-	return bu
+	return nil
 }
 
-// getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
+// getLeastLoadedBackendURL returns a non-broken backendURL with the lowest number of concurrent requests.
 //
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
 	if len(bus) == 1 {
 		// Fast path - return the only backend url.
 		bu := bus[0]
+		if bu.isBroken() {
+			return nil
+		}
 		bu.get()
 		return bu
 	}
@@ -494,7 +598,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
 	buMinIdx := n % uint32(len(bus))
 	minRequests := bus[buMinIdx].concurrentRequests.Load()
-	for i := uint32(0); i < uint32(len(bus)); i++ {
+	for i := uint32(1); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
 		bu := bus[idx]
 		if bu.isBroken() {
@@ -508,6 +612,9 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		}
 	}
 	buMin := bus[buMinIdx]
+	if buMin.isBroken() {
+		return nil
+	}
 	buMin.get()
 	atomicCounter.CompareAndSwap(n+1, buMinIdx+1)
 	return buMin
@@ -702,7 +809,7 @@ func reloadAuthConfig() (bool, error) {
 
 	ok, err := reloadAuthConfigData(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
 	}
 	if !ok {
 		return false, nil
@@ -732,6 +839,11 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 
 	acPrev := authConfig.Load()
 	if acPrev != nil {
+		acPrev.UnauthorizedUser.stopHealthChecks()
+		for i := range acPrev.Users {
+			acPrev.Users[i].stopHealthChecks()
+		}
+
 		metrics.UnregisterSet(acPrev.ms, true)
 	}
 	metrics.RegisterSet(ac.ms)
@@ -778,6 +890,8 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
 		}
 		ui.requests = ac.ms.NewCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.NewCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.NewSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
@@ -826,6 +940,8 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
 		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_requests_total` + metricLabels)
 		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
 		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
@@ -1060,13 +1176,11 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 	}
 
 	// Initialize up.bus
-	bus := make([]*backendURL, len(up.busOriginal))
-	for i, bu := range up.busOriginal {
-		bus[i] = &backendURL{
-			url: bu,
-		}
+	bus := newBackendURLs()
+	for _, bu := range up.busOriginal {
+		bus.add(bu)
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 
 	return nil
 }

app/vmauth/auth_config_test.go

@@ -753,7 +753,7 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	up.loadBalancingPolicy = "least_loaded"
 
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	fn := func(ns ...int) {
 		t.Helper()
@@ -825,7 +825,7 @@ func TestBrokenBackend(t *testing.T) {
 	})
 	up.loadBalancingPolicy = "least_loaded"
 	pbus := up.bus.Load()
-	bus := *pbus
+	bus := pbus.bus
 
 	// explicitly mark one of the backends as broken
 	bus[1].setBroken()
@@ -848,7 +848,7 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 
 		up.discoverBackendAddrsIfNeeded()
 		pbus := up.bus.Load()
-		bus := *pbus
+		bus := pbus.bus
 
 		if len(bus) != 1 {
 			t.Fatalf("expected url list to be of size 1; got %d instead", len(bus))
@@ -942,16 +942,14 @@ func mustParseURL(u string) *URLPrefix {
 }
 
 func mustParseURLs(us []string) *URLPrefix {
-	bus := make([]*backendURL, len(us))
+	bus := newBackendURLs()
 	urls := make([]*url.URL, len(us))
 	for i, u := range us {
 		pu, err := url.Parse(u)
 		if err != nil {
 			panic(fmt.Errorf("BUG: cannot parse %q: %w", u, err))
 		}
-		bus[i] = &backendURL{
-			url: pu,
-		}
+		bus.add(pu)
 		urls[i] = pu
 	}
 	up := &URLPrefix{}
@@ -960,7 +958,7 @@ func mustParseURLs(us []string) *URLPrefix {
 	} else {
 		up.vOriginal = us
 	}
-	up.bus.Store(&bus)
+	up.bus.Store(bus)
 	up.busOriginal = urls
 	return up
 }

app/vmauth/main.go

@@ -44,12 +44,17 @@ var (
 		"See also -maxConcurrentRequests")
 	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
-	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
+		"'429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
-		"in per-user config")
+		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options "+
+		"and max_concurrent_requests option in per-user config")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration the request waits for execution when the number of concurrently executed "+
+		"requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. "+
+		"This allows graceful handling of short spikes in the number of concurrent requests")
+
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
@@ -151,7 +156,6 @@ func requestHandlerWithInternalRoutes(w http.ResponseWriter, r *http.Request) bo
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
-
 	ats := getAuthTokensFromRequest(r)
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
@@ -208,20 +212,45 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 	ui.requests.Inc()
 
+	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
+	defer cancel()
+
 	// Limit the concurrency of requests to backends
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		if err := ui.beginConcurrencyLimit(); err != nil {
+		if err := ui.beginConcurrencyLimit(ctx); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
 			return
 		}
 	default:
-		concurrentRequestsLimitReached.Inc()
-		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
-		handleConcurrencyLimitError(w, r, err)
-		return
+		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
+		// so the current request could be executed.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
+		select {
+		case concurrencyLimitCh <- struct{}{}:
+			if err := ui.beginConcurrencyLimit(ctx); err != nil {
+				handleConcurrencyLimitError(w, r, err)
+				<-concurrencyLimitCh
+				return
+			}
+		case <-ctx.Done():
+			err := ctx.Err()
+
+			concurrentRequestsLimitReached.Inc()
+
+			if errors.Is(err, context.DeadlineExceeded) {
+				err = fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+					*maxQueueDuration, cap(concurrencyLimitCh))
+				handleConcurrencyLimitError(w, r, err)
+				return
+			}
+
+			err = fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
+			handleConcurrencyLimitError(w, r, err)
+			return
+		}
 	}
 	processRequest(w, r, ui)
 	ui.endConcurrencyLimit()
@@ -285,16 +314,18 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			return
 		}
 		bu.setBroken()
+		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
 		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)
-	ui.backendErrors.Inc()
+	ui.requestErrors.Inc()
 }
 
 func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
 	req.URL = targetURL
@@ -318,15 +349,17 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		err = ctxErr
 	}
 	if err != nil {
-		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-			// Do not retry canceled or timed out requests
+		// Do not retry canceled
+		if errors.Is(err, context.Canceled) {
+			clientCanceledRequests.Inc()
+			return true, false
+		}
+		// Do not retry timed out requests
+		if errors.Is(err, context.DeadlineExceeded) {
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
-			if errors.Is(err, context.DeadlineExceeded) {
-				// Timed out request must be counted as errors, since this usually means that the backend is slow.
-				logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
-				ui.backendErrors.Inc()
-			}
+			// Timed out request must be counted as errors, since this usually means that the backend is slow.
+			logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
 			return false, false
 		}
 		if !rtbOK || !rtb.canRetry() {
@@ -337,6 +370,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
@@ -344,11 +378,11 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			return false, true
 		}
 
-		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
+		// Request body wasn't read yet, this usually means that the backend isn't reachable; retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, req.URL, targetURL, err)
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
@@ -357,12 +391,13 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request on another backend, because the request has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
+			ui.requestErrors.Inc()
 			return true, false
 		}
 		// Retry requests at other backends if it matches retryStatusCodes.
@@ -370,7 +405,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
 			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
 		return false, false
 	}
@@ -381,11 +416,15 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-	if err != nil && !netutil.IsTrivialNetworkError(err) && !errors.Is(err, context.Canceled) {
+	if errors.Is(err, context.Canceled) {
+		clientCanceledRequests.Inc()
+		return true, false
+	} else if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
 
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+		ui.requestErrors.Inc()
 		return true, false
 	}
 	return true, false
@@ -513,6 +552,7 @@ var (
 	configReloadRequests     = metrics.NewCounter(`vmauth_http_requests_total{path="/-/reload"}`)
 	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
+	clientCanceledRequests   = metrics.NewCounter(`vmauth_http_request_errors_total{reason="client_canceled"}`)
 )
 
 func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
@@ -596,6 +636,14 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
+	ctx := r.Context()
+	if errors.Is(ctx.Err(), context.Canceled) {
+		// Do not return any response for the request canceled by the client,
+		// since the connection to the client is already closed.
+		clientCanceledRequests.Inc()
+		return
+	}
+
 	w.Header().Add("Retry-After", "10")
 	err = &httpserver.ErrorWithStatusCode{
 		Err:        err,
@@ -652,6 +700,7 @@ type zeroReader struct{}
 func (r *zeroReader) Read(_ []byte) (int, error) {
 	return 0, io.EOF
 }
+
 func (r *zeroReader) Close() error {
 	return nil
 }

app/vmbackup/main.go

@@ -212,7 +212,7 @@ func newSrcFS() (*fslocal.FS, error) {
 }
 
 func newDstFS(ctx context.Context) (common.RemoteFS, error) {
-	fs, err := actions.NewRemoteFS(ctx, *dst)
+	fs, err := actions.NewRemoteFS(ctx, *dst, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-dst`=%q: %w", *dst, err)
 	}
@@ -255,7 +255,7 @@ func newOriginFS(ctx context.Context) (common.OriginFS, error) {
 	if len(*origin) == 0 {
 		return &fsnil.FS{}, nil
 	}
-	fs, err := actions.NewRemoteFS(ctx, *origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}
@@ -266,7 +266,7 @@ func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
 	if len(*origin) == 0 {
 		return nil, fmt.Errorf("-origin cannot be empty when -snapshotName and -snapshot.createURL aren't set")
 	}
-	fs, err := actions.NewRemoteFS(ctx, *origin)
+	fs, err := actions.NewRemoteFS(ctx, *origin, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
 	}

app/vmctl/flags.go

@@ -468,7 +468,7 @@ var (
 			Name: vmNativeFilterMatch,
 			Usage: "Time series selector to match series for export. For example, select {instance!=\"localhost\"} will " +
 				"match all series with \"instance\" label different to \"localhost\".\n" +
-				" See more details here https://github.com/VictoriaMetrics/VictoriaMetrics#how-to-export-data-in-native-format",
+				" See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format",
 			Value: `{__name__!=""}`,
 		},
 		&cli.StringFlag{

app/vmctl/influx.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"log"
@@ -37,7 +38,7 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st
 	}
 }
 
-func (ip *influxProcessor) run() error {
+func (ip *influxProcessor) run(ctx context.Context) error {
 	series, err := ip.ic.Explore()
 	if err != nil {
 		return fmt.Errorf("explore query failed: %s", err)
@@ -47,7 +48,7 @@ func (ip *influxProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d timeseries to import. Continue?", len(series))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/main.go

@@ -103,7 +103,7 @@ func main() {
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency), c.Bool(globalVerbose))
-					return otsdbProcessor.run()
+					return otsdbProcessor.run(ctx)
 				},
 			},
 			{
@@ -164,7 +164,7 @@ func main() {
 						c.Bool(influxSkipDatabaseLabel),
 						c.Bool(influxPrometheusMode),
 						c.Bool(globalVerbose))
-					return processor.run()
+					return processor.run(ctx)
 				},
 			},
 			{
@@ -279,7 +279,7 @@ func main() {
 						cc:        c.Int(promConcurrency),
 						isVerbose: c.Bool(globalVerbose),
 					}
-					return pp.run()
+					return pp.run(ctx)
 				},
 			},
 			{

app/vmctl/opentsdb.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
@@ -37,7 +38,7 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, verbos
 	}
 }
 
-func (op *otsdbProcessor) run() error {
+func (op *otsdbProcessor) run(ctx context.Context) error {
 	log.Println("Loading all metrics from OpenTSDB for filters: ", op.oc.Filters)
 	var metrics []string
 	for _, filter := range op.oc.Filters {
@@ -53,7 +54,7 @@ func (op *otsdbProcessor) run() error {
 	}
 
 	question := fmt.Sprintf("Found %d metrics to import. Continue?", len(metrics))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 	op.im.ResetStats()

app/vmctl/prometheus.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
+	"strings"
 	"sync"
 
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 
@@ -30,7 +33,7 @@ type prometheusProcessor struct {
 	isVerbose bool
 }
 
-func (pp *prometheusProcessor) run() error {
+func (pp *prometheusProcessor) run(ctx context.Context) error {
 	blocks, err := pp.cl.Explore()
 	if err != nil {
 		return fmt.Errorf("explore failed: %s", err)
@@ -39,7 +42,7 @@ func (pp *prometheusProcessor) run() error {
 		return fmt.Errorf("found no blocks to import")
 	}
 	question := fmt.Sprintf("Found %d blocks to import. Continue?", len(blocks))
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 
@@ -60,19 +63,19 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
-		var labels []vm.LabelPair
+		var labelPairs []vm.LabelPair
 		series := ss.At()
 
-		for _, label := range series.Labels() {
+		series.Labels().Range(func(label labels.Label) {
 			if label.Name == "__name__" {
 				name = label.Value
-				continue
+				return
 			}
-			labels = append(labels, vm.LabelPair{
-				Name:  label.Name,
-				Value: label.Value,
+			labelPairs = append(labelPairs, vm.LabelPair{
+				Name:  strings.Clone(label.Name),
+				Value: strings.Clone(label.Value),
 			})
-		}
+		})
 		if name == "" {
 			return fmt.Errorf("failed to find `__name__` label in labelset for block %v", b.Meta().ULID)
 		}
@@ -98,7 +101,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 		}
 		ts := vm.TimeSeries{
 			Name:       name,
-			LabelPairs: labels,
+			LabelPairs: labelPairs,
 			Timestamps: timestamps,
 			Values:     values,
 		}

app/vmctl/remoteread.go

@@ -47,7 +47,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 
 	question := fmt.Sprintf("Selected time range %q - %q will be split into %d ranges according to %q step. Continue?",
 		rrp.filter.timeStart.String(), rrp.filter.timeEnd.String(), len(ranges), rrp.filter.chunk)
-	if !prompt(question) {
+	if !prompt(ctx, question) {
 		return nil
 	}
 

app/vmctl/utils.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -15,7 +16,7 @@ const barTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "
 // isSilent should be inited in main
 var isSilent bool
 
-func prompt(question string) bool {
+func prompt(ctx context.Context, question string) bool {
 	if isSilent {
 		return true
 	}
@@ -25,15 +26,32 @@ func prompt(question string) bool {
 	}
 	reader := bufio.NewReader(os.Stdin)
 	fmt.Print(question, " [Y/n] ")
-	answer, err := reader.ReadString('\n')
-	if err != nil {
+
+	answerCh := make(chan string, 1)
+	errCh := make(chan error, 1)
+
+	go func() {
+		answer, err := reader.ReadString('\n')
+		if err != nil {
+			errCh <- err
+			return
+		}
+		answerCh <- answer
+	}()
+
+	select {
+	case <-ctx.Done():
+		fmt.Println("\nCanceled.")
+		return false
+	case err := <-errCh:
 		panic(err)
+	case answer := <-answerCh:
+		answer = strings.TrimSpace(strings.ToLower(answer))
+		if answer == "" || answer == "yes" || answer == "y" {
+			return true
+		}
+		return false
 	}
-	answer = strings.TrimSpace(strings.ToLower(answer))
-	if answer == "" || answer == "yes" || answer == "y" {
-		return true
-	}
-	return false
 }
 
 func wrapErr(vmErr *vm.ImportError, verbose bool) error {

app/vmctl/vm_native.go

@@ -79,7 +79,7 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 			return fmt.Errorf("failed to get tenants: %w", err)
 		}
 		question := fmt.Sprintf("The following tenants were discovered: %s.\n Continue?", tenants)
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	}
@@ -233,7 +233,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		// do not prompt for intercluster because there could be many tenants,
 		// and we don't want to interrupt the process when moving to the next tenant.
 		question := foundSeriesMsg + ". Continue?"
-		if !prompt(question) {
+		if !prompt(ctx, question) {
 			return nil
 		}
 	} else {

app/vminsert/main.go

@@ -28,6 +28,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/promremotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/zabbixconnector"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -201,9 +202,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, `vminsert - a component of VictoriaMetrics cluster<br/>
-			<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/">docs</a><br>
-`)
+		fmt.Fprintf(w, `vminsert - a component of VictoriaMetrics cluster<br/>`)
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
+		fmt.Fprintf(w, `<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/">docs</a><br>`)
 		return true
 	}
 	p, err := httpserver.ParsePath(r.URL.Path)
@@ -310,6 +311,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
+	case "zabbixconnector/api/v1/history":
+		zabbixconnectorHistoryRequests.Inc()
+		if err := zabbixconnector.InsertHandlerForHTTP(at, r); err != nil {
+			zabbixconnectorHistoryErrors.Inc()
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":%q}`, err.Error())
+			return true
+		}
+		w.WriteHeader(http.StatusOK)
+		return true
 	case "newrelic":
 		newrelicCheckRequest.Inc()
 		w.Header().Set("Content-Type", "application/json")
@@ -443,8 +455,10 @@ var (
 	datadogv2WriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/datadog/api/v2/series", protocol="datadog"}`)
 	datadogv2WriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/datadog/api/v2/series", protocol="datadog"}`)
 
-	datadogsketchesWriteRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/datadog/api/beta/sketches", protocol="datadog"}`)
-	datadogsketchesWriteErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogsketchesWriteRequests   = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/datadog/api/beta/sketches", protocol="datadog"}`)
+	datadogsketchesWriteErrors     = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/datadog/api/beta/sketches", protocol="datadog"}`)
+	zabbixconnectorHistoryRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
+	zabbixconnectorHistoryErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/insert/{}/zabbixconnector/api/v1/history", protocol="zabbixconnector"}`)
 
 	datadogValidateRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/datadog/api/v1/validate", protocol="datadog"}`)
 	datadogCheckRunRequests = metrics.NewCounter(`vm_http_requests_total{path="/insert/{}/datadog/api/v1/check_run", protocol="datadog"}`)

app/vminsert/netstorage/emwa.go

@@ -0,0 +1,64 @@
+package netstorage
+
+// Copied and pasted from github.com/VividCortex/ewma to avoid adding a new dependency.
+// Reduced to only the needed functionality.
+
+const (
+	// For best results, the moving average should not be initialized to the
+	// samples it sees immediately. The book "Production and Operations
+	// Analysis" by Steven Nahmias suggests initializing the moving average to
+	// the mean of the first 10 samples. Until the VariableEwma has seen this
+	// many samples, it is not "ready" to be queried for the value of the
+	// moving average. This adds some memory cost.
+	warmupSamples uint8 = 10
+)
+
+func newMovingAverage(age float64) *variableEWMA {
+	return &variableEWMA{
+		decay: 2 / (age + 1),
+	}
+}
+
+// variableEWMA represents the exponentially weighted moving average of a series of
+// numbers. Unlike SimpleEWMA, it supports a custom age, and thus uses more memory.
+type variableEWMA struct {
+	// The multiplier factor by which the previous samples decay.
+	decay float64
+	// The current value of the average.
+	value float64
+	// The number of samples added to this instance.
+	count uint8
+}
+
+// Add adds a value to the series and updates the moving average.
+func (e *variableEWMA) Add(value float64) {
+	switch {
+	case e.count < warmupSamples:
+		e.count++
+		e.value += value
+	case e.count == warmupSamples:
+		e.count++
+		e.value = e.value / float64(warmupSamples)
+		e.value = (value * e.decay) + (e.value * (1 - e.decay))
+	default:
+		e.value = (value * e.decay) + (e.value * (1 - e.decay))
+	}
+}
+
+// Value returns the current value of the average, or 0.0 if the series hasn't
+// warmed up yet.
+func (e *variableEWMA) Value() float64 {
+	if e.count <= warmupSamples {
+		return 0.0
+	}
+
+	return e.value
+}
+
+// Set sets the EWMA's value.
+func (e *variableEWMA) Set(value float64) {
+	e.value = value
+	if e.count <= warmupSamples {
+		e.count = warmupSamples + 1
+	}
+}

app/vminsert/netstorage/netstorage.go

@@ -126,10 +126,11 @@ again:
 		return nil
 	}
 	// Slow path: the buf contents doesn't fit sn.buf, so try re-routing it to other vmstorage nodes.
-	if *disableRerouting || len(sns) == 1 {
+	if !allowRerouting(sn, sns) {
 		sn.brCond.Wait()
 		goto again
 	}
+
 	sn.brLock.Unlock()
 	rowsProcessed, err := rerouteRowsToFreeStorageNodes(snb, sn, buf, getRowHasher)
 	rows -= rowsProcessed
@@ -315,6 +316,12 @@ func (sn *storageNode) sendBufRowsNonblocking(br *bufRows) bool {
 	}
 	duration := time.Since(startTime)
 	sn.sendDurationSeconds.Add(duration.Seconds())
+
+	now := time.Now()
+	saturation := float64(now.Sub(startTime)) / float64(now.Sub(sn.lastSendTime))
+	sn.avgSaturation.Add(saturation)
+	sn.lastSendTime = now
+
 	if err == nil {
 		if deadline := sn.rpcIsNotSupportedDeadline.Load(); deadline > 0 {
 			sn.rpcIsNotSupportedDeadline.Store(0)
@@ -455,6 +462,11 @@ type storageNode struct {
 	// The total duration spent for sending data to vmstorage node.
 	// This metric is useful for determining the saturation of vminsert->vmstorage link.
 	sendDurationSeconds *metrics.FloatCounter
+
+	// avgSaturation tracks the moving average of (send duration / (now - lastSendTime)).
+	// Updated in run(). Used by allowRerouting to decide when to trigger slowness-based rerouting.
+	avgSaturation *variableEWMA
+	lastSendTime  time.Time
 }
 
 type storageNodesBucket struct {
@@ -557,6 +569,9 @@ func initStorageNodes(unsortedAddrs []string, rpcCall vminsertapi.RPCCall, hashS
 			rowsReroutedFromHere:  ms.NewCounter(fmt.Sprintf(`vm_rpc_rows_rerouted_from_here_total{name="vminsert", addr=%q, rpc_call=%q}`, addr, rpcCall.Name)),
 			rowsReroutedToHere:    ms.NewCounter(fmt.Sprintf(`vm_rpc_rows_rerouted_to_here_total{name="vminsert", addr=%q, rpc_call=%q}`, addr, rpcCall.Name)),
 			sendDurationSeconds:   ms.NewFloatCounter(fmt.Sprintf(`vm_rpc_send_duration_seconds_total{name="vminsert", addr=%q, rpc_call=%q}`, addr, rpcCall.Name)),
+
+			avgSaturation: newMovingAverage(180),
+			lastSendTime:  time.Now(),
 		}
 		sn.brCond = sync.NewCond(&sn.brLock)
 		_ = ms.NewGauge(fmt.Sprintf(`vm_rpc_rows_pending{name="vminsert", addr=%q, rpc_call=%q}`, addr, rpcCall.Name), func() float64 {
@@ -700,7 +715,69 @@ func rerouteRowsToReadyStorageNodes(snb *storageNodesBucket, snSource *storageNo
 	return rowsProcessed, nil
 }
 
-// reouteRowsToFreeStorageNodes re-routes src from snSource to other storage nodes.
+var reroutingLogger = logger.WithThrottler("allowRerouting", 5*time.Second)
+
+// allowRerouting determines whether data should be rerouted from snSource to other storage nodes (sns)
+// based on performance metrics.
+//
+// It returns true only when snSource is the slowest node in the cluster
+// and significantly slower than the cluster on average.
+// See the comments below for detailed conditions.
+func allowRerouting(snSource *storageNode, sns []*storageNode) bool {
+	if *disableRerouting {
+		return false
+	}
+
+	// Do not allow rerouting if saturation is not yet warmed up.
+	snSourceSaturation := snSource.avgSaturation.Value()
+	if snSourceSaturation == 0 {
+		return false
+	}
+
+	saturations := make([]float64, 0, len(sns))
+	for _, sn := range sns {
+		// Skip not ready storage nodes.
+		if !sn.isReady() {
+			continue
+		}
+		// Do not allow rerouting if avgSaturation is not yet warmed up.
+		if sn.avgSaturation.Value() == 0 {
+			return false
+		}
+
+		// Do not allow rerouting if there is a slower storage node
+		snSaturation := sn.avgSaturation.Value()
+		if snSourceSaturation < snSaturation {
+			return false
+		}
+
+		saturations = append(saturations, snSaturation)
+	}
+	// Do not allow rerouting if there are less than 3 ready storage nodes.
+	if len(saturations) < 3 {
+		return false
+	}
+
+	// Calculate median saturation
+	sort.Float64s(saturations)
+	var medianSaturation float64
+	n := len(saturations)
+	if n%2 == 0 {
+		medianSaturation = (saturations[n/2-1] + saturations[n/2]) / 2
+	} else {
+		medianSaturation = saturations[n/2]
+	}
+
+	// Do not allow rerouting if the cluster is significantly overloaded.
+	if medianSaturation > 0.80 {
+		return false
+	}
+
+	reroutingLogger.Warnf("reroute metrics from the slowest storage %q with saturation %.2f, where cluster median saturation is %.2f", snSource.dialer.Addr(), snSourceSaturation, medianSaturation)
+	return true
+}
+
+// rerouteRowsToFreeStorageNodes re-routes src from snSource to other storage nodes.
 //
 // It is expected that snSource has no enough buffer for sending src.
 // It is expected than *disableRerouting isn't set when calling this function.

app/vminsert/netstorage/netstorage_test.go

@@ -4,6 +4,10 @@ import (
 	"flag"
 	"runtime"
 	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 func TestInitStopNodes(t *testing.T) {
@@ -30,3 +34,103 @@ func TestInitStopNodes(t *testing.T) {
 		MustStop()
 	}
 }
+
+func TestAllowRerouting(t *testing.T) {
+	originDisableRerouting := *disableRerouting
+	t.Cleanup(func() {
+		*disableRerouting = originDisableRerouting
+	})
+	// Enable rerouting for the test
+	*disableRerouting = false
+
+	newStorage := func(avgSaturation float64, ready bool) *storageNode {
+		sn := &storageNode{
+			avgSaturation: newMovingAverage(180),
+			dialer:        netutil.NewTCPDialer(metrics.NewSet(), "aName", "anAddr", time.Second, time.Second),
+		}
+		sn.isBroken.Store(!ready)
+		sn.avgSaturation.Set(avgSaturation)
+
+		return sn
+	}
+
+	f := func(sns []*storageNode, snSourceIdx int, expected bool) {
+		t.Helper()
+
+		snSource := sns[snSourceIdx]
+
+		actual := allowRerouting(snSource, sns)
+
+		if actual != expected {
+			t.Errorf("unexpected allowRerouting result for snSourceIdx=%d from %d storages; got %v; want %v", snSourceIdx, len(sns), actual, expected)
+		}
+	}
+
+	// rerouting is triggered on the slowest node if cluster median saturation less than or equal 0.8
+	f([]*storageNode{
+		newStorage(0.81, true),
+		newStorage(0.79, true),
+		newStorage(0.1, true),
+	}, 0, true)
+
+	// four nodes median test
+	f([]*storageNode{
+		newStorage(0.82, true),
+		newStorage(0.81, true),
+		newStorage(0.79, true),
+		newStorage(0.1, true),
+	}, 0, true)
+
+	// rerouting not triggered because rerouting disabled by flag
+	*disableRerouting = true
+	f([]*storageNode{
+		newStorage(0.81, true),
+		newStorage(0.79, true),
+		newStorage(0.79, true),
+	}, 0, false)
+	*disableRerouting = false
+
+	// rerouting not triggered because cluster median saturation more than 0.8
+	f([]*storageNode{
+		newStorage(0.81, true),
+		newStorage(0.801, true),
+		newStorage(0.1, true),
+	}, 0, false)
+
+	// four nodes median test
+	f([]*storageNode{
+		newStorage(0.82, true),
+		newStorage(0.82, true),
+		newStorage(0.79, true),
+		newStorage(0.1, true),
+	}, 0, false)
+
+	// rerouting not triggered because snSource not the slowest
+	f([]*storageNode{
+		newStorage(0.81, true),
+		newStorage(0.801, true),
+		newStorage(0.1, true),
+	}, 2, false)
+
+	// rerouting not triggered if not enough nodes
+	f([]*storageNode{
+		newStorage(0.81, true),
+		newStorage(0.01, true),
+	}, 0, false)
+
+	// rerouting not triggered if not enough ready nodes
+	f([]*storageNode{
+		newStorage(0.81, true),
+		newStorage(0.01, true),
+		newStorage(0.01, false),
+		newStorage(0.01, false),
+	}, 0, false)
+
+	// rerouting triggered if enough ready nodes
+	f([]*storageNode{
+		newStorage(0.81, true),
+		newStorage(0.01, true),
+		newStorage(0.01, true),
+		newStorage(0, false),
+	}, 0, true)
+}

app/vminsert/zabbixconnector/request_handler.go

@@ -0,0 +1,73 @@
+package zabbixconnector
+
+import (
+	"net/http"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/zabbixconnector/stream"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
+)
+
+var (
+	rowsInserted       = metrics.NewCounter(`vm_rows_inserted_total{type="zabbixconnector"}`)
+	rowsPerInsert      = metrics.NewHistogram(`vm_rows_per_insert{type="zabbixconnector"}`)
+	rowsTenantInserted = tenantmetrics.NewCounterMap(`vm_tenant_inserted_rows_total{type="zabbixconnector"}`)
+)
+
+// InsertHandlerForHTTP processes remote write for ZabbixConnector POST /zabbixconnector/v1/history request.
+func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
+	extraLabels, err := protoparserutil.GetExtraLabels(req)
+	if err != nil {
+		return err
+	}
+	encoding := req.Header.Get("Content-Encoding")
+	return stream.Parse(req.Body, encoding, func(rows []zabbixconnector.Row) error {
+		return insertRows(at, rows, extraLabels)
+	})
+}
+
+func insertRows(at *auth.Token, rows []zabbixconnector.Row, extraLabels []prompb.Label) error {
+	ctx := netstorage.GetInsertCtx()
+	defer netstorage.PutInsertCtx(ctx)
+
+	ctx.Reset() // This line is required for initializing ctx internals.
+	rowsTotal := 0
+	perTenantRows := make(map[auth.Token]int)
+	hasRelabeling := relabel.HasRelabeling()
+	for i := range rows {
+		r := &rows[i]
+		rowsTotal++
+		ctx.Labels = ctx.Labels[:0]
+		for k := range r.Tags {
+			t := &r.Tags[k]
+			ctx.AddLabelBytes(t.Key, t.Value)
+		}
+		for k := range extraLabels {
+			label := &extraLabels[k]
+			ctx.AddLabel(label.Name, label.Value)
+		}
+		ctx.TryPrepareLabels(hasRelabeling)
+		if len(ctx.Labels) == 0 {
+			// Skip metric without labels.
+			continue
+		}
+		atLocal := ctx.GetLocalAuthToken(at)
+		if err := ctx.WriteDataPoint(atLocal, ctx.Labels, r.Timestamp, r.Value); err != nil {
+			return err
+		}
+		perTenantRows[*atLocal]++
+	}
+
+	rowsInserted.Add(rowsTotal)
+	rowsPerInsert.Update(float64(rowsTotal))
+	rowsTenantInserted.MultiAdd(perTenantRows)
+
+	return ctx.FlushBufs()
+}

app/vmrestore/main.go

@@ -104,7 +104,7 @@ func newDstFS() (*fslocal.FS, error) {
 }
 
 func newSrcFS(ctx context.Context) (common.RemoteFS, error) {
-	fs, err := actions.NewRemoteFS(ctx, *src)
+	fs, err := actions.NewRemoteFS(ctx, *src, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse `-src`=%q: %w", *src, err)
 	}

app/vmselect/clusternative/vmselect.go

@@ -10,6 +10,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
@@ -133,44 +134,48 @@ func (api *vmstorageAPI) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
 	workCh chan workItem
+	wis    []workItem
 	wg     sync.WaitGroup
 	err    error
 }
 
 type workItem struct {
-	mb     *storage.MetricBlock
-	doneCh chan struct{}
+	rawMetricBlock []byte
+	doneCh         chan struct{}
 }
 
 func newBlockIterator(qt *querytracer.Tracer, denyPartialResponse bool, sq *storage.SearchQuery, deadline searchutil.Deadline) *blockIterator {
-	var bi blockIterator
-	bi.workCh = make(chan workItem, 16)
+	bi := getBlockIterator()
 	bi.wg.Add(1)
+	workers, processBlocks := netstorage.PrepareProcessRawBlocks(qt, denyPartialResponse, sq, deadline)
+	bi.workCh = make(chan workItem, workers)
+	bi.wis = slicesutil.SetLength(bi.wis, workers)
+	for i := range bi.wis {
+		bi.wis[i].doneCh = make(chan struct{})
+	}
 	go func() {
-		_, err := netstorage.ProcessBlocks(qt, denyPartialResponse, sq, func(mb *storage.MetricBlock, _ uint) error {
-			wi := workItem{
-				mb:     mb,
-				doneCh: make(chan struct{}),
-			}
+		_, err := processBlocks(func(mb []byte, workerID uint) error {
+			wi := bi.wis[workerID]
+			wi.rawMetricBlock = mb
 			bi.workCh <- wi
 			<-wi.doneCh
 			return nil
-		}, deadline)
+		})
 		close(bi.workCh)
 		bi.err = err
 		bi.wg.Done()
 	}()
-	return &bi
+	return bi
 }
 
-func (bi *blockIterator) NextBlock(mb *storage.MetricBlock) bool {
+func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
 	wi, ok := <-bi.workCh
 	if !ok {
-		return false
+		return nil, false
 	}
-	mb.CopyFrom(wi.mb)
+	dst = append(dst, wi.rawMetricBlock...)
 	wi.doneCh <- struct{}{}
-	return true
+	return dst, true
 }
 
 func (bi *blockIterator) Error() error {
@@ -179,11 +184,34 @@ func (bi *blockIterator) Error() error {
 }
 
 func (bi *blockIterator) MustClose() {
-	var mb storage.MetricBlock
-	for bi.NextBlock(&mb) {
+	var buf []byte
+	var ok bool
+	for {
+		buf, ok = bi.NextBlock(buf[:0])
+		if !ok {
+			break
+		}
 		// Drain pending blocks before exit in order to free up
 		// the goroutine started at newBlockIterator
 	}
 	// Wait until the goroutine from newBlockIterator is finished.
 	bi.wg.Wait()
+	for i := range bi.wis {
+		wi := &bi.wis[i]
+		wi.rawMetricBlock = nil
+		wi.doneCh = nil
+	}
+	bi.err = nil
+	bi.workCh = nil
+	blockIteratorsPool.Put(bi)
+}
+
+var blockIteratorsPool sync.Pool
+
+func getBlockIterator() *blockIterator {
+	v := blockIteratorsPool.Get()
+	if v == nil {
+		v = &blockIterator{}
+	}
+	return v.(*blockIterator)
 }

app/vmselect/main.go

@@ -566,9 +566,9 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 			return false
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
-		fmt.Fprintf(w, `vmselect - a component of VictoriaMetrics cluster<br/>
-<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/">docs</a><br>
-`)
+		fmt.Fprintf(w, `vmselect - a component of VictoriaMetrics cluster<br/>`)
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
+		fmt.Fprintf(w, `<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/">docs</a><br>`)
 		return true
 	}
 	if path == "/api/v1/status/top_queries" {
@@ -597,6 +597,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>VictoriaMetrics cluster - vmselect</h2></br>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See <a href='https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format'>docs</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		fmt.Fprintf(w, `<a href="vmui">Web UI</a><br>`)
@@ -971,9 +972,10 @@ func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request, path string) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	r.URL.Path = strings.TrimPrefix(path, "prometheus")
-	r.Host = vmalertProxyHost
-	vmalertProxy.ServeHTTP(w, r)
+	req := r.Clone(r.Context())
+	req.URL.Path = strings.TrimPrefix(path, "prometheus")
+	req.Host = vmalertProxyHost
+	vmalertProxy.ServeHTTP(w, req)
 }
 
 var (

app/vmselect/netstorage/netstorage.go

@@ -1902,16 +1902,38 @@ func ProcessSearchQuery(qt *querytracer.Tracer, denyPartialResponse bool, sq *st
 	return &rss, isPartial, nil
 }
 
-// ProcessBlocks calls processBlock per each block matching the given sq.
-func ProcessBlocks(qt *querytracer.Tracer, denyPartialResponse bool, sq *storage.SearchQuery,
-	processBlock func(mb *storage.MetricBlock, workerID uint) error, deadline searchutil.Deadline,
-) (bool, error) {
+// PrepareProcessRawBlocks prepares metric blocks processor.
+//
+// Returns workers count and processBlocks function
+func PrepareProcessRawBlocks(qt *querytracer.Tracer, denyPartialResponse bool, sq *storage.SearchQuery,
+	deadline searchutil.Deadline,
+) (int, func(processBlock func(mb []byte, workerID uint) error) (bool, error)) {
 	sns := getStorageNodes()
-	return processBlocks(qt, sns, denyPartialResponse, sq, processBlock, deadline)
+	return len(sns), func(processBlock func(mb []byte, workerID uint) error) (bool, error) {
+		return processBlocksInternal(qt, sns, denyPartialResponse, sq, processBlock, deadline)
+	}
 }
 
 func processBlocks(qt *querytracer.Tracer, sns []*storageNode, denyPartialResponse bool, sq *storage.SearchQuery,
 	processBlock func(mb *storage.MetricBlock, workerID uint) error, deadline searchutil.Deadline,
+) (bool, error) {
+	mbs := make([]storage.MetricBlock, len(sns))
+	f := func(rawBlock []byte, workerID uint) error {
+		mb := &mbs[workerID]
+		tail, err := mb.Unmarshal(rawBlock)
+		if err != nil {
+			return fmt.Errorf("cannot unmarshal MetricBlock from %d bytes: %w", len(rawBlock), err)
+		}
+		if len(tail) > 0 {
+			return fmt.Errorf("non-empty tail after unmarshaling MetricBlock: (len=%d) %q", len(tail), tail)
+		}
+		return processBlock(mb, workerID)
+	}
+	return processBlocksInternal(qt, sns, denyPartialResponse, sq, f, deadline)
+}
+
+func processBlocksInternal(qt *querytracer.Tracer, sns []*storageNode, denyPartialResponse bool, sq *storage.SearchQuery,
+	processBlock func(rawBlock []byte, workerID uint) error, deadline searchutil.Deadline,
 ) (bool, error) {
 	// Make sure that processBlock is no longer called after the exit from processBlocks() function.
 	// Use per-worker WaitGroup instead of a shared WaitGroup in order to avoid inter-CPU contention,
@@ -1933,7 +1955,7 @@ func processBlocks(qt *querytracer.Tracer, sns []*storageNode, denyPartialRespon
 		_ [atomicutil.CacheLineSize - unsafe.Sizeof(wgStruct{})%atomicutil.CacheLineSize]byte
 	}
 	wgs := make([]wgWithPadding, len(sns))
-	f := func(mb *storage.MetricBlock, workerID uint) error {
+	f := func(rawBlock []byte, workerID uint) error {
 		muwg := &wgs[workerID]
 		muwg.mu.Lock()
 		if muwg.stop {
@@ -1942,7 +1964,7 @@ func processBlocks(qt *querytracer.Tracer, sns []*storageNode, denyPartialRespon
 		}
 		muwg.wg.Add(1)
 		muwg.mu.Unlock()
-		err := processBlock(mb, workerID)
+		err := processBlock(rawBlock, workerID)
 		muwg.wg.Done()
 		return err
 	}
@@ -2099,10 +2121,11 @@ func (snr *storageNodesRequest) collectResults(partialResultsCounter *metrics.Co
 		if err := f(result.data); err != nil {
 			snr.finishQueryTracer(result.qt, fmt.Sprintf("error: %s", err))
 			var er *errRemote
-			if errors.As(err, &er) {
+			if errors.As(err, &er) && !strings.Contains(er.msg, "search.maxConcurrentRequests") {
 				// Immediately return the error reported by vmstorage to the caller,
 				// since such errors usually mean misconfiguration at vmstorage.
 				// The misconfiguration must be known by the caller, so it is fixed ASAP.
+				// Hitting maxConcurrentRequests limit is not fatal if replicationFactor > 1.
 				snr.finishQueryTracers("cancel request because of error in other vmstorage nodes")
 				return false, err
 			}
@@ -2309,9 +2332,6 @@ type storageNode struct {
 	// The number of metric blocks read.
 	metricBlocksRead *metrics.Counter
 
-	// The number of read metric rows.
-	metricRowsRead *metrics.Counter
-
 	// The number of list tenants requests to storageNode.
 	tenantsRequests *metrics.Counter
 
@@ -2463,7 +2483,7 @@ func (sn *storageNode) processSearchMetricNames(qt *querytracer.Tracer, requestD
 	return metricNames, nil
 }
 
-func (sn *storageNode) processSearchQuery(qt *querytracer.Tracer, requestData []byte, processBlock func(mb *storage.MetricBlock, workerID uint) error,
+func (sn *storageNode) processSearchQuery(qt *querytracer.Tracer, requestData []byte, processBlock func(rawBlock []byte, workerID uint) error,
 	workerID uint, deadline searchutil.Deadline,
 ) error {
 	f := func(bc *handshake.BufferedConn) error {
@@ -3016,7 +3036,7 @@ func (sn *storageNode) processSearchMetricNamesOnConn(bc *handshake.BufferedConn
 const maxMetricNameSize = 64 * 1024
 
 func (sn *storageNode) processSearchQueryOnConn(bc *handshake.BufferedConn, requestData []byte,
-	processBlock func(mb *storage.MetricBlock, workerID uint) error, workerID uint,
+	processBlock func(rawBlock []byte, workerID uint) error, workerID uint,
 ) error {
 	// Send the request to sn.
 	if err := writeBytes(bc, requestData); err != nil {
@@ -3037,7 +3057,6 @@ func (sn *storageNode) processSearchQueryOnConn(bc *handshake.BufferedConn, requ
 
 	// Read response. It may consist of multiple MetricBlocks.
 	blocksRead := 0
-	var mb storage.MetricBlock
 	for {
 		buf, err = readBytes(buf[:0], bc, maxMetricBlockSize)
 		if err != nil {
@@ -3047,18 +3066,10 @@ func (sn *storageNode) processSearchQueryOnConn(bc *handshake.BufferedConn, requ
 			// Reached the end of the response
 			return nil
 		}
-		tail, err := mb.Unmarshal(buf)
-		if err != nil {
-			return fmt.Errorf("cannot unmarshal MetricBlock #%d from %d bytes: %w", blocksRead, len(buf), err)
-		}
-		if len(tail) != 0 {
-			return fmt.Errorf("non-empty tail after unmarshaling MetricBlock #%d: (len=%d) %q", blocksRead, len(tail), tail)
-		}
 		blocksRead++
 		sn.metricBlocksRead.Inc()
-		sn.metricRowsRead.Add(mb.Block.RowsCount())
-		if err := processBlock(&mb, workerID); err != nil {
-			return fmt.Errorf("cannot process MetricBlock #%d: %w", blocksRead, err)
+		if err := processBlock(buf, workerID); err != nil {
+			return fmt.Errorf("cannot process Raw MetricBlock #%d: %w", blocksRead, err)
 		}
 	}
 }
@@ -3278,7 +3289,6 @@ func newStorageNode(ms *metrics.Set, group *storageNodesGroup, addr string) *sto
 		metricsMetadataErrors:       ms.NewCounter(fmt.Sprintf(`vm_request_errors_total{action="metricsMetadata", type="rpcClient", name="vmselect", addr=%q}`, addr)),
 
 		metricBlocksRead: ms.NewCounter(fmt.Sprintf(`vm_metric_blocks_read_total{name="vmselect", addr=%q}`, addr)),
-		metricRowsRead:   ms.NewCounter(fmt.Sprintf(`vm_metric_rows_read_total{name="vmselect", addr=%q}`, addr)),
 	}
 	return sn
 }
@@ -3353,7 +3363,7 @@ func (pnc *perNodeCounter) GetTotal() uint64 {
 const maxFastAllocBlockSize = 32 * 1024
 
 // execSearchQueryRequest executes processSearchQuery for each searchQuery tenant.
-func execSearchQueryRequest(qt *querytracer.Tracer, sq *storage.SearchQuery, workerID uint, sn *storageNode, f func(mb *storage.MetricBlock, workerID uint) error, deadline searchutil.Deadline) error {
+func execSearchQueryRequest(qt *querytracer.Tracer, sq *storage.SearchQuery, workerID uint, sn *storageNode, f func(rawBlock []byte, workerID uint) error, deadline searchutil.Deadline) error {
 	var requestData []byte
 
 	for i := range sq.TenantTokens {

app/vmselect/prometheus/metadata_response.qtpl

@@ -1,6 +1,5 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
-  "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 ) %}
 
@@ -19,7 +18,7 @@ See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-m
 		{% for _, row := range result %}
 			"{%s string(row.MetricFamilyName) %}": [
 					{
-						"type": {%q= prompb.MetricMetadataTypeToString(row.Type) %},
+						"type": {%q= row.Type.String() %},
             {% if len(row.Unit) > 0 -%}
 						"unit": {%q= string(row.Unit) %},
             {% endif -%}

app/vmselect/prometheus/metadata_response.qtpl.go

@@ -6,116 +6,115 @@ package prometheus
 
 //line app/vmselect/prometheus/metadata_response.qtpl:1
 import (
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 )
 
 // MetadataResponse generates response for /api/v1/metadataSee https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
 
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 func StreamMetadataResponse(qw422016 *qt422016.Writer, isPartial bool, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/metadata_response.qtpl:10
+//line app/vmselect/prometheus/metadata_response.qtpl:9
 	qw422016.N().S(`{"status":"success","isPartial":`)
-//line app/vmselect/prometheus/metadata_response.qtpl:13
+//line app/vmselect/prometheus/metadata_response.qtpl:12
 	if isPartial {
-//line app/vmselect/prometheus/metadata_response.qtpl:13
+//line app/vmselect/prometheus/metadata_response.qtpl:12
 		qw422016.N().S(`true`)
-//line app/vmselect/prometheus/metadata_response.qtpl:13
+//line app/vmselect/prometheus/metadata_response.qtpl:12
 	} else {
-//line app/vmselect/prometheus/metadata_response.qtpl:13
+//line app/vmselect/prometheus/metadata_response.qtpl:12
 		qw422016.N().S(`false`)
-//line app/vmselect/prometheus/metadata_response.qtpl:13
+//line app/vmselect/prometheus/metadata_response.qtpl:12
 	}
-//line app/vmselect/prometheus/metadata_response.qtpl:13
+//line app/vmselect/prometheus/metadata_response.qtpl:12
 	qw422016.N().S(`,"data": {`)
-//line app/vmselect/prometheus/metadata_response.qtpl:16
+//line app/vmselect/prometheus/metadata_response.qtpl:15
 	mapItems := len(result)
 	currentItem := 0
 
-//line app/vmselect/prometheus/metadata_response.qtpl:19
+//line app/vmselect/prometheus/metadata_response.qtpl:18
 	for _, row := range result {
-//line app/vmselect/prometheus/metadata_response.qtpl:19
+//line app/vmselect/prometheus/metadata_response.qtpl:18
 		qw422016.N().S(`"`)
-//line app/vmselect/prometheus/metadata_response.qtpl:20
+//line app/vmselect/prometheus/metadata_response.qtpl:19
 		qw422016.E().S(string(row.MetricFamilyName))
-//line app/vmselect/prometheus/metadata_response.qtpl:20
+//line app/vmselect/prometheus/metadata_response.qtpl:19
 		qw422016.N().S(`": [{"type":`)
-//line app/vmselect/prometheus/metadata_response.qtpl:22
-		qw422016.N().Q(prompb.MetricMetadataTypeToString(row.Type))
-//line app/vmselect/prometheus/metadata_response.qtpl:22
+//line app/vmselect/prometheus/metadata_response.qtpl:21
+		qw422016.N().Q(row.Type.String())
+//line app/vmselect/prometheus/metadata_response.qtpl:21
 		qw422016.N().S(`,`)
-//line app/vmselect/prometheus/metadata_response.qtpl:23
+//line app/vmselect/prometheus/metadata_response.qtpl:22
 		if len(row.Unit) > 0 {
-//line app/vmselect/prometheus/metadata_response.qtpl:23
+//line app/vmselect/prometheus/metadata_response.qtpl:22
 			qw422016.N().S(`"unit":`)
-//line app/vmselect/prometheus/metadata_response.qtpl:24
+//line app/vmselect/prometheus/metadata_response.qtpl:23
 			qw422016.N().Q(string(row.Unit))
+//line app/vmselect/prometheus/metadata_response.qtpl:23
+			qw422016.N().S(`,`)
 //line app/vmselect/prometheus/metadata_response.qtpl:24
-			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/metadata_response.qtpl:25
 		}
-//line app/vmselect/prometheus/metadata_response.qtpl:25
+//line app/vmselect/prometheus/metadata_response.qtpl:24
 		qw422016.N().S(`"help":`)
-//line app/vmselect/prometheus/metadata_response.qtpl:26
+//line app/vmselect/prometheus/metadata_response.qtpl:25
 		qw422016.N().Q(string(row.Help))
-//line app/vmselect/prometheus/metadata_response.qtpl:26
+//line app/vmselect/prometheus/metadata_response.qtpl:25
 		qw422016.N().S(`}]`)
-//line app/vmselect/prometheus/metadata_response.qtpl:29
+//line app/vmselect/prometheus/metadata_response.qtpl:28
 		if currentItem != mapItems-1 {
-//line app/vmselect/prometheus/metadata_response.qtpl:29
+//line app/vmselect/prometheus/metadata_response.qtpl:28
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/metadata_response.qtpl:29
+//line app/vmselect/prometheus/metadata_response.qtpl:28
 		}
-//line app/vmselect/prometheus/metadata_response.qtpl:30
+//line app/vmselect/prometheus/metadata_response.qtpl:29
 		currentItem++
 
-//line app/vmselect/prometheus/metadata_response.qtpl:31
+//line app/vmselect/prometheus/metadata_response.qtpl:30
 	}
-//line app/vmselect/prometheus/metadata_response.qtpl:31
+//line app/vmselect/prometheus/metadata_response.qtpl:30
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/metadata_response.qtpl:33
+//line app/vmselect/prometheus/metadata_response.qtpl:32
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/metadata_response.qtpl:33
+//line app/vmselect/prometheus/metadata_response.qtpl:32
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 }
 
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 func WriteMetadataResponse(qq422016 qtio422016.Writer, isPartial bool, result []*metricsmetadata.Row, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	StreamMetadataResponse(qw422016, isPartial, result, qt)
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 }
 
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 func MetadataResponse(isPartial bool, result []*metricsmetadata.Row, qt *querytracer.Tracer) string {
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	WriteMetadataResponse(qb422016, isPartial, result, qt)
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 	return qs422016
-//line app/vmselect/prometheus/metadata_response.qtpl:35
+//line app/vmselect/prometheus/metadata_response.qtpl:34
 }

app/vmselect/prometheus/prometheus.go

@@ -49,7 +49,9 @@ var (
 		"If set to true, the query model becomes closer to InfluxDB data model. If set to true, then -search.maxLookback and -search.maxStalenessInterval are ignored")
 	maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+
 		"points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data")
-	selectNodes = flagutil.NewArrayString("selectNode", "Comma-separated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN")
+	selectNodes = flagutil.NewArrayString("selectNode", "A list of vmselect node addresses to propagate the '/internal/resetRollupResultCache' call. "+
+		"If this flag isn't set, then cache need to be purged from each vmselect individually. "+
+		"Comma-separated addresses of vmselect nodes; usage: -selectNode=vmselect-host1,...,vmselect-hostN")
 
 	maxUniqueTimeseries = flag.Int("search.maxUniqueTimeseries", 0, "The maximum number of unique time series, which can be selected during /api/v1/query and /api/v1/query_range queries. This option allows limiting memory usage. "+
 		"The limit can't exceed the explicitly set corresponding value `-search.maxUniqueTimeseries` on vmstorage side.")

app/vmselect/promql/eval.go

@@ -804,7 +804,8 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 // - rollupFunc(m) if iafc is nil
 // - aggrFunc(rollupFunc(m)) if iafc isn't nil
 func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr,
-	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	if re.At == nil {
 		return evalRollupFuncWithoutAt(qt, ec, funcName, rf, expr, re, iafc)
 	}
@@ -854,7 +855,8 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 }
 
 func evalRollupFuncWithoutAt(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
+	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
+) ([]*timeseries, error) {
 	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
@@ -1079,7 +1081,8 @@ func removeNanValues(dstValues []float64, dstTimestamps []int64, values []float6
 
 // evalInstantRollup evaluates instant rollup where ec.Start == ec.End.
 func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64,
+) ([]*timeseries, error) {
 	if ec.Start != ec.End {
 		logger.Panicf("BUG: evalInstantRollup cannot be called on non-empty time range; got %s", ec.timeRangeString())
 	}
@@ -1114,10 +1117,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		rollupResultCacheV.DeleteInstantValues(qt, at, expr, window, ec.Step, ec.EnforcedTagFilterss)
 	}
 	getCachedSeries := func(qt *querytracer.Tracer) ([]*timeseries, int64, error) {
+		rollupResultCacheV.rollupResultCacheRequests.Inc()
 	again:
 		offset := int64(0)
 		tssCached := rollupResultCacheV.GetInstantValues(qt, at, expr, window, ec.Step, ec.EnforcedTagFilterss)
 		if len(tssCached) == 0 {
+			rollupResultCacheV.rollupResultCacheMisses.Inc()
 			// Cache miss. Re-populate the missing data.
 			start := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds()
 			offset = timestamp - start
@@ -1160,6 +1165,7 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			deleteCachedSeries(qt)
 			goto again
 		}
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 		ec.QueryStats.addSeriesFetched(len(tssCached))
 		return tssCached, offset, nil
 	}
@@ -1200,60 +1206,6 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			},
 		}
 		return evalExpr(qt, ec, be)
-	case "rate":
-		if iafc != nil {
-			if !strings.EqualFold(iafc.ae.Name, "sum") {
-				qt.Printf("do not apply instant rollup optimization for incremental aggregate %s()", iafc.ae.Name)
-				return evalAt(qt, timestamp, window)
-			}
-			qt.Printf("optimized calculation for sum(rate(m[d])) as (sum(increase(m[d])) / d)")
-			afe := expr.(*metricsql.AggrFuncExpr)
-			fe := afe.Args[0].(*metricsql.FuncExpr)
-			feIncrease := *fe
-			feIncrease.Name = "increase"
-			// copy RollupExpr to drop possible offset,
-			// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-			newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-			newArg.Offset = nil
-			feIncrease.Args = []metricsql.Expr{newArg}
-			d := newArg.Window.Duration(ec.Step)
-			if d == 0 {
-				d = ec.Step
-			}
-			afeIncrease := *afe
-			afeIncrease.Args = []metricsql.Expr{&feIncrease}
-			be := &metricsql.BinaryOpExpr{
-				Op:              "/",
-				KeepMetricNames: true,
-				Left:            &afeIncrease,
-				Right: &metricsql.NumberExpr{
-					N: float64(d) / 1000,
-				},
-			}
-			return evalExpr(qt, ec, be)
-		}
-		qt.Printf("optimized calculation for instant rollup rate(m[d]) as (increase(m[d]) / d)")
-		fe := expr.(*metricsql.FuncExpr)
-		feIncrease := *fe
-		feIncrease.Name = "increase"
-		// copy RollupExpr to drop possible offset,
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
-		newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
-		newArg.Offset = nil
-		feIncrease.Args = []metricsql.Expr{newArg}
-		d := newArg.Window.Duration(ec.Step)
-		if d == 0 {
-			d = ec.Step
-		}
-		be := &metricsql.BinaryOpExpr{
-			Op:              "/",
-			KeepMetricNames: fe.KeepMetricNames,
-			Left:            &feIncrease,
-			Right: &metricsql.NumberExpr{
-				N: float64(d) / 1000,
-			},
-		}
-		return evalExpr(qt, ec, be)
 	case "max_over_time":
 		if iafc != nil {
 			if !strings.EqualFold(iafc.ae.Name, "max") {
@@ -1622,16 +1574,11 @@ func assertInstantValues(tss []*timeseries) {
 	}
 }
 
-var (
-	rollupResultCacheFullHits    = metrics.NewCounter(`vm_rollup_result_cache_full_hits_total`)
-	rollupResultCachePartialHits = metrics.NewCounter(`vm_rollup_result_cache_partial_hits_total`)
-	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
-
-	memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
-)
+var memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
 
 func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr,
+) ([]*timeseries, error) {
 	window, err := windowExpr.NonNegativeDuration(ec.Step)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", expr.AppendString(nil), err)
@@ -1667,19 +1614,20 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	}
 
 	// Search for cached results.
+	rollupResultCacheV.rollupResultCacheRequests.Inc()
 	tssCached, start := rollupResultCacheV.GetSeries(qt, ec, expr, window)
 	ec.QueryStats.addSeriesFetched(len(tssCached))
 	if start > ec.End {
 		qt.Printf("the result is fully cached")
-		rollupResultCacheFullHits.Inc()
+		rollupResultCacheV.rollupResultCacheFullHits.Inc()
 		return tssCached, nil
 	}
 	if start > ec.Start {
 		qt.Printf("partial cache hit")
-		rollupResultCachePartialHits.Inc()
+		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 	} else {
 		qt.Printf("cache miss")
-		rollupResultCacheMiss.Inc()
+		rollupResultCacheV.rollupResultCacheMisses.Inc()
 	}
 
 	// Fetch missing results, which aren't cached yet.
@@ -1720,7 +1668,8 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 //
 // pointsPerSeries is used only for estimating the needed memory for query processing
 func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64,
+) ([]*timeseries, error) {
 	if qt.Enabled() {
 		qt = qt.NewChild("rollup %s: timeRange=%s, step=%d, window=%d", expr.AppendString(nil), ec.timeRangeString(), ec.Step, window)
 		defer qt.Done()
@@ -1855,7 +1804,8 @@ func maxSilenceInterval() int64 {
 
 func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
 	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs)
 	defer qt.Done()
 	var samplesScannedTotal atomic.Uint64
@@ -1894,7 +1844,8 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 }
 
 func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
+) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
 
@@ -1934,7 +1885,8 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 }
 
 func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName,
-	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) uint64 {
+	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64,
+) uint64 {
 	tsDst.MetricName.CopyFrom(mnSrc)
 	if len(rc.TagValue) > 0 {
 		tsDst.MetricName.AddTag("rollup", rc.TagValue)

app/vmselect/promql/rollup.go

@@ -874,17 +874,17 @@ func getScrapeInterval(timestamps []int64, defaultInterval int64) int64 {
 		return defaultInterval
 	}
 
-	// Estimate scrape interval as 0.6 quantile for the first 20 intervals.
-	tsPrev := timestamps[0]
-	timestamps = timestamps[1:]
+	// Estimate scrape interval as 0.6 quantile of the last 20 intervals.
+	tsPrev := timestamps[len(timestamps)-1]
+	timestamps = timestamps[:len(timestamps)-1]
 	if len(timestamps) > 20 {
-		timestamps = timestamps[:20]
+		timestamps = timestamps[len(timestamps)-20:]
 	}
 	a := getFloat64s()
 	intervals := a.A[:0]
-	for _, ts := range timestamps {
-		intervals = append(intervals, float64(ts-tsPrev))
-		tsPrev = ts
+	for i := len(timestamps) - 1; i >= 0; i-- {
+		intervals = append(intervals, float64(tsPrev-timestamps[i]))
+		tsPrev = timestamps[i]
 	}
 	scrapeInterval := int64(quantile(0.6, intervals))
 	a.A = intervals

app/vmselect/promql/rollup_result_cache.go

@@ -80,7 +80,7 @@ func InitRollupResultCache(cachePath string) {
 		c = workingsetcache.New(cacheSize)
 		rollupResultCacheKeyPrefix.Store(newRollupResultCacheKeyPrefix())
 	}
-	if *disableCache {
+	if *disableCache && len(rollupResultCachePath) > 0 && !*resetRollupResultCacheOnStartup {
 		c.Reset()
 	}
 
@@ -126,6 +126,12 @@ func InitRollupResultCache(cachePath string) {
 
 	rollupResultCacheV = &rollupResultCache{
 		c: c,
+
+		rollupResultCacheRequests:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_requests_total`),
+		rollupResultCacheFullHits:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_full_hits_total`),
+		rollupResultCachePartialHits: metrics.GetOrCreateCounter(`vm_rollup_result_cache_partial_hits_total`),
+		rollupResultCacheMisses:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_miss_total`),
+		rollupResultCacheResets:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_resets_total`),
 	}
 }
 
@@ -143,13 +149,18 @@ func StopRollupResultCache() {
 // instances in the cluster.
 type rollupResultCache struct {
 	c *workingsetcache.Cache
-}
 
-var rollupResultCacheResets = metrics.NewCounter(`vm_cache_resets_total{type="promql/rollupResult"}`)
+	rollupResultCacheRequests    *metrics.Counter
+	rollupResultCacheFullHits    *metrics.Counter
+	rollupResultCachePartialHits *metrics.Counter
+	rollupResultCacheMisses      *metrics.Counter
+
+	rollupResultCacheResets *metrics.Counter
+}
 
 // ResetRollupResultCache resets rollup result cache.
 func ResetRollupResultCache() {
-	rollupResultCacheResets.Inc()
+	rollupResultCacheV.rollupResultCacheResets.Inc()
 	rollupResultCacheKeyPrefix.Add(1)
 	logger.Infof("rollupResult cache has been cleared")
 }

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-CpXmH91b.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-D5YL0cqB.js">
+  <script type="module" crossorigin src="./assets/index-B6lol36n.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C0BE-BnM.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-VQRcNK83.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -32,7 +32,8 @@ import (
 )
 
 var (
-	retentionPeriod  = flagutil.NewRetentionDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
+	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
 	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the given -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
@@ -128,7 +129,7 @@ func main() {
 
 	storage.SetDedupInterval(*minScrapeInterval)
 	storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
-	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
+	storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)
 	storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
 	storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
@@ -238,9 +239,9 @@ func newRequestHandler(strg *storage.Storage) httpserver.RequestHandler {
 				return false
 			}
 			w.Header().Add("Content-Type", "text/html; charset=utf-8")
-			fmt.Fprintf(w, `vmstorage - a component of VictoriaMetrics cluster<br/>
-			<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/">docs</a><br>
-`)
+			fmt.Fprintf(w, `vmstorage - a component of VictoriaMetrics cluster<br/>`)
+			fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
+			fmt.Fprintf(w, `<a href="https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/">docs</a><br>`)
 			return true
 		}
 		return requestHandler(w, r, strg)
@@ -297,7 +298,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 		fmt.Fprintf(w, `{"status":"success","data":{"logEndTime":%q}}`, time.Unix(int64(endTime), 0))
 		return true
 	}
-	if !strings.HasPrefix(path, "/snapshot") {
+	if !strings.HasPrefix(path, "/snapshot/") {
 		return false
 	}
 	if !httpserver.CheckAuthFlag(w, r, snapshotAuthKey) {
@@ -309,8 +310,20 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotPath := strg.MustCreateSnapshot()
-		fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotPath))
+		snapshotName := strg.MustCreateSnapshot()
+
+		// Verify whether the client already closed the connection.
+		// In this case it is better to drop the created snapshot, since the client isn't interested in it.
+		if err := r.Context().Err(); err != nil {
+			logger.Infof("deleting already created snapshot at %s because the client canceled the request", snapshotName)
+			if err := deleteSnapshot(strg, snapshotName); err != nil {
+				logger.Infof("cannot delete just created snapshot: %s", err)
+				return true
+			}
+			return true
+		}
+
+		fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotName))
 		return true
 	case "/list":
 		snapshotsListTotal.Inc()
@@ -329,22 +342,12 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 		snapshotsDeleteTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
-		snapshots := strg.MustListSnapshots()
-		for _, snName := range snapshots {
-			if snName == snapshotName {
-				if err := strg.DeleteSnapshot(snName); err != nil {
-					err = fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
-					jsonResponseError(w, err)
-					snapshotsDeleteErrorsTotal.Inc()
-					return true
-				}
-				fmt.Fprintf(w, `{"status":"ok"}`)
-				return true
-			}
+		if err := deleteSnapshot(strg, snapshotName); err != nil {
+			jsonResponseError(w, err)
+			snapshotsDeleteErrorsTotal.Inc()
+			return true
 		}
-
-		err := fmt.Errorf("cannot find snapshot %q", snapshotName)
-		jsonResponseError(w, err)
+		fmt.Fprintf(w, `{"status":"ok"}`)
 		return true
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
@@ -365,6 +368,19 @@ func requestHandler(w http.ResponseWriter, r *http.Request, strg *storage.Storag
 	}
 }
 
+func deleteSnapshot(strg *storage.Storage, snapshotName string) error {
+	snapshots := strg.MustListSnapshots()
+	for _, snName := range snapshots {
+		if snName == snapshotName {
+			if err := strg.DeleteSnapshot(snName); err != nil {
+				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("cannot find snapshot %q", snapshotName)
+}
+
 func initStaleSnapshotsRemover(strg *storage.Storage) {
 	staleSnapshotsRemoverCh = make(chan struct{})
 	if snapshotsMaxAge.Duration() <= 0 {
@@ -416,7 +432,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
 	tm := &m.TableMetrics
-	idbm := &m.IndexDBMetrics
+	idbm := &m.TableMetrics.IndexDBMetrics
 
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_free_disk_space_limit_bytes{path=%q}`, *storageDataPath), uint64(minFreeDiskSpaceBytes.N))
@@ -544,75 +560,82 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 
 	metrics.WriteCounterUint64(w, `vm_missing_metric_names_for_metric_id_total`, idbm.MissingMetricNamesForMetricID)
 
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_syncs_total`, idbm.DateMetricIDCacheSyncsCount)
-	metrics.WriteCounterUint64(w, `vm_date_metric_id_cache_resets_total`, idbm.DateMetricIDCacheResetsCount)
-
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/tsid"}`, m.TSIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricIDs"}`, m.MetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricName"}`, m.MetricNameCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/metricID"}`, idbm.MetricIDCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/tsid"}`, m.TSIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, storage.RegexpCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/metricID"}`, idbm.MetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, uint64(storage.RegexpCacheSizeBytes()))
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSizeBytes()))
 
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/tsid"}`, m.TSIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricIDs"}`, m.MetricIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/metricName"}`, m.MetricNameCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, storage.RegexpCacheMaxSizeBytes())
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMaxSizeBytes())
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, uint64(storage.RegexpCacheMaxSizeBytes()))
-	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheMaxSizeBytes()))
 
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/tsid"}`, m.TSIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricIDs"}`, m.MetricIDCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/metricName"}`, m.MetricNameCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheRequests)
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexps"}`, storage.RegexpCacheRequests())
-	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheRequests())
 
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/tsid"}`, m.TSIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricIDs"}`, m.MetricIDCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/metricName"}`, m.MetricNameCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
-	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
 
-	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_resets_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheResets)
 
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/tsid"}`, m.TSIDCacheCollisions)
 	metrics.WriteCounterUint64(w, `vm_cache_collisions_total{type="storage/metricName"}`, m.MetricNameCacheCollisions)
 
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/metricID"}`, idbm.MetricIDCacheSyncsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_syncs_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSyncsCount)
+
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/metricID"}`, idbm.MetricIDCacheRotationsCount)
+	metrics.WriteCounterUint64(w, `vm_cache_rotations_total{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheRotationsCount)
+
+	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
+
 	metrics.WriteGaugeUint64(w, `vm_next_retention_seconds`, m.NextRetentionSeconds)
 
 	if *trackMetricNamesStats {

app/vmstorage/servers/vminsert.go

@@ -14,9 +14,10 @@ import (
 var (
 	precisionBits = flag.Int("precisionBits", 64, "The number of precision bits to store per each value. Lower precision bits improves data compression "+
 		"at the cost of precision loss")
-	vminsertConnsShutdownDuration = flag.Duration("storage.vminsertConnsShutdownDuration", 25*time.Second, "The time needed for gradual closing of vminsert connections during "+
+	vminsertConnsShutdownDuration = flag.Duration("storage.vminsertConnsShutdownDuration", 10*time.Second, "The time needed for gradual closing of vminsert connections during "+
 		"graceful shutdown. Bigger duration reduces spikes in CPU, RAM and disk IO load on the remaining vmstorage nodes during rolling restart. "+
 		"Smaller duration reduces the time needed to close all the vminsert connections, thus reducing the time for graceful shutdown. "+
+		"Configured value must always be lower than the graceful shutdown period configured by the orchestration platform (terminationGracePeriodSeconds for Kubernetes). "+
 		"See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#improving-re-routing-performance-during-restart")
 )
 

app/vmstorage/servers/vmselect.go

@@ -245,12 +245,15 @@ func (api *vmstorageAPI) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
 	sr storage.Search
+	mb storage.MetricBlock
 }
 
 var blockIteratorsPool sync.Pool
 
 func (bi *blockIterator) MustClose() {
 	bi.sr.MustClose()
+	bi.mb.MetricName = nil
+	bi.mb.Block.Reset()
 	blockIteratorsPool.Put(bi)
 }
 
@@ -262,13 +265,15 @@ func getBlockIterator() *blockIterator {
 	return v.(*blockIterator)
 }
 
-func (bi *blockIterator) NextBlock(mb *storage.MetricBlock) bool {
+func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
 	if !bi.sr.NextMetricBlock() {
-		return false
+		return dst, false
 	}
-	mb.MetricName = append(mb.MetricName[:0], bi.sr.MetricBlockRef.MetricName...)
+	mb := bi.mb
+	mb.MetricName = bi.sr.MetricBlockRef.MetricName
 	bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
-	return true
+	dst = mb.Marshal(dst[:0])
+	return dst, true
 }
 
 func (bi *blockIterator) Error() error {

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.4 AS build-web-stage
+FROM golang:1.25.6 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/Makefile

@@ -14,14 +14,6 @@ vmui-build: copy-metricsql-docs vmui-package-base-image
 		--entrypoint=/bin/bash \
 		vmui-builder-image -c "npm install && npm run build"
 
-vmui-anomaly-build: vmui-package-base-image
-	docker run --rm \
-		--user $(shell id -u):$(shell id -g) \
-		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
-		-w /build/packages/vmui \
-		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build:anomaly"
-
 vmui-release: vmui-build
 	docker build -t ${DOCKER_NAMESPACE}/vmui:latest -f app/vmui/Dockerfile-web ./app/vmui/packages/vmui
 	docker tag ${DOCKER_NAMESPACE}/vmui:latest ${DOCKER_NAMESPACE}/vmui:${PKG_TAG}

app/vmui/packages/vmui/.env.vmanomaly

@@ -1 +0,0 @@
-VITE_APP_TYPE=vmanomaly

app/vmui/packages/vmui/config/plugins/dynamicIndexHtml.ts

@@ -1,23 +0,0 @@
-import { readFile } from "fs/promises";
-import { IndexHtmlTransform } from "vite";
-
-/**
- * Vite plugin to dynamically load index.html based on the current mode.
- * If a specific mode-based index file (e.g., index.vmanomaly.html) exists, it is used.
- * Otherwise, the default index.html is loaded.
- */
-export default function dynamicIndexHtmlPlugin({ mode }) {
-  return {
-    name: "vm-dynamic-index-html",
-    transformIndexHtml: {
-      order: "pre",
-      handler: async () => {
-        try {
-          return await readFile(`./index.${mode}.html`, "utf8");
-        } catch (error) {
-          return await readFile("./index.html", "utf8");
-        }
-      }
-    } as IndexHtmlTransform
-  };
-}

app/vmui/packages/vmui/eslint.config.js

@@ -46,7 +46,7 @@ export default [...compat.extends(
   settings: {
     react: {
       pragma: "React",
-      version: "detect",
+      version: "19.0",
     },
 
     linkComponents: ["Hyperlink", {
@@ -69,10 +69,11 @@ export default [...compat.extends(
       "varsIgnorePattern": "^_",
       "ignoreRestSiblings": true
     }],
-    
+
     "unused-imports/no-unused-imports": "error",
 
     "react/jsx-closing-bracket-location": [1, "line-aligned"],
+    "object-curly-spacing": [2, "always"],
 
     "react/jsx-max-props-per-line": [1, {
       maximum: 1,
@@ -81,13 +82,23 @@ export default [...compat.extends(
     "react/jsx-first-prop-new-line": [1, "multiline"],
 
     // Disable core indent rule due to recursion issues in ESLint 9; use JSX-specific rules instead
-    indent: "off",
+    indent: ["error", 2, {
+      SwitchCase: 1,
+      ignoredNodes: [
+        "JSXElement",
+        "JSXElement *",
+        "JSXFragment",
+        "JSXFragment *",
+      ],
+    }],
     "react/jsx-indent": ["error", 2],
     "react/jsx-indent-props": ["error", 2],
 
+    "linebreak-style": ["error", "unix"],
+    quotes: ["error", "double"],
+    semi: ["error", "always"],
     // Formatting rules moved out of ESLint core; omit here to avoid deprecation noise
     "react/prop-types": 0,
     "react/react-in-jsx-scope": "off",
-
   },
 }];

app/vmui/packages/vmui/index.vmanomaly.html

@@ -1,54 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="/favicon.svg" />
-  <link rel="apple-touch-icon" href="/favicon.svg" />
-  <link rel="mask-icon" href="/favicon.svg" color="#000000">
-
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta name="description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI"/>
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
-  <!--
-    Notice the use of in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>UI for VictoriaMetrics Anomaly Detection</title>
-
-  <meta name="twitter:card" content="summary">
-  <meta name="twitter:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta name="twitter:site" content="@https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta name="twitter:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-  <meta name="twitter:image" content="/preview.jpg">
-
-  <meta property="og:type" content="website">
-  <meta property="og:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta property="og:url" content="https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta property="og:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-<script type="module" src="/src/index.tsx"></script>
-</body>
-</html>

app/vmui/packages/vmui/package-lock.json

@@ -5194,9 +5194,9 @@
       "license": "MIT"
     },
     "node_modules/js-yaml": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
-      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
+      "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {

app/vmui/packages/vmui/package.json

@@ -7,10 +7,8 @@
   "scripts": {
     "prestart": "npm run copy-metricsql-docs",
     "start": "vite",
-    "start:playground": "cross-env PLAYGROUND=METRICS npm run start",
-    "start:anomaly": "vite --mode vmanomaly",
+    "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "build:anomaly": "vite build --mode vmanomaly",
     "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
@@ -18,7 +16,8 @@
     "preview": "vite preview",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:dev": "vitest"
+    "test:dev": "vitest",
+    "precommit": "npm run lint:local && npm run typecheck && npm run test"
   },
   "dependencies": {
     "classnames": "^2.5.1",

app/vmui/packages/vmui/src/AppAnomaly.tsx

@@ -1,41 +0,0 @@
-import { FC, useState } from "preact/compat";
-import { HashRouter, Route, Routes } from "react-router-dom";
-import AppContextProvider from "./contexts/AppContextProvider";
-import ThemeProvider from "./components/Main/ThemeProvider/ThemeProvider";
-import AnomalyLayout from "./layouts/AnomalyLayout/AnomalyLayout";
-import ExploreAnomaly from "./pages/ExploreAnomaly/ExploreAnomaly";
-import router from "./router";
-import CustomPanel from "./pages/CustomPanel";
-
-const AppAnomaly: FC = () => {
-  const [loadedTheme, setLoadedTheme] = useState(false);
-
-  return <>
-    <HashRouter>
-      <AppContextProvider>
-        <>
-          <ThemeProvider onLoaded={setLoadedTheme}/>
-          {loadedTheme && (
-            <Routes>
-              <Route
-                path={"/"}
-                element={<AnomalyLayout/>}
-              >
-                <Route
-                  path={"/"}
-                  element={<ExploreAnomaly/>}
-                />
-                <Route
-                  path={router.query}
-                  element={<CustomPanel/>}
-                />
-              </Route>
-            </Routes>
-          )}
-        </>
-      </AppContextProvider>
-    </HashRouter>
-  </>;
-};
-
-export default AppAnomaly;

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -20,6 +20,7 @@ export interface ChartTooltipProps {
   info?: ReactNode;
   marker?: string;
   show?: boolean;
+  duplicateCount?: number;
   onClose?: (id: string) => void;
 }
 
@@ -35,6 +36,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
   statsFormatted,
   isSticky,
   marker,
+  duplicateCount = 0,
   onClose
 }) => {
   const tooltipRef = useRef<HTMLDivElement>(null);
@@ -156,6 +158,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
         <p className="vm-chart-tooltip-data__value">
           <b>{value}</b>{unit}
         </p>
+        {duplicateCount > 1 && <p>(overlapping points: {duplicateCount})</p>}
       </div>
       {statsFormatted && (
         <table className="vm-chart-tooltip-stats">

app/vmui/packages/vmui/src/components/Chart/Line/Legend/Legend.tsx

@@ -14,12 +14,11 @@ export type QueryGroup = {
 interface LegendProps {
   labels: LegendItemType[];
   query: string[];
-  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
 
-const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPanel, onChange }) => {
+const Legend: FC<LegendProps> = ({ labels, query, isPredefinedPanel, onChange }) => {
   const { groupByLabel } = useLegendGroup();
   const groupSeries = useGroupSeries({ labels, query, groupByLabel });
 
@@ -33,7 +32,6 @@ const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPan
             key={group}
             labels={items}
             group={group}
-            isAnomalyView={isAnomalyView}
             onChange={onChange}
           />
         ))}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -8,11 +8,11 @@ import { useHideDuplicateFields } from "./hooks/useHideDuplicateFields";
 import Accordion from "../../../Main/Accordion/Accordion";
 import { useLegendGroup } from "./hooks/useLegendGroup";
 import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
-import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
+import { LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import { getFromStorage } from "../../../../utils/storage";
 
 export type LegendProps = {
   labels: LegendItemType[];
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
@@ -21,7 +21,7 @@ interface LegendGroupProps extends LegendProps {
   group: string | number;
 }
 
-const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onChange }) => {
+const LegendGroup: FC<LegendGroupProps> = ({ labels, group, onChange }) => {
   const { isTableView } = useLegendView();
   const { groupByLabel } = useLegendGroup();
   const copyToClipboard = useCopyToClipboard();
@@ -38,17 +38,26 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
 
   const Content = isTableView ? LegendTable : LegendLines;
 
+  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false";
+  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT;
+
+  const expandedWarning = (
+    <span className="vm-legend-group-header__warning">
+      Legend collapsed by default ({sortedLabels.length} series) — click to expand.
+    </span>
+  );
+
   return (
     <div
       className="vm-legend-group"
       key={group}
     >
       <Accordion
-        defaultExpanded={sortedLabels.length < DEFAULT_MAX_SERIES.chart}
+        defaultExpanded={defaultExpanded}
         title={(
           <div className="vm-legend-group-header">
             <div className="vm-legend-group-header-title">
-              Group by{groupByLabel ? "" : " query"}: <b>{group}</b>
+              Group by{groupByLabel ? "" : " query"}: <b>{group}</b> {!defaultExpanded && expandedWarning}
             </div>
             {!!duplicateFields.length && (
               <div className="vm-legend-group-header-labels">
@@ -71,7 +80,6 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
       >
         <Content
           labels={sortedLabels}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -13,11 +13,10 @@ import { getLabelAlias } from "../../../../../utils/metric";
 interface LegendItemProps {
   legend: LegendItemType;
   onChange?: (item: LegendItemType, metaKey: boolean) => void;
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
 }
 
-const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, isAnomalyView }) => {
+const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields }) => {
   const copyToClipboard = useCopyToClipboard();
   const { hideStats } = useShowStats();
 
@@ -52,12 +51,10 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, is
       })}
       onClick={createHandlerClick(legend)}
     >
-      {!isAnomalyView && (
-        <div
-          className="vm-legend-item__marker"
-          style={{ backgroundColor: legend.color }}
-        />
-      )}
+      <div
+        className="vm-legend-item__marker"
+        style={{ backgroundColor: legend.color }}
+      />
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
           {legend.hasAlias && legend.label}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendViews/LegendLines.tsx

@@ -2,7 +2,7 @@ import { FC } from "preact/compat";
 import LegendItem from "../LegendItem/LegendItem";
 import { LegendProps } from "../LegendGroup";
 
-const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields, onChange }) => {
+const LegendLines: FC<LegendProps> = ({ labels, duplicateFields, onChange }) => {
 
   return (
     <div className="vm-legend-item-container">
@@ -10,7 +10,6 @@ const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields,
         <LegendItem
           key={legendItem.label}
           legend={legendItem}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/Legend/style.scss

@@ -32,6 +32,14 @@
         }
       }
 
+      &__warning {
+        flex-grow: 1;
+        text-align: right;
+        padding-right: calc($padding-large * 2);
+        font-size: $font-size-small;
+        color: $color-warning;
+      }
+
       &-labels {
         display: flex;
         flex-wrap: wrap;

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/LegendAnomaly.tsx

@@ -1,82 +0,0 @@
-import { FC, useMemo } from "preact/compat";
-import { ForecastType, SeriesItem } from "../../../../types";
-import { anomalyColors } from "../../../../utils/color";
-import "./style.scss";
-
-type Props = {
-  series: SeriesItem[];
-};
-
-const titles: Partial<Record<ForecastType, string>> = {
-  [ForecastType.yhat]: "yhat",
-  [ForecastType.yhatLower]: "yhat_upper - yhat_lower",
-  [ForecastType.yhatUpper]: "yhat_upper - yhat_lower",
-  [ForecastType.anomaly]: "anomalies",
-  [ForecastType.training]: "training data",
-  [ForecastType.actual]: "y"
-};
-
-const LegendAnomaly: FC<Props> = ({ series }) => {
-
-  const uniqSeriesStyles = useMemo(() => {
-    const uniqSeries = series.reduce((accumulator, currentSeries) => {
-      const hasForecast = Object.prototype.hasOwnProperty.call(currentSeries, "forecast");
-      const isNotUpper = currentSeries.forecast !== ForecastType.yhatUpper;
-      const isUniqForecast = !accumulator.find(s => s.forecast === currentSeries.forecast);
-      if (hasForecast && isUniqForecast && isNotUpper) {
-        accumulator.push(currentSeries);
-      }
-      return accumulator;
-    }, [] as SeriesItem[]);
-
-    const trainingSeries = {
-      ...uniqSeries[0],
-      forecast: ForecastType.training,
-      color: anomalyColors[ForecastType.training],
-    };
-    uniqSeries.splice(1, 0, trainingSeries);
-
-    return uniqSeries.map(s => ({
-      ...s,
-      color: typeof s.stroke === "string" ? s.stroke : anomalyColors[s.forecast || ForecastType.actual],
-    }));
-  }, [series]);
-
-  return <>
-    <div className="vm-legend-anomaly">
-      {/* TODO: remove .filter() after the correct training data has been added */}
-      {uniqSeriesStyles.filter(f => f.forecast !== ForecastType.training).map((s, i) => (
-        <div
-          key={`${i}_${s.forecast}`}
-          className="vm-legend-anomaly-item"
-        >
-          <svg>
-            {s.forecast === ForecastType.anomaly ? (
-              <circle
-                cx="15"
-                cy="7"
-                r="4"
-                fill={s.color}
-                stroke={s.color}
-                strokeWidth="1.4"
-              />
-            ) : (
-              <line
-                x1="0"
-                y1="7"
-                x2="30"
-                y2="7"
-                stroke={s.color}
-                strokeWidth={s.width || 1}
-                strokeDasharray={s.dash?.join(",")}
-              />
-            )}
-          </svg>
-          <div className="vm-legend-anomaly-item__title">{titles[s.forecast || ForecastType.actual]}</div>
-        </div>
-      ))}
-    </div>
-  </>;
-};
-
-export default LegendAnomaly;

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/style.scss

@@ -1,23 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-legend-anomaly {
-  position: relative;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  flex-wrap: wrap;
-  gap: calc($padding-large * 2);
-  cursor: default;
-
-  &-item {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    gap: $padding-small;
-
-    svg {
-      width: 30px;
-      height: 14px;
-    }
-  }
-}

app/vmui/packages/vmui/src/components/Chart/Line/LineChart/LineChart.tsx

@@ -13,7 +13,6 @@ import {
   getRangeY,
   getScales,
   handleDestroy,
-  setBand,
   setSelect
 } from "../../../../utils/uplot";
 import { MetricResult } from "../../../../api/types";
@@ -40,7 +39,6 @@ export interface LineChartProps {
   setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
   layoutSize: ElementSize;
   height?: number;
-  isAnomalyView?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
 }
@@ -55,7 +53,6 @@ const LineChart: FC<LineChartProps> = ({
   setPeriod,
   layoutSize,
   height,
-  isAnomalyView,
   spanGaps = false,
   showAllPoints = false,
 }) => {
@@ -75,7 +72,7 @@ const LineChart: FC<LineChartProps> = ({
     seriesFocus,
     setCursor,
     resetTooltips
-  } = useLineTooltip({ u: uPlotInst, metrics, series, unit, isAnomalyView });
+  } = useLineTooltip({ u: uPlotInst, metrics, series, unit });
 
   const options: uPlotOptions = {
     ...getDefaultOptions({ width: layoutSize.width, height }),
@@ -111,7 +108,6 @@ const LineChart: FC<LineChartProps> = ({
     if (!uPlotInst) return;
     delSeries(uPlotInst);
     addSeries(uPlotInst, series, spanGaps, showAllPoints);
-    setBand(uPlotInst, series);
     uPlotInst.redraw();
   }, [series, spanGaps, showAllPoints]);
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -1,15 +1,17 @@
-import { forwardRef, useCallback, useImperativeHandle, useState } from "preact/compat";
+import { forwardRef, useCallback, useEffect, useImperativeHandle, useState } from "preact/compat";
 import { DisplayType, ErrorTypes } from "../../../../types";
 import TextField from "../../../Main/TextField/TextField";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import { InfoIcon, RestartIcon } from "../../../Main/Icons";
 import Button from "../../../Main/Button/Button";
-import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
+import { DEFAULT_MAX_SERIES, LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../../state/customPanel/CustomPanelStateContext";
+import Switch from "../../../Main/Switch/Switch";
+import { getFromStorage, saveToStorage } from "../../../../utils/storage";
 
 interface ServerConfiguratorProps {
   onClose: () => void
@@ -27,6 +29,9 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   const { seriesLimits } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
+  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
+  const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
+
   const [limits, setLimits] = useState(seriesLimits);
   const [error, setError] = useState({
     table: "",
@@ -52,6 +57,10 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     onClose();
   }, [limits]);
 
+  useEffect(() => {
+    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
+  }, [legendCollapse]);
+
   useImperativeHandle(ref, () => ({ handleApply }), [handleApply]);
 
   return (
@@ -97,6 +106,19 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
           </div>
         ))}
       </div>
+
+      <div className="vm-graph-settings-row">
+        <span className="vm-graph-settings-row__label">Auto-collapse legend</span>
+        <Switch
+          value={legendCollapse}
+          onChange={setLegendCollapse}
+          label={legendCollapse ? "Enabled" : "Disabled"}
+          fullWidth={isMobile}
+        />
+        <span className="vm-legend-configs-item__info">
+          Collapses the legend when series count exceeds {LEGEND_COLLAPSE_SERIES_LIMIT} to reduce UI load.
+        </span>
+      </div>
     </div>
   );
 });

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/style.scss

@@ -18,6 +18,7 @@
     align-items: center;
     justify-content: space-between;
     gap: $padding-global;
+    margin-bottom: $padding-global;
 
     &_mobile {
       gap: $padding-small;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/ServerConfigurator/ServerConfigurator.tsx

@@ -9,7 +9,6 @@ import { getFromStorage, removeFromStorage, saveToStorage } from "../../../../ut
 import useBoolean from "../../../../hooks/useBoolean";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
-import { getTenantIdFromUrl } from "../../../../utils/tenants";
 
 interface ServerConfiguratorProps {
   onClose: () => void;
@@ -39,10 +38,6 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   };
 
   const handleApply = useCallback(() => {
-    const tenantIdFromUrl = getTenantIdFromUrl(serverUrl);
-    if (tenantIdFromUrl !== "") {
-      dispatch({ type: "SET_TENANT_ID", payload: tenantIdFromUrl });
-    }
     dispatch({ type: "SET_SERVER", payload: serverUrl });
     onClose();
   }, [serverUrl]);
@@ -60,12 +55,6 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     }
   }, [enabledStorage]);
 
-  useEffect(() => {
-    if (enabledStorage) {
-      saveToStorage("SERVER_URL", serverUrl);
-    }
-  }, [serverUrl]);
-
   useEffect(() => {
     // the tenant selector can change the serverUrl
     if (stateServerUrl === serverUrl) return;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration.tsx

@@ -1,4 +1,4 @@
-import { FC, useState, useRef, useEffect, useMemo } from "preact/compat";
+import { FC, useState, useRef, useMemo } from "preact/compat";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { ArrowDownIcon, StorageIcon } from "../../../Main/Icons";
@@ -10,14 +10,14 @@ import { getAppModeEnable } from "../../../../utils/app-mode";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import TextField from "../../../Main/TextField/TextField";
-import { getTenantIdFromUrl, replaceTenantId } from "../../../../utils/tenants";
+import { replaceTenantId } from "../../../../utils/tenants";
 import useBoolean from "../../../../hooks/useBoolean";
 
 const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   const appModeEnable = getAppModeEnable();
   const { isMobile } = useDeviceDetect();
 
-  const { tenantId: tenantIdState, serverUrl } = useAppState();
+  const { tenantId, serverUrl } = useAppState();
   const dispatch = useAppDispatch();
   const timeDispatch = useTimeDispatch();
 
@@ -48,10 +48,8 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   }, [accountIds]);
 
   const createHandlerChange = (value: string) => () => {
-    const tenant = value;
-    dispatch({ type: "SET_TENANT_ID", payload: tenant });
     if (serverUrl) {
-      const updateServerUrl = replaceTenantId(serverUrl, tenant);
+      const updateServerUrl = replaceTenantId(serverUrl, value);
       if (updateServerUrl === serverUrl) return;
       dispatch({ type: "SET_SERVER", payload: updateServerUrl });
       timeDispatch({ type: "RUN_QUERY" });
@@ -59,16 +57,6 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
     handleCloseOptions();
   };
 
-  useEffect(() => {
-    const id = getTenantIdFromUrl(serverUrl);
-
-    if (tenantIdState && tenantIdState !== id) {
-      createHandlerChange(tenantIdState)();
-    } else {
-      createHandlerChange(id)();
-    }
-  }, [serverUrl]);
-
   if (!showTenantSelector) return null;
 
   return (
@@ -83,7 +71,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               <span className="vm-mobile-option__icon"><StorageIcon/></span>
               <div className="vm-mobile-option-text">
                 <span className="vm-mobile-option-text__label">Tenant ID</span>
-                <span className="vm-mobile-option-text__value">{tenantIdState}</span>
+                <span className="vm-mobile-option-text__value">{tenantId}</span>
               </div>
               <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
             </div>
@@ -106,7 +94,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               )}
               onClick={toggleOpenOptions}
             >
-              {tenantIdState}
+              {tenantId}
             </Button>
           )}
         </div>
@@ -138,7 +126,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               className={classNames({
                 "vm-list-item": true,
                 "vm-list-item_mobile": isMobile,
-                "vm-list-item_active": id === tenantIdState
+                "vm-list-item_active": id === tenantId
               })}
               key={id}
               onClick={createHandlerChange(id)}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/hooks/useFetchAccountIds.ts

@@ -3,19 +3,18 @@ import { useEffect, useMemo, useState } from "preact/compat";
 import { ErrorTypes } from "../../../../../types";
 import { getAccountIds } from "../../../../../api/accountId";
 import { getAppModeEnable, getAppModeParams } from "../../../../../utils/app-mode";
-import { getTenantIdFromUrl } from "../../../../../utils/tenants";
 
 export const useFetchAccountIds = () => {
   const { useTenantID } = getAppModeParams();
   const appModeEnable = getAppModeEnable();
-  const { serverUrl } = useAppState();
+  const { tenantId, serverUrl } = useAppState();
 
   const [isLoading, setIsLoading] = useState(false);
   const [error, setError] = useState<ErrorTypes | string>();
   const [accountIds, setAccountIds] = useState<string[]>([]);
 
   const fetchUrl = useMemo(() => getAccountIds(serverUrl), [serverUrl]);
-  const isServerUrlWithTenant = useMemo(() => !!getTenantIdFromUrl(serverUrl), [serverUrl]);
+  const isServerUrlWithTenant = useMemo(() => !!tenantId, [tenantId]);
   const preventFetch = appModeEnable ? !useTenantID : !isServerUrlWithTenant;
 
   useEffect(() => {

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/index.tsx

@@ -154,7 +154,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
       {!!item?.alerts?.length && (
         <>
           <span className="vm-alerts-title">Alerts</span>
-          <table>
+          <table className="vm-alerts-table">
             <colgroup>
               <col className="vm-col-sm"/>
               <col className="vm-col-sm"/>
@@ -190,7 +190,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
                   </td>
                   <td>
                     <Badges
-                      align="center"
+                      align="start"
                       items={Object.fromEntries(Object.entries(alert.labels || {}).map(([name, value]) => [name, {
                         color: "passive",
                         value: value,

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/style.scss

@@ -44,6 +44,7 @@
     word-break: break-word;
     table-layout: fixed;
     width: 100%;
+
     td, th {
       line-height: 30px;
       padding: 4px $padding-small;
@@ -52,15 +53,33 @@
       overflow: hidden;
       text-overflow: ellipsis;
     }
+
     th {
       white-space: nowrap;
     }
+
     td.align-center {
       text-align: center
     }
+
     th {
       font-weight: bold;
       padding: 0 $padding-small;
     }
   }
+
+  .vm-alerts-table {
+    tr {
+      border-bottom: $border-divider;
+
+      &:hover {
+        background: $color-background-hover;
+      }
+    }
+
+    td {
+      vertical-align: top;
+      padding-block: $padding-small;
+    }
+  }
 }

app/vmui/packages/vmui/src/components/ExploreAlerts/NotifiersHeader/style.scss

@@ -46,9 +46,6 @@
     .vm-text-field__input {
       padding: 11px 28px;
     }
-    .vm-text-field__icon-start {
-      height: 42px;
-    }
   }
 
   &__clear-icon {

app/vmui/packages/vmui/src/components/ExploreAlerts/Rule/style.scss

@@ -8,7 +8,7 @@
   flex-direction: column;
   position: relative;
 
-  &:has(>details[open]) {
+  &:has(>.vm-accordion-header_open) {
     background-color: $color-background-item;
   }
 

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/index.tsx

@@ -61,7 +61,7 @@ const RulesHeader: FC<RulesHeaderProps> = ({
             value={states}
             list={allStates}
             label="State"
-            placeholder="Please rule state"
+            placeholder="Please select rule state"
             onChange={onChangeStates}
             noOptionsText={noStateText}
             includeAll

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/style.scss

@@ -26,9 +26,6 @@
     .vm-text-field__input {
       padding: 11px 28px;
     }
-    .vm-text-field__icon-start {
-      height: 42px;
-    }
   }
 
   &__clear-icon {

app/vmui/packages/vmui/src/components/ExploreAlerts/Target/style.scss

@@ -34,7 +34,7 @@
   position: relative;
   border-radius: $border-radius-small;
 
-  &:has(>details[open]) {
+  &:has(>.vm-accordion-header_open) {
     background-color: $color-background-item;
   }
 

app/vmui/packages/vmui/src/components/ExploreAlerts/helpers.ts

@@ -17,4 +17,4 @@ export const formatDuration = (raw: number) => {
 export const formatEventTime = (raw: string) => {
   const t = dayjs(raw);
   return t.year() <= 1 ? "Never" : t.format("DD MMM YYYY HH:mm:ss");
-}
+};

app/vmui/packages/vmui/src/components/ExploreAnomaly/AnomalyConfig.tsx

@@ -1,132 +0,0 @@
-import { FC, useState } from "preact/compat";
-import Button from "../Main/Button/Button";
-import TextField from "../Main/TextField/TextField";
-import Modal from "../Main/Modal/Modal";
-import Spinner from "../Main/Spinner/Spinner";
-import { DownloadIcon, ErrorIcon } from "../Main/Icons";
-import useBoolean from "../../hooks/useBoolean";
-import useDeviceDetect from "../../hooks/useDeviceDetect";
-import { useAppState } from "../../state/common/StateContext";
-import classNames from "classnames";
-import "./style.scss";
-import { useQueryState } from "../../state/query/QueryStateContext";
-import { useTimeState } from "../../state/time/TimeStateContext";
-import { getStepFromDuration } from "../../utils/time";
-
-const AnomalyConfig: FC = () => {
-  const { serverUrl } = useAppState();
-  const { isMobile } = useDeviceDetect();
-
-  const {
-    value: isModalOpen,
-    setTrue: setOpenModal,
-    setFalse: setCloseModal,
-  } = useBoolean(false);
-
-  const { query } = useQueryState();
-  const { period } = useTimeState();
-  const [isLoading, setIsLoading] = useState(false);
-  const [textConfig, setTextConfig] = useState<string>("");
-  const [downloadUrl, setDownloadUrl] = useState<string>("");
-  const [error, setError] = useState<string>("");
-
-  const fetchConfig = async () => {
-    setIsLoading(true);
-    try {
-      const queryParam = encodeURIComponent(query[0] || "");
-      const stepParam = encodeURIComponent(period.step || getStepFromDuration(period.end - period.start, false));
-
-      const url = `${serverUrl}/api/vmanomaly/config.yaml?query=${queryParam}&step=${stepParam}`;
-      const response = await fetch(url);
-      const contentType = response.headers.get("Content-Type");
-      if (!response.ok) {
-        const bodyText = await response.text();
-        setError(` ${response.status} ${response.statusText}: ${bodyText}`);
-      } else if (contentType == "application/yaml") {
-        const blob = await response.blob();
-        const yamlAsString = await blob.text();
-        setTextConfig(yamlAsString);
-        setDownloadUrl(URL.createObjectURL(blob));
-      } else {
-        setError("Response Content-Type is not YAML, does `Server URL` point to VMAnomaly server?");
-      }
-    } catch (error) {
-      console.error(error);
-      setError(String(error));
-    }
-    setIsLoading(false);
-  };
-
-  const handleOpenModal = () => {
-    setOpenModal();
-    setError("");
-    URL.revokeObjectURL(downloadUrl);
-    setTextConfig("");
-    setDownloadUrl("");
-    return fetchConfig();
-  };
-
-  return (
-    <>
-      <Button
-        color="secondary"
-        variant="outlined"
-        onClick={handleOpenModal}
-      >
-        Open Config
-      </Button>
-      {isModalOpen && (
-        <Modal
-          title="Download config"
-          onClose={setCloseModal}
-        >
-          <div
-            className={classNames({
-              "vm-anomaly-config": true,
-              "vm-anomaly-config_mobile": isMobile,
-            })}
-          >
-            {isLoading && (
-              <Spinner
-                containerStyles={{ position: "relative" }}
-                message={"Loading config..."}
-              />
-            )}
-            {!isLoading && error && (
-              <div className="vm-anomaly-config-error">
-                <div className="vm-anomaly-config-error__icon"><ErrorIcon/></div>
-                <h3 className="vm-anomaly-config-error__title">Cannot download config</h3>
-                <p className="vm-anomaly-config-error__text">{error}</p>
-              </div>
-            )}
-            {!isLoading && textConfig && (
-              <TextField
-                value={textConfig}
-                label={"config.yaml"}
-                type="textarea"
-                disabled={true}
-              />
-            )}
-            <div className="vm-anomaly-config-footer">
-              {downloadUrl && (
-                <a
-                  href={downloadUrl}
-                  download={"config.yaml"}
-                >
-                  <Button
-                    variant="contained"
-                    startIcon={<DownloadIcon/>}
-                  >
-                    download
-                  </Button>
-                </a>
-              )}
-            </div>
-          </div>
-        </Modal>
-      )}
-    </>
-  );
-};
-
-export default AnomalyConfig;

app/vmui/packages/vmui/src/components/ExploreAnomaly/style.scss

@@ -1,61 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-anomaly-config {
-  display: grid;
-  grid-template-rows: calc(($vh * 70) - 78px - ($padding-medium*3)) auto;
-  gap: $padding-global;
-  min-width: 400px;
-  max-width: 80vw;
-  min-height: 300px;
-
-  &_mobile {
-    width: 100%;
-    max-width: none;
-    min-height: 100%;
-    grid-template-rows: calc(($vh * 100) - 78px - ($padding-global*3)) auto;
-  }
-
-  textarea {
-    overflow: auto;
-    width: 100%;
-    height: 100%;
-    max-height: 900px;
-  }
-
-  &-error {
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    justify-content: center;
-    width: 100%;
-    gap: $padding-small;
-    text-align: center;
-
-    &__icon {
-      display: flex;
-      align-items: center;
-      justify-content: center;
-      width: 30px;
-      height: 30px;
-      margin-bottom: $padding-small;
-      color: $color-error;
-    }
-
-    &__title {
-      font-size: $font-size-medium;
-      font-weight: bold;
-    }
-
-    &__text {
-      max-width: 700px;
-      line-height: 1.3;
-    }
-  }
-
-  &-footer {
-    display: flex;
-    align-items: center;
-    justify-content: flex-end;
-    gap: $padding-small;
-  }
-}

app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricGraph/ExploreMetricItemGraph.tsx

@@ -43,7 +43,7 @@ const ExploreMetricItem: FC<ExploreMetricItemGraphProps> = ({
   const step = isHeatmap && customStep === defaultStep ? heatmapStep : customStep;
 
 
-  const query = useMemo(() => {
+  const queries = useMemo(() => {
     const params = Object.entries({ job, instance })
       .filter(val => val[1])
       .map(([key, val]) => `${key}=${JSON.stringify(val)}`);
@@ -55,19 +55,19 @@ const ExploreMetricItem: FC<ExploreMetricItemGraphProps> = ({
 
     const base = `{${params.join(",")}}`;
     if (isBucket) {
-      return `sum(rate(${base})) by (vmrange, le)`;
+      return [`sum(rate(${base})) by (vmrange, le)`];
     }
     const queryBase = rateEnabled ? `rollup_rate(${base})` : `rollup(${base})`;
-    return `
+    return [`
 with (q = ${queryBase}) (
   alias(min(label_match(q, "rollup", "min")), "min"),
   alias(max(label_match(q, "rollup", "max")), "max"),
   alias(avg(label_match(q, "rollup", "avg")), "avg"),
-)`;
+)`];
   }, [name, job, instance, rateEnabled, isBucket]);
 
   const { isLoading, graphData, error, queryErrors, warning, isHistogram } = useFetchQuery({
-    predefinedQuery: [query],
+    predefinedQuery: queries,
     visible: true,
     customStep: step,
     showAllSeries
@@ -98,7 +98,7 @@ with (q = ${queryBase}) (
       {warning && (
         <WarningLimitSeries
           warning={warning}
-          query={[query]}
+          query={queries}
           onChange={setShowAllSeries}
         />
       )}
@@ -107,7 +107,7 @@ with (q = ${queryBase}) (
           data={graphData}
           period={period}
           customStep={step}
-          query={[query]}
+          query={queries}
           yaxis={yaxis}
           setYaxisLimits={setYaxisLimits}
           setPeriod={setPeriod}

app/vmui/packages/vmui/src/components/Main/Accordion/Accordion.tsx

@@ -1,4 +1,5 @@
 import { FC, useState, useEffect } from "preact/compat";
+import classNames from "classnames";
 import { JSX } from "preact";
 import { ArrowDownIcon } from "../Icons";
 import "./style.scss";
@@ -31,9 +32,12 @@ const Accordion: FC<AccordionProps> = ({
       event.preventDefault();
       return; // If the text is selected, cancel the execution of toggle.
     }
-    const details = event.currentTarget.parentElement as HTMLDetailsElement;
-    onChange && onChange(details.open);
-    setIsOpen(details.open);
+
+    setIsOpen((prev) => {
+      const newState = !prev;
+      onChange && onChange(newState);
+      return newState;
+    });
   };
 
   useEffect(() => {
@@ -42,23 +46,32 @@ const Accordion: FC<AccordionProps> = ({
 
   return (
     <>
-      <details
-        className="vm-accordion-section"
-        key="content"
-        open={isOpen}
+      <header
+        className={classNames({
+          "vm-accordion-header": true,
+          "vm-accordion-header_open": isOpen,
+        })}
+        onClick={toggleOpen}
         id={id}
       >
-        <summary
-          className="vm-accordion-header"
-          onClick={toggleOpen}
+        {title}
+        <div
+          className={classNames({
+            "vm-accordion-header__arrow": true,
+            "vm-accordion-header__arrow_open": isOpen,
+          })}
         >
-          {title}
-          <div className="vm-accordion-header__arrow">
-            <ArrowDownIcon />
-          </div>
-        </summary>
-        {children}
-      </details>
+          <ArrowDownIcon />
+        </div>
+      </header>
+      {isOpen && (
+        <section
+          className="vm-accordion-section"
+          key="content"
+        >
+          {children}
+        </section>
+      )}
     </>
   );
 };

app/vmui/packages/vmui/src/components/Main/Accordion/style.scss

@@ -17,6 +17,10 @@
     transform: rotate(0);
     transition: transform 200ms ease-in-out;
 
+    &_open {
+      transform: rotate(180deg);
+    }
+
     svg {
       width: 14px;
       height: auto;
@@ -24,14 +28,6 @@
   }
 }
 
-.vm-accordion-section[open] > summary {
-  & > .vm-accordion-header {
-    &__arrow {
-      transform: rotate(180deg);
-    }
-  }
-}
-
 .accordion-section {
   overflow: hidden;
 }

app/vmui/packages/vmui/src/components/Main/Select/Select.tsx

@@ -137,6 +137,7 @@ const Select: FC<SelectProps> = ({
         "vm-select_disabled": disabled
       })}
     >
+      {label && <span className="vm-text-field__label">{label}</span>}
       <div
         className="vm-select-input"
         onClick={handleToggleList}
@@ -150,7 +151,7 @@ const Select: FC<SelectProps> = ({
               onRemoveItem={handleSelected}
             />
           )}
-          {!hideInput && !selectedValues?.length && (
+          {!hideInput && (
             <input
               value={textFieldValue}
               type="text"
@@ -164,7 +165,6 @@ const Select: FC<SelectProps> = ({
             />
           )}
         </div>
-        {label && <span className="vm-text-field__label">{label}</span>}
         {clearable && value && (
           <div
             className="vm-select-input__icon"

app/vmui/packages/vmui/src/components/Main/Select/style.scss

@@ -1,6 +1,8 @@
 @use "src/styles/variables" as *;
 
 .vm-select {
+  position: relative;
+  display: grid;
   &-input {
     position: relative;
     display: flex;

app/vmui/packages/vmui/src/components/QueryHistory/utils.ts

@@ -1,4 +1,8 @@
-import { getFromStorage, removeFromStorage, saveToStorage, StorageKeys } from "../../utils/storage";
+import {
+  getFromStorage,
+  saveToStorage,
+  StorageKeys,
+} from "../../utils/storage";
 import { QueryHistoryType } from "../../state/query/reducer";
 import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../constants/graph";
 
@@ -73,17 +77,3 @@ export const getUpdatedHistory = (query: string, queryHistory?: QueryHistoryType
     values: newValues
   };
 };
-
-const migrateMetricsQueryHistoryToHistoryByKey = () => {
-  const migrateHistory = (type: HistoryType) => {
-    const queryList = getFromStorage(type) as string;
-    if (queryList) {
-      const queryHistory: string[][] = JSON.parse(queryList);
-      saveHistoryToStorage("METRICS_QUERY_HISTORY", type, queryHistory);
-      removeFromStorage([type]);
-    }
-  };
-  migrateHistory("QUERY_HISTORY");
-  migrateHistory("QUERY_FAVORITES");
-};
-migrateMetricsQueryHistoryToHistoryByKey();

app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx

@@ -12,7 +12,7 @@ import {
   getMinMaxBuffer,
   getTimeSeries,
 } from "../../../utils/uplot";
-import { TimeParams, SeriesItem, LegendItemType } from "../../../types";
+import { TimeParams, LegendItemType } from "../../../types";
 import { AxisRange, YaxisState } from "../../../state/graph/reducer";
 import { getMathStats } from "../../../utils/math";
 import classNames from "classnames";
@@ -23,8 +23,6 @@ import { promValueToNumber } from "../../../utils/metric";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useElementSize from "../../../hooks/useElementSize";
 import { ChartTooltipProps } from "../../Chart/ChartTooltip/ChartTooltip";
-import LegendAnomaly from "../../Chart/Line/LegendAnomaly/LegendAnomaly";
-import { groupByMultipleKeys } from "../../../utils/array";
 import { useGraphDispatch } from "../../../state/graph/GraphStateContext";
 import { sameTs } from "../../../utils/time";
 import { useLocation } from "react-router-dom";
@@ -44,7 +42,6 @@ export interface GraphViewProps {
   fullWidth?: boolean;
   height?: number;
   isHistogram?: boolean;
-  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
@@ -64,7 +61,6 @@ const GraphView: FC<GraphViewProps> = ({
   fullWidth = true,
   height,
   isHistogram,
-  isAnomalyView,
   isPredefinedPanel,
   spanGaps,
   showAllPoints
@@ -89,8 +85,8 @@ const GraphView: FC<GraphViewProps> = ({
   const [legendValue, setLegendValue] = useState<ChartTooltipProps | null>(null);
 
   const getSeriesItem = useMemo(() => {
-    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isAnomalyView);
-  }, [data, hideSeries, alias, showAllPoints, isAnomalyView]);
+    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isRawQuery);
+  }, [data, hideSeries, alias, showAllPoints, isRawQuery]);
 
   const setLimitsYaxis = (minVal: number, maxVal: number) => {
     let min = Number.isFinite(minVal) ? minVal : 0;
@@ -102,7 +98,7 @@ const GraphView: FC<GraphViewProps> = ({
   };
 
   const onChangeLegend = (legend: LegendItemType, metaKey: boolean) => {
-    setHideSeries(getHideSeries({ hideSeries, legend, metaKey, series, isAnomalyView }));
+    setHideSeries(getHideSeries({ hideSeries, legend, metaKey, series }));
   };
 
   const prepareHistogramData = (data: (number | null)[][]) => {
@@ -127,25 +123,11 @@ const GraphView: FC<GraphViewProps> = ({
     return [null, [xs, ys, counts]];
   };
 
-  const prepareAnomalyLegend = (legend: LegendItemType[]): LegendItemType[] => {
-    if (!isAnomalyView) return legend;
-
-    // For vmanomaly: Only select the first series per group (due to API specs) and clear __name__ in freeFormFields.
-    const grouped = groupByMultipleKeys(legend, ["group", "label"]);
-    return grouped.map((group) => {
-      const firstEl = group.values[0];
-      return {
-        ...firstEl,
-        freeFormFields: { ...firstEl.freeFormFields, __name__: "" }
-      };
-    });
-  };
-
   useEffect(() => {
     const dLen = data.length;
 
-    const tsAnchor = data?.[0]?.values?.[0]?.[0]
-    const tsSet = new Set<number>([])
+    const tsAnchor = data?.[0]?.values?.[0]?.[0];
+    const tsArray: number[] = [];
     const tempLegend = new Array<LegendItemType>(dLen);
     const tempSeries = new Array<uPlotSeries>(dLen + 1);
     tempSeries[0] = {};
@@ -155,14 +137,14 @@ const GraphView: FC<GraphViewProps> = ({
 
     for (let i = 0; i < dLen; i++) {
       const d = data[i];
-      const seriesItem = getSeriesItem(d, i);
+      const seriesItem = getSeriesItem(d);
       tempSeries[i + 1] = seriesItem;
       tempLegend[i] = getLegendItem(seriesItem, d.group);
 
       const vals = d.values;
       for (let j = 0, vLen = vals.length; j < vLen; j++) {
         const v = vals[j];
-        if (isRawQuery) tsSet.add(v[0])
+        if (isRawQuery) tsArray.push(v[0]);
         const num = promValueToNumber(v[1]);
         if (Number.isFinite(num)) {
           if (num < minVal) minVal = num;
@@ -171,12 +153,12 @@ const GraphView: FC<GraphViewProps> = ({
       }
     }
 
-    const dpr = window.devicePixelRatio || 1
+    const dpr = window.devicePixelRatio || 1;
     const widthPx = containerSize.width || window.innerWidth || 4096;
     const pixels = Math.max(1, Math.floor(widthPx * Math.max(1, dpr)));
 
     const timeSeries = isRawQuery
-      ? Array.from(tsSet).sort((a,b) => a - b)
+      ? tsArray.sort((a, b) => a - b)
       : getTimeSeries(currentStep, period, pixels, tsAnchor);
 
     const timeDataSeries: (number | null)[][] = data.map(d => {
@@ -195,6 +177,8 @@ const GraphView: FC<GraphViewProps> = ({
           // Treat special values as nulls in order to satisfy uPlot.
           // Otherwise it may draw unexpected graphs.
           v = Number.isFinite(num) ? num : null;
+          // Advance to next value
+          j++;
         }
         results[k] = v;
       }
@@ -204,7 +188,7 @@ const GraphView: FC<GraphViewProps> = ({
       const avg = Math.abs(Number(avgRaw));
       const range = getMinMaxBuffer(min, max);
       const rangeStep = Math.abs(range[1] - range[0]);
-      const needStabilize = (avg > rangeStep * 1e10) && !isAnomalyView;
+      const needStabilize = (avg > rangeStep * 1e10);
 
       return needStabilize ? results.fill(avg) : results;
     });
@@ -212,13 +196,11 @@ const GraphView: FC<GraphViewProps> = ({
     timeDataSeries.unshift(timeSeries);
 
     const result = isHistogram ? prepareHistogramData(timeDataSeries) : timeDataSeries;
-    const legend = prepareAnomalyLegend(tempLegend);
 
     setLimitsYaxis(minVal, maxVal);
     setDataChart(result as uPlotData);
     setSeries(tempSeries);
-    setLegend(legend);
-    isAnomalyView && setHideSeries(legend.map(s => s.label || "").slice(1));
+    setLegend(tempLegend);
   }, [data, timezone, isHistogram, currentStep, isRawQuery]);
 
   useEffect(() => {
@@ -230,13 +212,13 @@ const GraphView: FC<GraphViewProps> = ({
 
     for (let i = 0; i < dLen; i++) {
       const d = data[i];
-      const seriesItem = getSeriesItem(d, i);
+      const seriesItem = getSeriesItem(d);
       tempSeries[i + 1] = seriesItem;
       tempLegend[i] = getLegendItem(seriesItem, d.group);
     }
 
     setSeries(tempSeries);
-    setLegend(prepareAnomalyLegend(tempLegend));
+    setLegend(tempLegend);
   }, [hideSeries]);
 
   const hasTimeData = dataChart[0]?.length > 0;
@@ -279,9 +261,8 @@ const GraphView: FC<GraphViewProps> = ({
           setPeriod={setPeriod}
           layoutSize={containerSize}
           height={height}
-          isAnomalyView={isAnomalyView}
           spanGaps={spanGaps}
-          showAllPoints={showAllPoints}
+          showAllPoints={isRawQuery ? true : showAllPoints}
         />
       )}
       {isHistogram && (
@@ -296,12 +277,10 @@ const GraphView: FC<GraphViewProps> = ({
           onChangeLegend={setLegendValue}
         />
       )}
-      {isAnomalyView && showLegend && (<LegendAnomaly series={series as SeriesItem[]}/>)}
       {!isHistogram && showLegend && (
         <Legend
           labels={legend}
           query={query}
-          isAnomalyView={isAnomalyView}
           onChange={onChangeLegend}
           isPredefinedPanel={isPredefinedPanel}
         />

app/vmui/packages/vmui/src/components/WebStorageCheck/WebStorageCheck.tsx

@@ -0,0 +1,69 @@
+import { useEffect } from "react";
+import { StorageErrorCode } from "./types";
+import { useSnack } from "../../contexts/Snackbar";
+import { storageErrorInfo } from "./storageErrors";
+import "./style.scss";
+
+const classifyStorageException = (e: unknown): StorageErrorCode => {
+  if (!(e instanceof DOMException)) return StorageErrorCode.UNKNOWN;
+
+  switch (e.name) {
+    case "QuotaExceededError":
+      return StorageErrorCode.QUOTA_EXCEEDED;
+    case "SecurityError":
+      return StorageErrorCode.SECURITY_ERROR;
+    default:
+      return StorageErrorCode.UNKNOWN;
+  }
+};
+
+const getStorageError = (storage: Storage | null | undefined): StorageErrorCode | null => {
+  if (!storage) {
+    return StorageErrorCode.NO_STORAGE;
+  }
+
+  try {
+    const key = "__vmui_test__";
+    storage.setItem(key, "1");
+    storage.removeItem(key);
+    return null;
+  } catch (e) {
+    return classifyStorageException(e);
+  }
+};
+
+const WebStorageCheck = () => {
+  const { showInfoMessage } = useSnack();
+
+  useEffect(() => {
+    const error = getStorageError(window.localStorage);
+
+    if (error) {
+      const { title, description, fix } = storageErrorInfo[error];
+
+      const text = (
+        <div className="vm-storage-check">
+          <h3>{title}</h3>
+          <p>{description}</p>
+
+          {!!fix?.length && (
+            <div className="vm-storage-check__fix">
+              <div>Try this:</div>
+              <ul>
+                {fix.map((step, i) => (
+                  <li key={`${i}-${step}`}>{step}</li>
+                ))}
+              </ul>
+            </div>
+          )}
+        </div>
+      );
+
+      showInfoMessage({ text: text, type: "error", timeout: 600000 });
+    }
+  }, []);
+
+  return null;
+};
+
+export default WebStorageCheck;