fix

Related to
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10189

Debug notes
https://github.com/VictoriaMetrics/debug-notes/tree/main/gh10189

.github/workflows/vmui.yml

@@ -34,33 +34,39 @@ jobs:
       - name: Code checkout
         uses: actions/checkout@v6
 
-      - name: Setup Node
-        uses: actions/setup-node@v6
+      - name: Cache node_modules
+        id: cache
+        uses: actions/cache@v5
         with:
-          node-version: '24.x'
+          path: app/vmui/packages/vmui/node_modules
+          key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
+          restore-keys: |
+            vmui-deps-${{ runner.os }}-
 
-      - name: Cache node-modules
-        uses: actions/cache@v4
-        with:
-          path: |
-            app/vmui/packages/vmui/node_modules
-          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: vmui-artifacts-${{ runner.os }}-
+      - name: Install dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make vmui-install
 
       - name: Run lint
         id: lint
         run: make vmui-lint
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run tests
         id: test
         run: make vmui-test
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run typecheck
         id: typecheck
         run: make vmui-typecheck
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
         uses: ataylorme/eslint-annotate-action@v3

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2026 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

app/victoria-metrics/main.go

@@ -134,6 +134,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/victoria-metrics/self_scraper.go

@@ -29,11 +29,9 @@ var selfScraperWG sync.WaitGroup
 
 func startSelfScraper() {
 	selfScraperStopCh = make(chan struct{})
-	selfScraperWG.Add(1)
-	go func() {
-		defer selfScraperWG.Done()
+	selfScraperWG.Go(func() {
 		selfScraper(*selfScrapeInterval)
-	}()
+	})
 }
 
 func stopSelfScraper() {

app/vmagent/main.go

@@ -245,6 +245,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/vmagent/remotewrite/client.go

@@ -202,14 +202,10 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
 	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(*queues)
+		return float64(concurrency)
 	})
-	for i := 0; i < concurrency; i++ {
-		c.wg.Add(1)
-		go func() {
-			defer c.wg.Done()
-			c.runWorker()
-		}()
+	for range concurrency {
+		c.wg.Go(c.runWorker)
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 }

app/vmagent/remotewrite/pendingseries.go

@@ -48,11 +48,7 @@ func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Boo
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
-	ps.periodicFlusherWG.Add(1)
-	go func() {
-		defer ps.periodicFlusherWG.Done()
-		ps.periodicFlusher()
-	}()
+	ps.periodicFlusherWG.Go(ps.periodicFlusher)
 	return &ps
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -59,7 +59,7 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. "+
 		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -176,13 +176,6 @@ func Init() {
 		})
 	}
 
-	if *queues > maxQueues {
-		*queues = maxQueues
-	}
-	if *queues <= 0 {
-		*queues = 1
-	}
-
 	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
@@ -215,9 +208,7 @@ func Init() {
 	dropDanglingQueues()
 
 	// Start config reloader.
-	configReloaderWG.Add(1)
-	go func() {
-		defer configReloaderWG.Done()
+	configReloaderWG.Go(func() {
 		for {
 			select {
 			case <-configReloaderStopCh:
@@ -227,7 +218,7 @@ func Init() {
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
 		}
-	}()
+	})
 }
 
 func dropDanglingQueues() {
@@ -267,17 +258,6 @@ func initRemoteWriteCtxs(urls []string) {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
-
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
-	if maxInmemoryBlocks / *queues > 100 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 100 * *queues
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
 	rwctxIdx := make([]int, len(urls))
 	if retryMaxTime.String() != "" {
@@ -292,7 +272,7 @@ func initRemoteWriteCtxs(urls []string) {
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, sanitizedURL)
 		rwctxIdx[i] = i
 	}
 
@@ -558,11 +538,9 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 	// Push metadata to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.tryPushMetadataInternal(mms) {
 				rwctx.pushFailures.Inc()
 				if forceDropSamplesOnFailure {
@@ -571,7 +549,7 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 				}
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -603,15 +581,13 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 	// Push tssBlock to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -633,13 +609,11 @@ func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock
 		if len(shard) == 0 {
 			continue
 		}
-		wg.Add(1)
-		go func(rwctx *remoteWriteCtx, tss []prompb.TimeSeries) {
-			defer wg.Done()
-			if !rwctx.TryPushTimeSeries(tss, forceDropSamplesOnFailure) {
+		wg.Go(func() {
+			if !rwctx.TryPushTimeSeries(shard, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx, shard)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -848,7 +822,7 @@ type remoteWriteCtx struct {
 	rowsDroppedOnPushFailure     *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
@@ -863,6 +837,23 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
+	queuesSize := queues.GetOptionalArg(argIdx)
+	if queuesSize > maxQueues {
+		queuesSize = maxQueues
+	} else if queuesSize <= 0 {
+		queuesSize = 1
+	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks/queuesSize > 100 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 100 * queuesSize
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
@@ -880,16 +871,16 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
-		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, queuesSize)
 	default:
 		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
 	}
-	c.init(argIdx, *queues, sanitizedURL)
+	c.init(argIdx, queuesSize, sanitizedURL)
 
 	// Initialize pss
 	sf := significantFigures.GetOptionalArg(argIdx)
 	rd := roundDigits.GetOptionalArg(argIdx)
-	pssLen := *queues
+	pssLen := queuesSize
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
 		// since every pendingSeries can saturate up to a single CPU.

app/vmalert/config/types.go

@@ -76,11 +76,14 @@ func (t *Type) ValidateExpr(expr string) error {
 		if err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
-		fields, _ := q.GetStatsByFields()
-		for i := range fields {
+		labels, err := q.GetStatsLabels()
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
+		}
+		for i := range labels {
 			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
 			// making the result meaningless and may lead to cardinality issues.
-			if fields[i] == "_time" {
+			if labels[i] == "_time" {
 				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
 			}
 		}

app/vmalert/main.go

@@ -81,9 +81,7 @@ absolute path to all .tpl files in root.
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
-var (
-	extURL *url.URL
-)
+var extURL *url.URL
 
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
@@ -161,7 +159,7 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	manager, err := newManager(ctx)
 	if err != nil {
-		logger.Fatalf("failed to init: %s", err)
+		logger.Fatalf("failed to create manager: %s", err)
 	}
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
 	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)

app/vmalert/manager_test.go

@@ -65,11 +65,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
@@ -79,7 +77,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 				}
 				_ = m.update(context.Background(), cfg, false)
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }

app/vmalert/notifier/alertmanager.go

@@ -14,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -172,11 +171,6 @@ const alertManagerPath = "/api/v2/alerts"
 func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
 	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration,
 ) (*AlertManager, error) {
-
-	if err := httputil.CheckURL(alertManagerURL); err != nil {
-		return nil, fmt.Errorf("invalid alertmanager URL: %w", err)
-	}
-
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig

app/vmalert/notifier/config_watcher_test.go

@@ -212,18 +212,16 @@ consul_sd_configs:
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
 				_ = cw.reload(paths[rnd]) // update can fail and this is expected
 				_ = cw.notifiers()
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }

app/vmalert/notifier/init.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -229,6 +230,9 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
+		if err := httputil.CheckURL(addr); err != nil {
+			return nil, fmt.Errorf("invalid notifier.url %q: %w", addr, err)
+		}
 		addr = strings.TrimSuffix(addr, "/")
 		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, sendTimeout.GetOptionalArg(i))
 		if err != nil {
@@ -266,7 +270,7 @@ func GetTargets() map[TargetType][]Target {
 	if getActiveNotifiers == nil {
 		return nil
 	}
-	var targets = make(map[TargetType][]Target)
+	targets := make(map[TargetType][]Target)
 	// use cached targets from configWatcher instead of getActiveNotifiers for the extra target labels
 	if cw != nil {
 		cw.targetsMu.RLock()

app/vmalert/notifier/init_test.go

@@ -55,9 +55,9 @@ func TestInitNegative(t *testing.T) {
 		*blackHole = oldBlackHole
 	}()
 
-	f := func(path, addr string, bh bool) {
+	f := func(path string, addr []string, bh bool) {
 		*configPath = path
-		*addrs = flagutil.ArrayString{addr}
+		*addrs = flagutil.ArrayString(addr)
 		*blackHole = bh
 		if err := Init(nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
@@ -65,9 +65,12 @@ func TestInitNegative(t *testing.T) {
 	}
 
 	// *configPath, *addrs and *blackhole are mutually exclusive
-	f("/dummy/path", "127.0.0.1", false)
-	f("/dummy/path", "", true)
-	f("", "127.0.0.1", true)
+	f("/dummy/path", []string{"127.0.0.1"}, false)
+	f("/dummy/path", []string{}, true)
+	f("", []string{"127.0.0.1"}, true)
+	// addr cannot be ""
+	f("", []string{""}, false)
+	f("", []string{"127.0.0.1", ""}, false)
 }
 
 func TestBlackHole(t *testing.T) {

app/vmalert/rule/rule_test.go

@@ -65,17 +65,15 @@ func TestRule_stateConcurrent(_ *testing.T) {
 	r := &AlertingRule{state: &ruleState{entries: make([]StateEntry, 20)}}
 	const workers = 50
 	const iterations = 100
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func() {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for range workers {
+		wg.Go(func() {
 			for i := 0; i < iterations; i++ {
 				r.state.add(StateEntry{At: time.Now()})
 				r.state.getAll()
 				r.state.getLast()
 			}
-		}()
+		})
 	}
 	wg.Wait()
 }

app/vmalert/web.qtpl

@@ -9,6 +9,7 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 ) %}
 
 {% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
@@ -78,6 +79,8 @@
 {% func Welcome(r *http.Request) %}
     {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
     <p>
+        Version {%s buildinfo.Version %} <br>
+
         API:<br>
         {% for _, p := range apiLinks  %}
             {%code p, doc := p[0], p[1] %}

app/vmauth/auth_config.go

@@ -113,10 +113,8 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
-		ui.concurrencyLimitReached.Inc()
-
-		// The per-user limit for the number of concurrent requests is reached.
-		// Wait until the currently executed requests are finished, so the current request could be executed.
+		// The number of concurrently executed requests for the given user equals the limt.
+		// Wait until some of the currently executed requests are finished, so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case ui.concurrencyLimitCh <- struct{}{}:
@@ -124,6 +122,8 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 		case <-ctx.Done():
 			err := ctx.Err()
 			if errors.Is(err, context.DeadlineExceeded) {
+				// The current request couldn't be executed until the request timeout.
+				ui.concurrencyLimitReached.Inc()
 				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
 					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
 			}
@@ -150,12 +150,22 @@ func (ui *UserInfo) stopHealthChecks() {
 	if ui == nil {
 		return
 	}
-	if ui.URLPrefix == nil {
-		return
-	}
 
-	bus := ui.URLPrefix.bus.Load()
-	bus.stopHealthChecks()
+	if ui.URLPrefix != nil {
+		bus := ui.URLPrefix.bus.Load()
+		bus.stopHealthChecks()
+	}
+	if ui.DefaultURL != nil {
+		bus := ui.DefaultURL.bus.Load()
+		bus.stopHealthChecks()
+	}
+	for i := range ui.URLMaps {
+		um := &ui.URLMaps[i]
+		if um.URLPrefix != nil {
+			bus := um.URLPrefix.bus.Load()
+			bus.stopHealthChecks()
+		}
+	}
 }
 
 // Header is `Name: Value` http header, which must be added to the proxied request.
@@ -363,12 +373,10 @@ func (bu *backendURL) isBroken() bool {
 
 func (bu *backendURL) setBroken() {
 	if bu.broken.CompareAndSwap(false, true) {
-		bu.healthCheckWG.Add(1)
-		go func() {
-			defer bu.healthCheckWG.Done()
+		bu.healthCheckWG.Go(func() {
 			bu.runHealthCheck()
 			bu.broken.Store(false)
-		}()
+		})
 	}
 }
 
@@ -733,11 +741,9 @@ func initAuthConfig() {
 	configTimestamp.Set(fasttime.UnixTimestamp())
 
 	stopCh = make(chan struct{})
-	authConfigWG.Add(1)
-	go func() {
-		defer authConfigWG.Done()
+	authConfigWG.Go(func() {
 		authConfigReloader(sighupCh)
-	}()
+	})
 }
 
 func stopAuthConfig() {

app/vmauth/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ioutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -40,27 +41,38 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
-		"See also -maxConcurrentRequests")
-	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
+	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
+	idleConnTimeout        = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
 	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 
-	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
-	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options "+
-		"and max_concurrent_requests option in per-user config")
-	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration the request waits for execution when the number of concurrently executed "+
-		"requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. "+
-		"This allows graceful handling of short spikes in the number of concurrent requests")
+	requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
+		"This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. "+
+		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
+	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
+		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
+		"See also -requestBufferSize")
+
+	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process simultaneously. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This protects vmauth itself from overloading and out-of-memory (OOM) failures. See also -maxConcurrentPerUserRequests "+
+		"and https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 100, "The maximum number of concurrent requests vmauth can process per each configured user. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This provides fairness and isolation between users, preventing a single user from consuming all the available resources. "+
+		"It works in conjunction with -maxConcurrentRequests, which sets the global limit across all users. "+
+		"This default can be overridden for individual users via max_concurrent_requests option in per-user config. "+
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration to wait before rejecting incoming requests if concurrency limit "+
+		"specified via -maxConcurrentRequests or -maxConcurrentPerUserRequests command-line flags is reached. "+
+		"Requests are rejected with '429 Too Many Requests' http status code if the limit is still reached after the -maxQueueDuration duration. "+
+		"This allows graceful handling of short spikes in concurrent requests. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
 
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
-	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
-	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
-		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
+	failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
+
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -215,48 +227,121 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
 	defer cancel()
 
-	// Limit the concurrency of requests to backends
+	// Acquire global concurrency limit.
+	if err := beginConcurrencyLimit(ctx); err != nil {
+		handleConcurrencyLimitError(w, r, err)
+		return
+	}
+	defer endConcurrencyLimit()
+
+	// Set read deadline for reading the initial chunk for the request body.
+	rc := http.NewResponseController(w)
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		logger.Panicf("BUG: expecting valid deadline for the context")
+	}
+	if err := rc.SetReadDeadline(deadline); err != nil {
+		logger.Panicf("BUG: cannot set read deadline: %s", err)
+	}
+
+	// Read the initial chunk for the request body.
+	userName := ui.name()
+	if userName == "" {
+		userName = "unauthorized"
+	}
+	bb, err := bufferRequestBody(ctx, r.Body, userName)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	r.Body = bb
+
+	// Disable the read deadline for the rest of the request body.
+	if err := rc.SetReadDeadline(time.Time{}); err != nil {
+		logger.Panicf("BUG: cannot reset read deadline: %s", err)
+	}
+
+	// Acquire concurrency limit for the given user.
+	if err := ui.beginConcurrencyLimit(ctx); err != nil {
+		handleConcurrencyLimitError(w, r, err)
+		return
+	}
+	defer ui.endConcurrencyLimit()
+
+	// Process the request.
+	processRequest(w, r, ui)
+}
+
+func beginConcurrencyLimit(ctx context.Context) error {
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		if err := ui.beginConcurrencyLimit(ctx); err != nil {
-			handleConcurrencyLimitError(w, r, err)
-			<-concurrencyLimitCh
-			return
-		}
+		return nil
 	default:
 		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
 		// so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case concurrencyLimitCh <- struct{}{}:
-			if err := ui.beginConcurrencyLimit(ctx); err != nil {
-				handleConcurrencyLimitError(w, r, err)
-				<-concurrencyLimitCh
-				return
-			}
+			return nil
 		case <-ctx.Done():
 			err := ctx.Err()
-
-			concurrentRequestsLimitReached.Inc()
-
 			if errors.Is(err, context.DeadlineExceeded) {
-				err = fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+				// The current request couldn't be executed until the request timeout.
+				concurrentRequestsLimitReached.Inc()
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
 					*maxQueueDuration, cap(concurrencyLimitCh))
-				handleConcurrencyLimitError(w, r, err)
-				return
 			}
-
-			err = fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
-			handleConcurrencyLimitError(w, r, err)
-			return
+			return fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
 		}
 	}
-	processRequest(w, r, ui)
-	ui.endConcurrencyLimit()
+}
+
+func endConcurrencyLimit() {
 	<-concurrencyLimitCh
 }
 
+func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (io.ReadCloser, error) {
+	if r == nil {
+		// This is a GET request with nil reader.
+		return nil, nil
+	}
+
+	maxBufSize := max(requestBufferSize.IntN(), maxRequestBodySizeToRetry.IntN())
+	if maxBufSize <= 0 {
+		return r, nil
+	}
+
+	lr := ioutil.GetLimitedReader(r, int64(maxBufSize))
+	defer ioutil.PutLimitedReader(lr)
+
+	start := time.Now()
+	buf, err := io.ReadAll(lr)
+	bufferRequestBodyDuration.UpdateDuration(start)
+
+	if err != nil {
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			rejectSlowClientRequests.Inc()
+
+			d := time.Since(start)
+
+			return nil, &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
+					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
+				StatusCode: http.StatusBadRequest,
+			}
+		}
+
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot read request body: %w", err),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+
+	bb := newBufferedBody(r, buf, maxBufSize)
+	return bb, nil
+}
+
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
@@ -282,9 +367,6 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		isDefault = true
 	}
 
-	rtb := newReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
-	r.Body = rtb
-
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getBackendURL()
@@ -292,18 +374,19 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			break
 		}
 		targetURL := bu.url
-		// Don't change path and add request_path query param for default route.
 		if isDefault {
+			// Don't change path and add request_path query param for default route.
 			query := targetURL.Query()
 			query.Set("request_path", u.String())
 			targetURL.RawQuery = query.Encode()
-		} else { // Update path for regular routes.
+		} else {
+			// Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		}
 
 		wasLocalRetry := false
 	again:
-		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui, bu)
 		if needLocalRetry && !wasLocalRetry {
 			wasLocalRetry = true
 			goto again
@@ -313,6 +396,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if ok {
 			return
 		}
+
 		bu.setBroken()
 		ui.backendErrors.Inc()
 	}
@@ -324,7 +408,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ui.requestErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo, bu *backendURL) (bool, bool) {
 	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
@@ -339,30 +423,19 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		}
 	}
 
-	rtb, rtbOK := req.Body.(*readTrackingBody)
+	bb, bbOK := req.Body.(*bufferedBody)
+	canRetry := !bbOK || bb.canRetry()
+
 	res, err := ui.rt.RoundTrip(req)
 
-	if ctxErr := r.Context().Err(); ctxErr != nil {
-		// Override the error returned by the RoundTrip with the context error if it isn't non-nil
-		// This makes sure the proper logging for canceled and timed out requests - log the real cause of the error
-		// instead of the random error, which could be returned from RoundTrip because of canceled or timed out request.
-		err = ctxErr
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
+		clientCanceledRequests.Inc()
+		return true, false
 	}
+
 	if err != nil {
-		// Do not retry canceled
-		if errors.Is(err, context.Canceled) {
-			clientCanceledRequests.Inc()
-			return true, false
-		}
-		// Do not retry timed out requests
-		if errors.Is(err, context.DeadlineExceeded) {
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			// Timed out request must be counted as errors, since this usually means that the backend is slow.
-			logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
-			return false, false
-		}
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
@@ -371,27 +444,32 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
 			ui.requestErrors.Inc()
+			bu.setBroken()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
 			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
+			if bbOK {
+				bb.resetReader()
+			}
 			return false, true
 		}
 
-		// Request body wasn't read yet, this usually means that the backend isn't reachable; retry the request at another backend
+		// Retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, req.URL, targetURL, err)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, requestURI, targetURL, err)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
-		_ = res.Body.Close()
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request body has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
@@ -400,13 +478,16 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			ui.requestErrors.Inc()
 			return true, false
 		}
+
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
+		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
-			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
+			remoteAddr, requestURI, targetURL, res.StatusCode, retryStatusCodes)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	removeHopHeaders(res.Header)
@@ -416,13 +497,16 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-	if errors.Is(err, context.Canceled) {
+
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
 		clientCanceledRequests.Inc()
 		return true, false
-	} else if err != nil && !netutil.IsTrivialNetworkError(err) {
+	}
+
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
-
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
 		ui.requestErrors.Inc()
 		return true, false
@@ -553,6 +637,9 @@ var (
 	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
 	clientCanceledRequests   = metrics.NewCounter(`vmauth_http_request_errors_total{reason="client_canceled"}`)
+	rejectSlowClientRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="reject_slow_client"}`)
+
+	bufferRequestBodyDuration = metrics.NewSummary(`vmauth_buffer_request_body_duration_seconds`)
 )
 
 func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
@@ -636,8 +723,7 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
-	ctx := r.Context()
-	if errors.Is(ctx.Err(), context.Canceled) {
+	if errors.Is(r.Context().Err(), context.Canceled) {
 		// Do not return any response for the request canceled by the client,
 		// since the connection to the client is already closed.
 		clientCanceledRequests.Inc()
@@ -652,123 +738,78 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-// readTrackingBody must be obtained via getReadTrackingBody()
-type readTrackingBody struct {
-	// maxBodySize is the maximum body size to cache in buf.
+// bufferedBody serves two purposes:
+//  1. Enables request retries when the body size does not exceed maxBodySize
+//     by fully buffering the body in memory.
+//  2. Prevents slow clients from reducing effective server capacity by
+//     buffering the request body before acquiring a per-user concurrency slot.
+//
+// See bufferRequestBody for details on how bufferedBody is used.
+type bufferedBody struct {
+	// r contains reader for reading the data after buf is read.
 	//
-	// Bigger bodies cannot be retried.
-	maxBodySize int
-
-	// r contains reader for initial data reading
+	// r is nil if buf contains all the data.
 	r io.ReadCloser
 
-	// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
-	// If more than maxBodySize is read from r, then cannotRetry is set to true.
+	// buf contains the initial buffer read from r.
 	buf []byte
 
-	// readBuf points to the cached data at buf, which must be read in the next call to Read().
-	readBuf []byte
+	// bufOffset is the offset at buf for already read bytes.
+	bufOffset int
 
-	// cannotRetry is set to true when more than maxBodySize bytes are read from r.
-	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
+	// cannotRetry is set to true after Close() call on non-nil r.
 	cannotRetry bool
-
-	// bufComplete is set to true when buf contains complete request body read from r.
-	bufComplete bool
 }
 
-func newReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
-	// do not use sync.Pool there
-	// since http.RoundTrip may still use request body after return
-	// See this issue for details https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
-	rtb := &readTrackingBody{}
-	if maxBodySize < 0 {
-		maxBodySize = 0
+func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody {
+	// Do not use sync.Pool here, since http.RoundTrip may still use request body after return.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
+
+	if len(buf) < maxBufSize {
+		// Read the full request body into buf.
+		r = nil
 	}
-	rtb.maxBodySize = maxBodySize
 
-	if r == nil {
-		// This is GET request without request body
-		r = (*zeroReader)(nil)
+	return &bufferedBody{
+		r:   r,
+		buf: buf,
 	}
-	rtb.r = r
-	return rtb
-}
-
-type zeroReader struct{}
-
-func (r *zeroReader) Read(_ []byte) (int, error) {
-	return 0, io.EOF
-}
-
-func (r *zeroReader) Close() error {
-	return nil
 }
 
 // Read implements io.Reader interface.
-func (rtb *readTrackingBody) Read(p []byte) (int, error) {
-	if len(rtb.readBuf) > 0 {
-		n := copy(p, rtb.readBuf)
-		rtb.readBuf = rtb.readBuf[n:]
+func (bb *bufferedBody) Read(p []byte) (int, error) {
+	if bb.cannotRetry {
+		return 0, fmt.Errorf("cannot read already closed body")
+	}
+	if bb.bufOffset < len(bb.buf) {
+		n := copy(p, bb.buf[bb.bufOffset:])
+		bb.bufOffset += n
 		return n, nil
 	}
-
-	if rtb.r == nil {
-		if rtb.bufComplete {
-			return 0, io.EOF
-		}
-		return 0, fmt.Errorf("cannot read client request body after closing client reader")
+	if bb.r == nil {
+		return 0, io.EOF
 	}
-
-	n, err := rtb.r.Read(p)
-	if rtb.cannotRetry {
-		return n, err
-	}
-
-	if len(rtb.buf)+n > rtb.maxBodySize {
-		rtb.cannotRetry = true
-		return n, err
-	}
-	rtb.buf = append(rtb.buf, p[:n]...)
-	if err == io.EOF {
-		rtb.bufComplete = true
-	}
-	return n, err
+	return bb.r.Read(p)
 }
 
-func (rtb *readTrackingBody) canRetry() bool {
-	if rtb.cannotRetry {
-		return false
-	}
-	if rtb.bufComplete {
-		return true
-	}
-	return rtb.r != nil
+func (bb *bufferedBody) canRetry() bool {
+	return bb.r == nil
 }
 
 // Close implements io.Closer interface.
-func (rtb *readTrackingBody) Close() error {
-	if !rtb.cannotRetry {
-		rtb.readBuf = rtb.buf
-	} else {
-		rtb.readBuf = nil
+func (bb *bufferedBody) Close() error {
+	bb.resetReader()
+	if bb.r != nil {
+		bb.cannotRetry = true
+		return bb.r.Close()
 	}
-
-	// Close rtb.r only if the request body is completely read or if it is too big.
-	// http.Roundtrip performs body.Close call even without any Read calls,
-	// so this hack allows us to reuse request body.
-	if rtb.bufComplete || rtb.cannotRetry {
-		if rtb.r == nil {
-			return nil
-		}
-		err := rtb.r.Close()
-		rtb.r = nil
-		return err
-	}
-
 	return nil
 }
 
+func (bb *bufferedBody) resetReader() {
+	bb.bufOffset = 0
+}
+
 func debugInfo(u *url.URL, r *http.Request) string {
 	s := &strings.Builder{}
 	fmt.Fprintf(s, " (host: %q; ", r.Host)

app/vmauth/main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net"
@@ -10,6 +11,7 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
@@ -546,28 +548,300 @@ func (w *fakeResponseWriter) WriteHeader(statusCode int) {
 	}
 }
 
-func TestReadTrackingBody_RetrySuccess(t *testing.T) {
+// This is needed for net/http.ResponseController
+func (w *fakeResponseWriter) SetReadDeadline(deadline time.Time) error {
+	return nil
+}
+
+func TestBufferRequestBody_Success(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	f := func(body *bytes.Buffer, requestBufferSizeFlag, maxRequestBodySizeToRetryFlag string) {
+		t.Helper()
+
+		expectedResponse := "statusCode=200"
+		if body.Len() > 0 {
+			expectedResponse += "\n" + body.String()
+		}
+
+		if err := requestBufferSize.Set(requestBufferSizeFlag); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		if err := maxRequestBodySizeToRetry.Set(maxRequestBodySizeToRetryFlag); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if !backendCalled {
+			t.Fatalf("backend is not called")
+		}
+	}
+
+	// no body, no buffering, no retry
+	f(bytes.NewBuffer(nil), "0", "0")
+
+	// no body, buffering on, no retry
+	f(bytes.NewBuffer(nil), "100", "0")
+
+	// no body, no buffering, retry on
+	f(bytes.NewBuffer(nil), "0", "100")
+
+	// no body, buffering on, retry on
+	f(bytes.NewBuffer(nil), "100", "100")
+
+	// body smaller than buffer, retry max on
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "101", "101")
+
+	// body smaller than buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "501", "0")
+
+	// body same size as buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "500", "0")
+
+	// body bigger than a buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "499", "0")
+
+	// body bigger than tmpBuf 8KiB used in buffering
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16384", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16385", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16383", "")
+}
+
+func TestBufferRequestBody_Failure(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	defaultMaxQueueDuration := *maxQueueDuration
+	defer func() {
+		*maxQueueDuration = defaultMaxQueueDuration
+	}()
+
+	f := func(body *mockBody, expectedResponse string) {
+		t.Helper()
+
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+		if err := requestBufferSize.Set("2048"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		*maxQueueDuration = 100 * time.Millisecond
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if backendCalled {
+			t.Fatalf("backend is called")
+		}
+	}
+
+	// an error at the beginning of reading
+	f(&mockBody{err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+
+	// an error after reading 1024 bytes, buffer size is 2048 bytes
+	f(&mockBody{head: make([]byte, 1024), err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+}
+
+type mockBody struct {
+	head []byte
+	err  error
+	tail []byte
+}
+
+func (r *mockBody) Read(p []byte) (n int, err error) {
+	if len(r.head) > 0 {
+		n = copy(p, r.head)
+		r.head = r.head[n:]
+		return n, nil
+	}
+
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	if len(r.tail) > 0 {
+		n = copy(p, r.tail)
+		r.tail = r.tail[n:]
+		return n, nil
+	}
+
+	return 0, io.EOF
+}
+
+func TestBufferedBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		for i := 0; i < 5; i++ {
-			data, err := io.ReadAll(rtb)
+			data, err := io.ReadAll(rb)
 			if err != nil {
 				t.Fatalf("unexpected error when reading all the data at iteration %d: %s", i, err)
 			}
 			if string(data) != s {
 				t.Fatalf("unexpected data read at iteration %d\ngot\n%s\nwant\n%s", i, data, s)
 			}
-			if err := rtb.Close(); err != nil {
-				t.Fatalf("unexpected error when closing readTrackingBody at iteration %d: %s", i, err)
-			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true at iteration %d", i)
+			if err := rb.Close(); err != nil {
+				t.Fatalf("unexpected error when closing bufferedBody at iteration %d: %s", i, err)
 			}
 		}
 	}
@@ -577,19 +851,48 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
+func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		// Check the case with partial read
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
+			t.Fatalf("canRetry must return true")
+		}
 		for i := 0; i < len(s); i++ {
 			buf := make([]byte, i)
-			n, err := io.ReadFull(rtb, buf)
+			n, err := io.ReadFull(rb, buf)
 			if err != nil {
 				t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
 			}
@@ -599,26 +902,20 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 			if string(buf) != s[:i] {
 				t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
 			}
-			if err := rtb.Close(); err != nil {
+			if err := rb.Close(); err != nil {
 				t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
 			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
-			}
 		}
 
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 	}
 
@@ -627,30 +924,53 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
+func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set("0"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true before reading anything")
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if canRetry {
+			t.Fatalf("canRetry() must return false because of too big request body")
 		}
 		buf := make([]byte, 1)
-		n, err := io.ReadFull(rtb, buf)
+		n, err := io.ReadFull(rb, buf)
 		if err != nil {
 			t.Fatalf("unexpected error when reading a single byte: %s", err)
 		}
 		if n != 1 {
 			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
 		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after reading one byte")
-		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
@@ -658,14 +978,11 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 		if dataRead != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		data, err = io.ReadAll(rtb)
+		data, err = io.ReadAll(rb)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -679,35 +996,48 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 	f(newTestString(2*maxBodySize), maxBodySize)
 }
 
-func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
+func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		data, err = io.ReadAll(rb)
+		if err != nil {
+			t.Fatalf("unexpected error in io.ReadAll: %s", err)
 		}
-		data, err = io.ReadAll(rtb)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(data) != 0 {
-			t.Fatalf("unexpected non-empty data read: %q", data)
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
 	}
 

app/vmctl/flags.go

@@ -123,32 +123,32 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
-				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
+				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination.",
 		},
 		&cli.StringFlag{
 			Name:  vmCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS key to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used",
+			Usage: "Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used",
 		},
 		&cli.StringFlag{
 			Name:  vmServerName,
-			Usage: "Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used",
+			Usage: "Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used",
 		},
 		&cli.BoolFlag{
 			Name:  vmInsecureSkipVerify,
-			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
+			Usage: "Whether to skip tls verification when connecting to '--vm-addr'",
 			Value: false,
 		},
 		&cli.IntFlag{
@@ -598,7 +598,7 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
@@ -625,8 +625,8 @@ var (
 		&cli.BoolFlag{
 			Name: vmNativeDisableBinaryProtocol,
 			Usage: "Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format " +
-				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API." +
-				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
+				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API. " +
+				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks. " +
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},

app/vmctl/influx.go

@@ -63,10 +63,8 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 	ip.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(ip.cc)
-	for i := 0; i < ip.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range ip.cc {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if err := ip.do(s); err != nil {
 					errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
@@ -74,7 +72,7 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 				}
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import

app/vmctl/opentsdb.go

@@ -89,10 +89,8 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 			bar.Finish()
 		}(bar)
 		var wg sync.WaitGroup
-		wg.Add(op.otsdbcc)
-		for i := 0; i < op.otsdbcc; i++ {
-			go func() {
-				defer wg.Done()
+		for range op.otsdbcc {
+			wg.Go(func() {
 				for s := range seriesCh {
 					if err := op.do(s); err != nil {
 						errCh <- fmt.Errorf("couldn't retrieve series for %s : %s", metric, err)
@@ -100,7 +98,7 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 					}
 					bar.Increment()
 				}
-			}()
+			})
 		}
 		/*
 			Loop through all series for this metric, processing all retentions and time ranges

app/vmctl/prometheus.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 	"sync"
 
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 
@@ -61,19 +63,19 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
-		var labels []vm.LabelPair
+		var labelPairs []vm.LabelPair
 		series := ss.At()
 
-		for _, label := range series.Labels() {
+		series.Labels().Range(func(label labels.Label) {
 			if label.Name == "__name__" {
 				name = label.Value
-				continue
+				return
 			}
-			labels = append(labels, vm.LabelPair{
-				Name:  label.Name,
-				Value: label.Value,
+			labelPairs = append(labelPairs, vm.LabelPair{
+				Name:  strings.Clone(label.Name),
+				Value: strings.Clone(label.Value),
 			})
-		}
+		})
 		if name == "" {
 			return fmt.Errorf("failed to find `__name__` label in labelset for block %v", b.Meta().ULID)
 		}
@@ -99,7 +101,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 		}
 		ts := vm.TimeSeries{
 			Name:       name,
-			LabelPairs: labels,
+			LabelPairs: labelPairs,
 			Timestamps: timestamps,
 			Values:     values,
 		}
@@ -122,10 +124,8 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	pp.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(pp.cc)
-	for i := 0; i < pp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range pp.cc {
+		wg.Go(func() {
 			for br := range blockReadersCh {
 				if err := pp.do(br); err != nil {
 					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
@@ -133,7 +133,7 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 				}
 				bar.Increment()
 			}
-		}()
+		})
 	}
 	// any error breaks the import
 	for _, br := range blocks {

app/vmctl/remoteread.go

@@ -66,10 +66,8 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	errCh := make(chan error)
 
 	var wg sync.WaitGroup
-	wg.Add(rrp.cc)
-	for i := 0; i < rrp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range rrp.cc {
+		wg.Go(func() {
 			for r := range rangeC {
 				if err := rrp.do(ctx, r); err != nil {
 					errCh <- fmt.Errorf("request failed for: %s", err)
@@ -77,7 +75,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 				}
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	for _, r := range ranges {

app/vmctl/vm/vm.go

@@ -156,15 +156,13 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		cfg.BatchSize = 1e5
 	}
 
-	im.wg.Add(int(cfg.Concurrency))
-	for i := 0; i < int(cfg.Concurrency); i++ {
+	for i := range int(cfg.Concurrency) {
 		pbPrefix := fmt.Sprintf(`{{ green "VM worker %d:" }}`, i)
 		bar := barpool.AddWithTemplate(pbPrefix+pbTpl, 0)
 
-		go func(bar barpool.Bar) {
-			defer im.wg.Done()
+		im.wg.Go(func() {
 			im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
-		}(bar)
+		})
 	}
 	im.ResetStats()
 	return im, nil

app/vmctl/vm_native.go

@@ -249,9 +249,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var wg sync.WaitGroup
 	for i := 0; i < p.cc; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			for f := range filterCh {
 				if !p.disablePerMetricRequests {
 					if err := p.do(ctx, f, srcURL, dstURL, nil); err != nil {
@@ -266,7 +264,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 					}
 				}
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import

app/vminsert/common/streamaggr.go

@@ -111,9 +111,7 @@ func InitStreamAggr() {
 	saCfgTimestamp.Set(fasttime.UnixTimestamp())
 
 	// Start config reloader.
-	saCfgReloaderWG.Add(1)
-	go func() {
-		defer saCfgReloaderWG.Done()
+	saCfgReloaderWG.Go(func() {
 		for {
 			select {
 			case <-sighupCh:
@@ -122,7 +120,7 @@ func InitStreamAggr() {
 			}
 			reloadStreamAggrConfig()
 		}
-	}()
+	})
 }
 
 func reloadStreamAggrConfig() {

app/vmselect/graphite/transform.go

@@ -3896,27 +3896,9 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 	seriesCh := make(chan *series, goroutines)
 	errCh := make(chan error, 1)
 	var wg sync.WaitGroup
-	wg.Add(goroutines)
-	go func() {
-		var err error
-		for {
-			s, e := nextSeries()
-			if e != nil || s == nil {
-				err = e
-				break
-			}
-			seriesCh <- s
-		}
-		close(seriesCh)
-		wg.Wait()
-		close(resultCh)
-		errCh <- err
-		close(errCh)
-	}()
 	var skipProcessing atomic.Bool
-	for i := 0; i < goroutines; i++ {
-		go func() {
-			defer wg.Done()
+	for range goroutines {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if skipProcessing.Load() {
 					continue
@@ -3934,8 +3916,24 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 					}
 				}
 			}
-		}()
+		})
 	}
+	go func() {
+		var err error
+		for {
+			s, e := nextSeries()
+			if e != nil || s == nil {
+				err = e
+				break
+			}
+			seriesCh <- s
+		}
+		close(seriesCh)
+		wg.Wait()
+		close(resultCh)
+		errCh <- err
+		close(errCh)
+	}()
 	wrapper := func() (*series, error) {
 		r := <-resultCh
 		if r == nil {

app/vmselect/netstorage/netstorage.go

@@ -296,14 +296,12 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := range workChs {
-		wg.Add(1)
-		qtChild := qt.NewChild("worker #%d", i)
-		go func(workerID uint) {
-			timeseriesWorker(qtChild, workChs, workerID)
+	for workerID := range workChs {
+		qtChild := qt.NewChild("worker #%d", workerID)
+		wg.Go(func() {
+			timeseriesWorker(qtChild, workChs, uint(workerID))
 			qtChild.Done()
-			wg.Done()
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 
@@ -514,12 +512,10 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := 0; i < workers; i++ {
-		wg.Add(1)
-		go func(workerID uint) {
-			unpackWorker(workChs, workerID)
-			wg.Done()
-		}(uint(i))
+	for workerID := range workers {
+		wg.Go(func() {
+			unpackWorker(workChs, uint(workerID))
+		})
 	}
 	wg.Wait()
 
@@ -1020,12 +1016,10 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 		mustStop      atomic.Bool
 	)
 	var wg sync.WaitGroup
-	wg.Add(gomaxprocs)
-	for i := 0; i < gomaxprocs; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range gomaxprocs {
+		wg.Go(func() {
 			for xw := range workCh {
-				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
+				if err := f(&xw.mn, &xw.b, tr, uint(workerID)); err != nil {
 					errGlobalLock.Lock()
 					if errGlobal == nil {
 						errGlobal = err
@@ -1036,7 +1030,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 				xw.reset()
 				exportWorkPool.Put(xw)
 			}
-		}(uint(i))
+		})
 	}
 
 	// Feed workers with work

app/vmselect/promql/aggr_incremental_test.go

@@ -103,15 +103,13 @@ func testIncrementalParallelAggr(iafc *incrementalAggrFuncContext, tssSrc, tssEx
 	workersCount := netstorage.MaxWorkers()
 	tsCh := make(chan *timeseries)
 	var wg sync.WaitGroup
-	wg.Add(workersCount)
-	for i := 0; i < workersCount; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workersCount {
+		wg.Go(func() {
 			for ts := range tsCh {
 				runtime.Gosched() // allow other goroutines performing the work
-				iafc.updateTimeseries(ts, workerID)
+				iafc.updateTimeseries(ts, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	for _, ts := range tssSrc {
 		tsCh <- ts

app/vmselect/promql/eval.go

@@ -477,22 +477,18 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		var tssFirst []*timeseries
 		var errFirst error
 		qtFirst := qt.NewChild("expr1")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
 			qtFirst.Done()
-		}()
+		})
 
 		var tssSecond []*timeseries
 		var errSecond error
 		qtSecond := qt.NewChild("expr2")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
 			qtSecond.Done()
-		}()
+		})
 
 		wg.Wait()
 		if errFirst != nil {
@@ -710,17 +706,13 @@ func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.
 	qt.Printf("eval function args in parallel")
 	var wg sync.WaitGroup
 	for i, e := range es {
-		wg.Add(1)
 		qtChild := qt.NewChild("eval arg %d", i)
-		go func(e metricsql.Expr, i int) {
-			defer func() {
-				qtChild.Done()
-				wg.Done()
-			}()
+		wg.Go(func() {
+			defer qtChild.Done()
 			rv, err := evalExpr(qtChild, ec, e)
 			rvs[i] = rv
 			errs[i] = err
-		}(e, i)
+		})
 	}
 	wg.Wait()
 	for _, err := range errs {
@@ -1019,16 +1011,14 @@ func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, time
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workers {
+		wg.Go(func() {
 			var tmpValues []float64
 			var tmpTimestamps []int64
 			for ts := range workChs[workerID] {
-				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, workerID)
+				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 }

app/vmselect/promql/rollup.go

@@ -534,7 +534,10 @@ type rollupFuncArg struct {
 	timestamps []int64
 
 	// Real value preceding values.
-	// Is populated if preceding value is within the rc.LookbackDelta.
+	// Is populated if the preceding sample falls within the rc.LookbackDelta range, or if rc.LookbackDelta is not set.
+	//
+	// It provides an additional check and value for rollup functions such as increase(), changes(),
+	// when the prevValue is NaN due to a gap or a small lookback window.
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -713,7 +716,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	// Extend dstValues in order to remove mallocs below.
 	dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
 
-	// Use step as the scrape interval for instant queries (when start == end).
+	// Set maxPrevInterval for subsequent rfa.prevValue calculations in rollupFunc:
+	// For instant queries, use rc.Step directly as maxPrevInterval.
+	// For range queries, rc.Step is typically too small to serve as the lookback window between two rollup points.
+	// Instead, estimate the scrape interval from raw sample timestamps (using the 0.6 quantile of the last 20 intervals)
+	// and slightly inflate the scrape interval to set maxPrevInterval, allowing for some tolerance to jitter.
 	maxPrevInterval := rc.Step
 	if rc.Start < rc.End {
 		scrapeInterval := getScrapeInterval(timestamps, rc.Step)
@@ -729,22 +736,21 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		}
 	}
 	window := rc.Window
+	// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
+	// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
+	// when the window doesn't cover at least two raw samples (this is what most users expect).
+	//
+	// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
+	// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 	if window <= 0 {
 		window = rc.Step
 		if rc.MayAdjustWindow && window < maxPrevInterval {
-			// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
-			// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
-			// when the window doesn't cover at least two raw samples (this is what most users expect).
-			//
-			// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
-			// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
-			//
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 			window = maxPrevInterval
 		}
+		// Artificial window cannot exceed explicit rc.LookbackDelta, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 		if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
-			// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
-			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 			window = rc.LookbackDelta
 		}
 	}
@@ -869,17 +875,17 @@ func getScrapeInterval(timestamps []int64, defaultInterval int64) int64 {
 		return defaultInterval
 	}
 
-	// Estimate scrape interval as 0.6 quantile for the first 20 intervals.
-	tsPrev := timestamps[0]
-	timestamps = timestamps[1:]
+	// Estimate scrape interval as 0.6 quantile of the last 20 intervals.
+	tsPrev := timestamps[len(timestamps)-1]
+	timestamps = timestamps[:len(timestamps)-1]
 	if len(timestamps) > 20 {
-		timestamps = timestamps[:20]
+		timestamps = timestamps[len(timestamps)-20:]
 	}
 	a := getFloat64s()
 	intervals := a.A[:0]
-	for _, ts := range timestamps {
-		intervals = append(intervals, float64(ts-tsPrev))
-		tsPrev = ts
+	for i := len(timestamps) - 1; i >= 0; i-- {
+		intervals = append(intervals, float64(tsPrev-timestamps[i]))
+		tsPrev = timestamps[i]
 	}
 	scrapeInterval := int64(quantile(0.6, intervals))
 	a.A = intervals
@@ -2107,9 +2113,15 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		prevValue = values[0]
-		values = values[1:]
-		n++
+		// Assume that the value didn't change during the current gap
+		// if realPrevValue exists.
+		if !math.IsNaN(rfa.realPrevValue) {
+			prevValue = rfa.realPrevValue
+		} else {
+			n++
+			prevValue = values[0]
+			values = values[1:]
+		}
 	}
 	for _, v := range values {
 		if v != prevValue {

app/vmselect/promql/rollup_test.go

@@ -232,6 +232,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
 	}
 	var rfa rollupFuncArg
 	rfa.prevValue = nan
+	rfa.realPrevValue = nan
 	rfa.prevTimestamp = 0
 	rfa.values = append(rfa.values, testValues...)
 	rfa.timestamps = append(rfa.timestamps, testTimestamps...)
@@ -1654,7 +1655,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1674,7 +1675,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1794,7 +1795,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1814,7 +1815,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1888,3 +1889,126 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
+
+func TestRollupChangesWithStaleness(t *testing.T) {
+	// there is a gap between samples in the dataset below
+	timestamps := []int64{0, 15000, 30000, 70000}
+	values := []float64{1, 1, 1, 1}
+
+	// if step > gap, then changes will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta>0 then changes will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// there is a staleness marker between samples in the dataset below
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{1, 1, 1, decimal.StaleNaN, 1}
+
+	t.Run("staleness marker", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                40000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 1, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280
+	//
+	// When there are gaps between samples that exceed maxPrevInterval,
+	// either due to changes in the scrape interval or missing scrapes.
+	// For example, if the scrape interval was initially 30s and later changed to 10s,
+	// the auto-calculated scrape interval is 10s, with maxPrevInterval inflated to 15s.
+	//
+	// At t=30s:
+	// prevValue is NaN, as the last sample at t=0s is considered stale for t=30s given the maxPrevInterval.
+	// realPrevValue is 1, taken from t=0s, since LookbackDelta=0 ignores staleness.
+	// the result should be `changes(1, 1) -> 0` instead of `changes(1, NaN)`.
+	// At t=100s:
+	// preValue is also NaN, as the last sample at t=70s is considered stale for t=100s.
+	// realPrevValue is 1, taken from t=70s,
+	// result should be `changes(2, 1) -> 1`.
+	timestamps = []int64{0, 30000, 40000, 50000, 60000, 70000, 100000}
+	values = []float64{1, 1, 1, 1, 1, 1, 2}
+	t.Run("issue-10280", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                100e3,
+			Step:               10e3,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, _ := rc.Do(nil, values, timestamps)
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3, 80e3, 90e3, 100e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}

app/vmselect/promql/transform_test.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"fmt"
+	"math/rand"
 	"reflect"
 	"strconv"
 	"strings"
@@ -280,6 +281,87 @@ func timeseriesToPromMetrics(tss []*timeseries) string {
 	return strings.Join(a, "\n")
 }
 
+func TestTransformFuncSort(t *testing.T) {
+	f := func(isDesc bool, metrics, expectedMetrics string) {
+		t.Helper()
+
+		tss := promMetricsToTimeseries(metrics)
+
+		// Input tss order is not stable in VictoriaMetrics
+		// Shuffle tss to reflect that
+		// Commenting out the shuffle to make the test stable
+		rand.Shuffle(len(tss), func(i, j int) {
+			tss[i], tss[j] = tss[j], tss[i]
+		})
+
+		sortFunc := newTransformFuncSort(isDesc)
+		sorted, err := sortFunc(&transformFuncArg{
+			args: [][]*timeseries{tss},
+		})
+		if err != nil {
+			t.Fatalf("sort failed: %s", err)
+		}
+
+		result := timeseriesToPromMetrics(sorted)
+		if result != expectedMetrics {
+			t.Fatalf("unexpected sort result:\ngot\n%s\nwant\n%s", result, expectedMetrics)
+		}
+	}
+
+	// Test asc sort with different values
+	f(
+		false,
+		`foo{label="a"} 3 123
+foo{label="b"} 2 123
+foo{label="c"} 1 123`,
+		`foo{label="c"} 1 123
+foo{label="b"} 2 123
+foo{label="a"} 3 123`,
+	)
+
+	// Test desc sort with different values
+	f(
+		true,
+		`foo{label="a"} 3 123
+foo{label="b"} 2 123
+foo{label="c"} 1 123`,
+		`foo{label="a"} 3 123
+foo{label="b"} 2 123
+foo{label="c"} 1 123`,
+	)
+
+	// Test asc sort with mixed values
+	f(
+		false,
+		`foo{label="a"} 1 123
+	foo{label="b"} 1 123
+	foo{label="c"} 2 123
+	foo{label="d"} 2 123
+	foo{label="e"} 3 123
+	`,
+		`foo{label="a"} 1 123
+foo{label="b"} 1 123
+foo{label="c"} 2 123
+foo{label="d"} 2 123
+foo{label="e"} 3 123`,
+	)
+
+	// Test desc sort with mixed values
+	f(
+		true,
+		`foo{label="a"} 1 123
+	foo{label="b"} 1 123
+	foo{label="c"} 2 123
+	foo{label="d"} 2 123
+	foo{label="e"} 3 123`,
+		`foo{label="e"} 3 123
+foo{label="c"} 2 123
+foo{label="d"} 2 123
+foo{label="a"} 1 123
+foo{label="b"} 1 123`,
+	)
+}
+
 func TestGetNumPrefix(t *testing.T) {
 	f := func(s, prefixExpected string) {
 		t.Helper()

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-Clpj_g75.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-D5YL0cqB.js">
+  <script type="module" crossorigin src="./assets/index-B6lol36n.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-jEWkrqzO.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-VQRcNK83.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -470,9 +470,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 		return
 	}
 	snapshotsMaxAgeDur := snapshotsMaxAge.Duration()
-	staleSnapshotsRemoverWG.Add(1)
-	go func() {
-		defer staleSnapshotsRemoverWG.Done()
+	staleSnapshotsRemoverWG.Go(func() {
 		d := timeutil.AddJitterToDuration(time.Second * 11)
 		t := time.NewTicker(d)
 		defer t.Stop()
@@ -484,7 +482,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 			}
 			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
 		}
-	}()
+	})
 }
 
 func stopStaleSnapshotsRemover() {

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.5 AS build-web-stage
+FROM golang:1.25.6 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/Makefile

@@ -1,26 +1,26 @@
 # All these commands must run from repository root.
-
 copy-metricsql-docs:
 	cp docs/victoriametrics/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
 
 vmui-package-base-image:
 	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
 
-vmui-build: copy-metricsql-docs vmui-package-base-image
+vmui-run-npm-command: vmui-package-base-image
 	docker run --rm \
 		--user $(shell id -u):$(shell id -g) \
 		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
 		-w /build/packages/vmui \
 		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build"
+		vmui-builder-image -c "[ \"$$VMUI_SKIP_INSTALL\" = \"true\" ] || npm ci; $(NPM_COMMAND)"
 
-vmui-anomaly-build: vmui-package-base-image
-	docker run --rm \
-		--user $(shell id -u):$(shell id -g) \
-		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
-		-w /build/packages/vmui \
-		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build:anomaly"
+vmui-install:
+	NPM_COMMAND="true" $(MAKE) vmui-run-npm-command
+
+vmui-package-base-image:
+	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
+
+vmui-build: copy-metricsql-docs
+	NPM_COMMAND="npm run build" $(MAKE) vmui-run-npm-command
 
 vmui-release: vmui-build
 	docker build -t ${DOCKER_NAMESPACE}/vmui:latest -f app/vmui/Dockerfile-web ./app/vmui/packages/vmui
@@ -38,11 +38,11 @@ vmui-update: vmui-build
 vmui-install-dependencies:
 	cd app/vmui/packages/vmui && npm ci
 
-vmui-lint: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run lint
+vmui-lint:
+	NPM_COMMAND="npm run lint" $(MAKE) vmui-run-npm-command
 
-vmui-typecheck: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run typecheck
+vmui-typecheck:
+	NPM_COMMAND="npm run typecheck" $(MAKE) vmui-run-npm-command
 
-vmui-test: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run test
+vmui-test:
+	NPM_COMMAND="npm run test" $(MAKE) vmui-run-npm-command

app/vmui/packages/vmui/.env.vmanomaly

@@ -1 +0,0 @@
-VITE_APP_TYPE=vmanomaly

app/vmui/packages/vmui/config/plugins/dynamicIndexHtml.ts

@@ -1,23 +0,0 @@
-import { readFile } from "fs/promises";
-import { IndexHtmlTransform } from "vite";
-
-/**
- * Vite plugin to dynamically load index.html based on the current mode.
- * If a specific mode-based index file (e.g., index.vmanomaly.html) exists, it is used.
- * Otherwise, the default index.html is loaded.
- */
-export default function dynamicIndexHtmlPlugin({ mode }) {
-  return {
-    name: "vm-dynamic-index-html",
-    transformIndexHtml: {
-      order: "pre",
-      handler: async () => {
-        try {
-          return await readFile(`./index.${mode}.html`, "utf8");
-        } catch (error) {
-          return await readFile("./index.html", "utf8");
-        }
-      }
-    } as IndexHtmlTransform
-  };
-}

app/vmui/packages/vmui/index.vmanomaly.html

@@ -1,54 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="/favicon.svg" />
-  <link rel="apple-touch-icon" href="/favicon.svg" />
-  <link rel="mask-icon" href="/favicon.svg" color="#000000">
-
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta name="description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI"/>
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
-  <!--
-    Notice the use of in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>UI for VictoriaMetrics Anomaly Detection</title>
-
-  <meta name="twitter:card" content="summary">
-  <meta name="twitter:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta name="twitter:site" content="@https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta name="twitter:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-  <meta name="twitter:image" content="/preview.jpg">
-
-  <meta property="og:type" content="website">
-  <meta property="og:title" content="UI for VictoriaMetrics Anomaly Detection">
-  <meta property="og:url" content="https://victoriametrics.com/products/enterprise/anomaly-detection/">
-  <meta property="og:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-<script type="module" src="/src/index.tsx"></script>
-</body>
-</html>

app/vmui/packages/vmui/package.json

@@ -7,10 +7,8 @@
   "scripts": {
     "prestart": "npm run copy-metricsql-docs",
     "start": "vite",
-    "start:playground": "cross-env PLAYGROUND=METRICS npm run start",
-    "start:anomaly": "vite --mode vmanomaly",
+    "start:playground": "cross-env PLAYGROUND=true npm run start",
     "build": "vite build",
-    "build:anomaly": "vite build --mode vmanomaly",
     "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",

app/vmui/packages/vmui/src/AppAnomaly.tsx

@@ -1,41 +0,0 @@
-import { FC, useState } from "preact/compat";
-import { HashRouter, Route, Routes } from "react-router-dom";
-import AppContextProvider from "./contexts/AppContextProvider";
-import ThemeProvider from "./components/Main/ThemeProvider/ThemeProvider";
-import AnomalyLayout from "./layouts/AnomalyLayout/AnomalyLayout";
-import ExploreAnomaly from "./pages/ExploreAnomaly/ExploreAnomaly";
-import router from "./router";
-import CustomPanel from "./pages/CustomPanel";
-
-const AppAnomaly: FC = () => {
-  const [loadedTheme, setLoadedTheme] = useState(false);
-
-  return <>
-    <HashRouter>
-      <AppContextProvider>
-        <>
-          <ThemeProvider onLoaded={setLoadedTheme}/>
-          {loadedTheme && (
-            <Routes>
-              <Route
-                path={"/"}
-                element={<AnomalyLayout/>}
-              >
-                <Route
-                  path={"/"}
-                  element={<ExploreAnomaly/>}
-                />
-                <Route
-                  path={router.query}
-                  element={<CustomPanel/>}
-                />
-              </Route>
-            </Routes>
-          )}
-        </>
-      </AppContextProvider>
-    </HashRouter>
-  </>;
-};
-
-export default AppAnomaly;

app/vmui/packages/vmui/src/components/Chart/Line/Legend/Legend.tsx

@@ -14,12 +14,11 @@ export type QueryGroup = {
 interface LegendProps {
   labels: LegendItemType[];
   query: string[];
-  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
 
-const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPanel, onChange }) => {
+const Legend: FC<LegendProps> = ({ labels, query, isPredefinedPanel, onChange }) => {
   const { groupByLabel } = useLegendGroup();
   const groupSeries = useGroupSeries({ labels, query, groupByLabel });
 
@@ -33,7 +32,6 @@ const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPan
             key={group}
             labels={items}
             group={group}
-            isAnomalyView={isAnomalyView}
             onChange={onChange}
           />
         ))}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -13,7 +13,6 @@ import { getFromStorage } from "../../../../utils/storage";
 
 export type LegendProps = {
   labels: LegendItemType[];
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
@@ -22,7 +21,7 @@ interface LegendGroupProps extends LegendProps {
   group: string | number;
 }
 
-const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onChange }) => {
+const LegendGroup: FC<LegendGroupProps> = ({ labels, group, onChange }) => {
   const { isTableView } = useLegendView();
   const { groupByLabel } = useLegendGroup();
   const copyToClipboard = useCopyToClipboard();
@@ -81,7 +80,6 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
       >
         <Content
           labels={sortedLabels}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -13,11 +13,10 @@ import { getLabelAlias } from "../../../../../utils/metric";
 interface LegendItemProps {
   legend: LegendItemType;
   onChange?: (item: LegendItemType, metaKey: boolean) => void;
-  isAnomalyView?: boolean;
   duplicateFields?: string[];
 }
 
-const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, isAnomalyView }) => {
+const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields }) => {
   const copyToClipboard = useCopyToClipboard();
   const { hideStats } = useShowStats();
 
@@ -52,12 +51,10 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, is
       })}
       onClick={createHandlerClick(legend)}
     >
-      {!isAnomalyView && (
-        <div
-          className="vm-legend-item__marker"
-          style={{ backgroundColor: legend.color }}
-        />
-      )}
+      <div
+        className="vm-legend-item__marker"
+        style={{ backgroundColor: legend.color }}
+      />
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
           {legend.hasAlias && legend.label}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendViews/LegendLines.tsx

@@ -2,7 +2,7 @@ import { FC } from "preact/compat";
 import LegendItem from "../LegendItem/LegendItem";
 import { LegendProps } from "../LegendGroup";
 
-const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields, onChange }) => {
+const LegendLines: FC<LegendProps> = ({ labels, duplicateFields, onChange }) => {
 
   return (
     <div className="vm-legend-item-container">
@@ -10,7 +10,6 @@ const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields,
         <LegendItem
           key={legendItem.label}
           legend={legendItem}
-          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/LegendAnomaly.tsx

@@ -1,82 +0,0 @@
-import { FC, useMemo } from "preact/compat";
-import { ForecastType, SeriesItem } from "../../../../types";
-import { anomalyColors } from "../../../../utils/color";
-import "./style.scss";
-
-type Props = {
-  series: SeriesItem[];
-};
-
-const titles: Partial<Record<ForecastType, string>> = {
-  [ForecastType.yhat]: "yhat",
-  [ForecastType.yhatLower]: "yhat_upper - yhat_lower",
-  [ForecastType.yhatUpper]: "yhat_upper - yhat_lower",
-  [ForecastType.anomaly]: "anomalies",
-  [ForecastType.training]: "training data",
-  [ForecastType.actual]: "y"
-};
-
-const LegendAnomaly: FC<Props> = ({ series }) => {
-
-  const uniqSeriesStyles = useMemo(() => {
-    const uniqSeries = series.reduce((accumulator, currentSeries) => {
-      const hasForecast = Object.prototype.hasOwnProperty.call(currentSeries, "forecast");
-      const isNotUpper = currentSeries.forecast !== ForecastType.yhatUpper;
-      const isUniqForecast = !accumulator.find(s => s.forecast === currentSeries.forecast);
-      if (hasForecast && isUniqForecast && isNotUpper) {
-        accumulator.push(currentSeries);
-      }
-      return accumulator;
-    }, [] as SeriesItem[]);
-
-    const trainingSeries = {
-      ...uniqSeries[0],
-      forecast: ForecastType.training,
-      color: anomalyColors[ForecastType.training],
-    };
-    uniqSeries.splice(1, 0, trainingSeries);
-
-    return uniqSeries.map(s => ({
-      ...s,
-      color: typeof s.stroke === "string" ? s.stroke : anomalyColors[s.forecast || ForecastType.actual],
-    }));
-  }, [series]);
-
-  return <>
-    <div className="vm-legend-anomaly">
-      {/* TODO: remove .filter() after the correct training data has been added */}
-      {uniqSeriesStyles.filter(f => f.forecast !== ForecastType.training).map((s, i) => (
-        <div
-          key={`${i}_${s.forecast}`}
-          className="vm-legend-anomaly-item"
-        >
-          <svg>
-            {s.forecast === ForecastType.anomaly ? (
-              <circle
-                cx="15"
-                cy="7"
-                r="4"
-                fill={s.color}
-                stroke={s.color}
-                strokeWidth="1.4"
-              />
-            ) : (
-              <line
-                x1="0"
-                y1="7"
-                x2="30"
-                y2="7"
-                stroke={s.color}
-                strokeWidth={s.width || 1}
-                strokeDasharray={s.dash?.join(",")}
-              />
-            )}
-          </svg>
-          <div className="vm-legend-anomaly-item__title">{titles[s.forecast || ForecastType.actual]}</div>
-        </div>
-      ))}
-    </div>
-  </>;
-};
-
-export default LegendAnomaly;

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/style.scss

@@ -1,23 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-legend-anomaly {
-  position: relative;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  flex-wrap: wrap;
-  gap: calc($padding-large * 2);
-  cursor: default;
-
-  &-item {
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    gap: $padding-small;
-
-    svg {
-      width: 30px;
-      height: 14px;
-    }
-  }
-}

app/vmui/packages/vmui/src/components/Chart/Line/LineChart/LineChart.tsx

@@ -13,7 +13,6 @@ import {
   getRangeY,
   getScales,
   handleDestroy,
-  setBand,
   setSelect
 } from "../../../../utils/uplot";
 import { MetricResult } from "../../../../api/types";
@@ -40,7 +39,6 @@ export interface LineChartProps {
   setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
   layoutSize: ElementSize;
   height?: number;
-  isAnomalyView?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
 }
@@ -55,7 +53,6 @@ const LineChart: FC<LineChartProps> = ({
   setPeriod,
   layoutSize,
   height,
-  isAnomalyView,
   spanGaps = false,
   showAllPoints = false,
 }) => {
@@ -75,7 +72,7 @@ const LineChart: FC<LineChartProps> = ({
     seriesFocus,
     setCursor,
     resetTooltips
-  } = useLineTooltip({ u: uPlotInst, metrics, series, unit, isAnomalyView });
+  } = useLineTooltip({ u: uPlotInst, metrics, series, unit });
 
   const options: uPlotOptions = {
     ...getDefaultOptions({ width: layoutSize.width, height }),
@@ -111,7 +108,6 @@ const LineChart: FC<LineChartProps> = ({
     if (!uPlotInst) return;
     delSeries(uPlotInst);
     addSeries(uPlotInst, series, spanGaps, showAllPoints);
-    setBand(uPlotInst, series);
     uPlotInst.redraw();
   }, [series, spanGaps, showAllPoints]);
 

app/vmui/packages/vmui/src/components/ExploreAnomaly/AnomalyConfig.tsx

@@ -1,132 +0,0 @@
-import { FC, useState } from "preact/compat";
-import Button from "../Main/Button/Button";
-import TextField from "../Main/TextField/TextField";
-import Modal from "../Main/Modal/Modal";
-import Spinner from "../Main/Spinner/Spinner";
-import { DownloadIcon, ErrorIcon } from "../Main/Icons";
-import useBoolean from "../../hooks/useBoolean";
-import useDeviceDetect from "../../hooks/useDeviceDetect";
-import { useAppState } from "../../state/common/StateContext";
-import classNames from "classnames";
-import "./style.scss";
-import { useQueryState } from "../../state/query/QueryStateContext";
-import { useTimeState } from "../../state/time/TimeStateContext";
-import { getStepFromDuration } from "../../utils/time";
-
-const AnomalyConfig: FC = () => {
-  const { serverUrl } = useAppState();
-  const { isMobile } = useDeviceDetect();
-
-  const {
-    value: isModalOpen,
-    setTrue: setOpenModal,
-    setFalse: setCloseModal,
-  } = useBoolean(false);
-
-  const { query } = useQueryState();
-  const { period } = useTimeState();
-  const [isLoading, setIsLoading] = useState(false);
-  const [textConfig, setTextConfig] = useState<string>("");
-  const [downloadUrl, setDownloadUrl] = useState<string>("");
-  const [error, setError] = useState<string>("");
-
-  const fetchConfig = async () => {
-    setIsLoading(true);
-    try {
-      const queryParam = encodeURIComponent(query[0] || "");
-      const stepParam = encodeURIComponent(period.step || getStepFromDuration(period.end - period.start, false));
-
-      const url = `${serverUrl}/api/vmanomaly/config.yaml?query=${queryParam}&step=${stepParam}`;
-      const response = await fetch(url);
-      const contentType = response.headers.get("Content-Type");
-      if (!response.ok) {
-        const bodyText = await response.text();
-        setError(` ${response.status} ${response.statusText}: ${bodyText}`);
-      } else if (contentType == "application/yaml") {
-        const blob = await response.blob();
-        const yamlAsString = await blob.text();
-        setTextConfig(yamlAsString);
-        setDownloadUrl(URL.createObjectURL(blob));
-      } else {
-        setError("Response Content-Type is not YAML, does `Server URL` point to VMAnomaly server?");
-      }
-    } catch (error) {
-      console.error(error);
-      setError(String(error));
-    }
-    setIsLoading(false);
-  };
-
-  const handleOpenModal = () => {
-    setOpenModal();
-    setError("");
-    URL.revokeObjectURL(downloadUrl);
-    setTextConfig("");
-    setDownloadUrl("");
-    return fetchConfig();
-  };
-
-  return (
-    <>
-      <Button
-        color="secondary"
-        variant="outlined"
-        onClick={handleOpenModal}
-      >
-        Open Config
-      </Button>
-      {isModalOpen && (
-        <Modal
-          title="Download config"
-          onClose={setCloseModal}
-        >
-          <div
-            className={classNames({
-              "vm-anomaly-config": true,
-              "vm-anomaly-config_mobile": isMobile,
-            })}
-          >
-            {isLoading && (
-              <Spinner
-                containerStyles={{ position: "relative" }}
-                message={"Loading config..."}
-              />
-            )}
-            {!isLoading && error && (
-              <div className="vm-anomaly-config-error">
-                <div className="vm-anomaly-config-error__icon"><ErrorIcon/></div>
-                <h3 className="vm-anomaly-config-error__title">Cannot download config</h3>
-                <p className="vm-anomaly-config-error__text">{error}</p>
-              </div>
-            )}
-            {!isLoading && textConfig && (
-              <TextField
-                value={textConfig}
-                label={"config.yaml"}
-                type="textarea"
-                disabled={true}
-              />
-            )}
-            <div className="vm-anomaly-config-footer">
-              {downloadUrl && (
-                <a
-                  href={downloadUrl}
-                  download={"config.yaml"}
-                >
-                  <Button
-                    variant="contained"
-                    startIcon={<DownloadIcon/>}
-                  >
-                    download
-                  </Button>
-                </a>
-              )}
-            </div>
-          </div>
-        </Modal>
-      )}
-    </>
-  );
-};
-
-export default AnomalyConfig;

app/vmui/packages/vmui/src/components/ExploreAnomaly/style.scss

@@ -1,61 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-anomaly-config {
-  display: grid;
-  grid-template-rows: calc(($vh * 70) - 78px - ($padding-medium*3)) auto;
-  gap: $padding-global;
-  min-width: 400px;
-  max-width: 80vw;
-  min-height: 300px;
-
-  &_mobile {
-    width: 100%;
-    max-width: none;
-    min-height: 100%;
-    grid-template-rows: calc(($vh * 100) - 78px - ($padding-global*3)) auto;
-  }
-
-  textarea {
-    overflow: auto;
-    width: 100%;
-    height: 100%;
-    max-height: 900px;
-  }
-
-  &-error {
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    justify-content: center;
-    width: 100%;
-    gap: $padding-small;
-    text-align: center;
-
-    &__icon {
-      display: flex;
-      align-items: center;
-      justify-content: center;
-      width: 30px;
-      height: 30px;
-      margin-bottom: $padding-small;
-      color: $color-error;
-    }
-
-    &__title {
-      font-size: $font-size-medium;
-      font-weight: bold;
-    }
-
-    &__text {
-      max-width: 700px;
-      line-height: 1.3;
-    }
-  }
-
-  &-footer {
-    display: flex;
-    align-items: center;
-    justify-content: flex-end;
-    gap: $padding-small;
-  }
-}

app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx

@@ -12,7 +12,7 @@ import {
   getMinMaxBuffer,
   getTimeSeries,
 } from "../../../utils/uplot";
-import { TimeParams, SeriesItem, LegendItemType } from "../../../types";
+import { TimeParams, LegendItemType } from "../../../types";
 import { AxisRange, YaxisState } from "../../../state/graph/reducer";
 import { getMathStats } from "../../../utils/math";
 import classNames from "classnames";
@@ -23,8 +23,6 @@ import { promValueToNumber } from "../../../utils/metric";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useElementSize from "../../../hooks/useElementSize";
 import { ChartTooltipProps } from "../../Chart/ChartTooltip/ChartTooltip";
-import LegendAnomaly from "../../Chart/Line/LegendAnomaly/LegendAnomaly";
-import { groupByMultipleKeys } from "../../../utils/array";
 import { useGraphDispatch } from "../../../state/graph/GraphStateContext";
 import { sameTs } from "../../../utils/time";
 import { useLocation } from "react-router-dom";
@@ -44,7 +42,6 @@ export interface GraphViewProps {
   fullWidth?: boolean;
   height?: number;
   isHistogram?: boolean;
-  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
@@ -64,7 +61,6 @@ const GraphView: FC<GraphViewProps> = ({
   fullWidth = true,
   height,
   isHistogram,
-  isAnomalyView,
   isPredefinedPanel,
   spanGaps,
   showAllPoints
@@ -89,8 +85,8 @@ const GraphView: FC<GraphViewProps> = ({
   const [legendValue, setLegendValue] = useState<ChartTooltipProps | null>(null);
 
   const getSeriesItem = useMemo(() => {
-    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isAnomalyView, isRawQuery);
-  }, [data, hideSeries, alias, showAllPoints, isAnomalyView, isRawQuery]);
+    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isRawQuery);
+  }, [data, hideSeries, alias, showAllPoints, isRawQuery]);
 
   const setLimitsYaxis = (minVal: number, maxVal: number) => {
     let min = Number.isFinite(minVal) ? minVal : 0;
@@ -102,7 +98,7 @@ const GraphView: FC<GraphViewProps> = ({
   };
 
   const onChangeLegend = (legend: LegendItemType, metaKey: boolean) => {
-    setHideSeries(getHideSeries({ hideSeries, legend, metaKey, series, isAnomalyView }));
+    setHideSeries(getHideSeries({ hideSeries, legend, metaKey, series }));
   };
 
   const prepareHistogramData = (data: (number | null)[][]) => {
@@ -127,20 +123,6 @@ const GraphView: FC<GraphViewProps> = ({
     return [null, [xs, ys, counts]];
   };
 
-  const prepareAnomalyLegend = (legend: LegendItemType[]): LegendItemType[] => {
-    if (!isAnomalyView) return legend;
-
-    // For vmanomaly: Only select the first series per group (due to API specs) and clear __name__ in freeFormFields.
-    const grouped = groupByMultipleKeys(legend, ["group", "label"]);
-    return grouped.map((group) => {
-      const firstEl = group.values[0];
-      return {
-        ...firstEl,
-        freeFormFields: { ...firstEl.freeFormFields, __name__: "" }
-      };
-    });
-  };
-
   useEffect(() => {
     const dLen = data.length;
 
@@ -155,7 +137,7 @@ const GraphView: FC<GraphViewProps> = ({
 
     for (let i = 0; i < dLen; i++) {
       const d = data[i];
-      const seriesItem = getSeriesItem(d, i);
+      const seriesItem = getSeriesItem(d);
       tempSeries[i + 1] = seriesItem;
       tempLegend[i] = getLegendItem(seriesItem, d.group);
 
@@ -206,7 +188,7 @@ const GraphView: FC<GraphViewProps> = ({
       const avg = Math.abs(Number(avgRaw));
       const range = getMinMaxBuffer(min, max);
       const rangeStep = Math.abs(range[1] - range[0]);
-      const needStabilize = (avg > rangeStep * 1e10) && !isAnomalyView;
+      const needStabilize = (avg > rangeStep * 1e10);
 
       return needStabilize ? results.fill(avg) : results;
     });
@@ -214,13 +196,11 @@ const GraphView: FC<GraphViewProps> = ({
     timeDataSeries.unshift(timeSeries);
 
     const result = isHistogram ? prepareHistogramData(timeDataSeries) : timeDataSeries;
-    const legend = prepareAnomalyLegend(tempLegend);
 
     setLimitsYaxis(minVal, maxVal);
     setDataChart(result as uPlotData);
     setSeries(tempSeries);
-    setLegend(legend);
-    isAnomalyView && setHideSeries(legend.map(s => s.label || "").slice(1));
+    setLegend(tempLegend);
   }, [data, timezone, isHistogram, currentStep, isRawQuery]);
 
   useEffect(() => {
@@ -232,13 +212,13 @@ const GraphView: FC<GraphViewProps> = ({
 
     for (let i = 0; i < dLen; i++) {
       const d = data[i];
-      const seriesItem = getSeriesItem(d, i);
+      const seriesItem = getSeriesItem(d);
       tempSeries[i + 1] = seriesItem;
       tempLegend[i] = getLegendItem(seriesItem, d.group);
     }
 
     setSeries(tempSeries);
-    setLegend(prepareAnomalyLegend(tempLegend));
+    setLegend(tempLegend);
   }, [hideSeries]);
 
   const hasTimeData = dataChart[0]?.length > 0;
@@ -281,7 +261,6 @@ const GraphView: FC<GraphViewProps> = ({
           setPeriod={setPeriod}
           layoutSize={containerSize}
           height={height}
-          isAnomalyView={isAnomalyView}
           spanGaps={spanGaps}
           showAllPoints={isRawQuery ? true : showAllPoints}
         />
@@ -298,12 +277,10 @@ const GraphView: FC<GraphViewProps> = ({
           onChangeLegend={setLegendValue}
         />
       )}
-      {isAnomalyView && showLegend && (<LegendAnomaly series={series as SeriesItem[]}/>)}
       {!isHistogram && showLegend && (
         <Legend
           labels={legend}
           query={query}
-          isAnomalyView={isAnomalyView}
           onChange={onChangeLegend}
           isPredefinedPanel={isPredefinedPanel}
         />

app/vmui/packages/vmui/src/constants/appType.ts

@@ -1,8 +0,0 @@
-export enum AppType {
-  victoriametrics = "victoriametrics",
-  vmanomaly = "vmanomaly",
-}
-
-export const APP_TYPE = import.meta.env.VITE_APP_TYPE;
-export const APP_TYPE_VM = APP_TYPE === AppType.victoriametrics;
-export const APP_TYPE_ANOMALY = APP_TYPE === AppType.vmanomaly;

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -13,10 +13,9 @@ interface LineTooltipHook {
   metrics: MetricResult[];
   series: uPlotSeries[];
   unit?: string;
-  isAnomalyView?: boolean;
 }
 
-const useLineTooltip = ({ u, metrics, series, unit, isAnomalyView }: LineTooltipHook) => {
+const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
   const [showTooltip, setShowTooltip] = useState(false);
   const [tooltipIdx, setTooltipIdx] = useState({ seriesIdx: -1, dataIdx: -1 });
   const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipProps[]>([]);
@@ -79,7 +78,7 @@ const useLineTooltip = ({ u, metrics, series, unit, isAnomalyView }: LineTooltip
       point,
       u: u,
       id: `${seriesIdx}_${dataIdx}`,
-      title: groups.size > 1 && !isAnomalyView ? `Query ${group}` : "",
+      title: groups.size > 1 ? `Query ${group}` : "",
       dates: [date ? dayjs(date * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT) : "-"],
       value: formatPrettyNumber(value, min, max),
       info: getMetricName(metricItem, seriesItem),
@@ -87,7 +86,7 @@ const useLineTooltip = ({ u, metrics, series, unit, isAnomalyView }: LineTooltip
       marker: `${seriesItem?.stroke}`,
       duplicateCount,
     };
-  }, [u, tooltipIdx, metrics, series, unit, isAnomalyView]);
+  }, [u, tooltipIdx, metrics, series, unit]);
 
   const handleClick = useCallback(() => {
     if (!showTooltip) return;

app/vmui/packages/vmui/src/hooks/useFetchAppConfig.ts

@@ -1,7 +1,6 @@
 import { useAppDispatch, useAppState } from "../state/common/StateContext";
 import { useEffect, useState } from "preact/compat";
 import { ErrorTypes } from "../types";
-import { APP_TYPE_VM } from "../constants/appType";
 
 const useFetchAppConfig = () => {
   const { serverUrl } = useAppState();
@@ -12,7 +11,6 @@ const useFetchAppConfig = () => {
 
   useEffect(() => {
     const fetchAppConfig = async () => {
-      if (!APP_TYPE_VM) return;
       setError("");
       setIsLoading(true);
 

app/vmui/packages/vmui/src/hooks/useFetchDefaultTimezone.ts

@@ -5,7 +5,6 @@ import { useTimeDispatch } from "../state/time/TimeStateContext";
 import { getFromStorage } from "../utils/storage";
 import dayjs from "dayjs";
 import { getBrowserTimezone } from "../utils/time";
-import { APP_TYPE_VM } from "../constants/appType";
 
 const disabledDefaultTimezone = Boolean(getFromStorage("DISABLED_DEFAULT_TIMEZONE"));
 
@@ -29,7 +28,7 @@ const useFetchDefaultTimezone = () => {
   };
 
   const fetchDefaultTimezone = async () => {
-    if (!serverUrl || !APP_TYPE_VM) return;
+    if (!serverUrl) return;
     setError("");
     setIsLoading(true);
 

app/vmui/packages/vmui/src/hooks/useFetchQuery.ts

@@ -13,7 +13,6 @@ import { isHistogramData } from "../utils/metric";
 import { useGraphState } from "../state/graph/GraphStateContext";
 import { getStepFromDuration } from "../utils/time";
 import { getQueryStringValue } from "../utils/query-string";
-import { APP_TYPE_ANOMALY } from "../constants/appType";
 
 interface FetchQueryParams {
   predefinedQuery?: string[]
@@ -135,7 +134,7 @@ export const useFetchQuery = ({
           }
 
           const preventChangeType = !!getQueryStringValue("display_mode", null);
-          isHistogramResult = !APP_TYPE_ANOMALY && isDisplayChart && !preventChangeType && isHistogramData(resp.data.result);
+          isHistogramResult = isDisplayChart && !preventChangeType && isHistogramData(resp.data.result);
           seriesLimit = isHistogramResult ? Infinity : defaultLimit;
           const freeTempSize = Math.max(0, seriesLimit - tempData.length);
           resp.data.result.slice(0, freeTempSize).forEach((d: MetricBase) => {

app/vmui/packages/vmui/src/index.tsx

@@ -3,20 +3,9 @@ import "./constants/dayjsPlugins";
 import App from "./App";
 import reportWebVitals from "./reportWebVitals";
 import "./styles/style.scss";
-import { APP_TYPE, AppType } from "./constants/appType";
-import AppAnomaly from "./AppAnomaly";
-
-const getAppComponent = () => {
-  switch (APP_TYPE) {
-    case AppType.vmanomaly:
-      return <AppAnomaly/>;
-    default:
-      return <App/>;
-  }
-};
 
 const root = document.getElementById("root");
-if (root) render(getAppComponent(), root);
+if (root) render(<App/>, root);
 
 
 // If you want to start measuring performance in your app, pass a function

app/vmui/packages/vmui/src/layouts/AnomalyLayout/AnomalyLayout.tsx

@@ -1,50 +0,0 @@
-import Header from "../Header/Header";
-import { FC, useEffect } from "preact/compat";
-import { Outlet, useSearchParams } from "react-router-dom";
-import qs from "qs";
-import "../MainLayout/style.scss";
-import { getAppModeEnable } from "../../utils/app-mode";
-import classNames from "classnames";
-import Footer from "../Footer/Footer";
-import useFetchDefaultTimezone from "../../hooks/useFetchDefaultTimezone";
-import useDeviceDetect from "../../hooks/useDeviceDetect";
-import ControlsAnomalyLayout from "./ControlsAnomalyLayout";
-
-const AnomalyLayout: FC = () => {
-  const appModeEnable = getAppModeEnable();
-  const { isMobile } = useDeviceDetect();
-  const [searchParams, setSearchParams] = useSearchParams();
-
-  useFetchDefaultTimezone();
-
-  // for support old links with search params
-  const redirectSearchToHashParams = () => {
-    const { search, href } = window.location;
-    if (search) {
-      const query = qs.parse(search, { ignoreQueryPrefix: true });
-      Object.entries(query).forEach(([key, value]) => searchParams.set(key, value as string));
-      setSearchParams(searchParams);
-      window.location.search = "";
-    }
-    const newHref = href.replace(/\/\?#\//, "/#/");
-    if (newHref !== href) window.location.replace(newHref);
-  };
-
-  useEffect(redirectSearchToHashParams, []);
-
-  return <section className="vm-container">
-    <Header controlsComponent={ControlsAnomalyLayout}/>
-    <div
-      className={classNames({
-        "vm-container-body": true,
-        "vm-container-body_mobile": isMobile,
-        "vm-container-body_app": appModeEnable
-      })}
-    >
-      <Outlet/>
-    </div>
-    {!appModeEnable && <Footer/>}
-  </section>;
-};
-
-export default AnomalyLayout;

app/vmui/packages/vmui/src/layouts/AnomalyLayout/ControlsAnomalyLayout.tsx

@@ -1,43 +0,0 @@
-import { FC } from "preact/compat";
-import classNames from "classnames";
-import TenantsConfiguration
-  from "../../components/Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration";
-import StepConfigurator from "../../components/Configurators/StepConfigurator/StepConfigurator";
-import { TimeSelector } from "../../components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector";
-import CardinalityDatePicker from "../../components/Configurators/CardinalityDatePicker/CardinalityDatePicker";
-import { ExecutionControls } from "../../components/Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls";
-import GlobalSettings from "../../components/Configurators/GlobalSettings/GlobalSettings";
-import ShortcutKeys from "../../components/Main/ShortcutKeys/ShortcutKeys";
-import { ControlsProps } from "../Header/HeaderControls/HeaderControls";
-
-const ControlsAnomalyLayout: FC<ControlsProps> = ({
-  displaySidebar,
-  isMobile,
-  headerSetup,
-  accountIds,
-  closeModal,
-}) => {
-
-  return (
-    <div
-      className={classNames({
-        "vm-header-controls": true,
-        "vm-header-controls_mobile": isMobile,
-      })}
-    >
-      {headerSetup?.tenant && <TenantsConfiguration accountIds={accountIds || []}/>}
-      {headerSetup?.stepControl && <StepConfigurator/>}
-      {headerSetup?.timeSelector && <TimeSelector/>}
-      {headerSetup?.cardinalityDatePicker && <CardinalityDatePicker/>}
-      {headerSetup?.executionControls && <ExecutionControls
-        tooltip={headerSetup?.executionControls?.tooltip}
-        useAutorefresh={headerSetup?.executionControls?.useAutorefresh}
-        closeModal={closeModal}
-      />}
-      <GlobalSettings/>
-      {!displaySidebar && <ShortcutKeys/>}
-    </div>
-  );
-};
-
-export default ControlsAnomalyLayout;

app/vmui/packages/vmui/src/layouts/Header/Header.tsx

@@ -2,7 +2,7 @@ import { FC, useMemo } from "preact/compat";
 import { useNavigate } from "react-router-dom";
 import router from "../../router";
 import { getAppModeEnable, getAppModeParams } from "../../utils/app-mode";
-import { LogoAnomalyIcon, LogoIcon } from "../../components/Main/Icons";
+import { LogoIcon } from "../../components/Main/Icons";
 import { getCssVariable } from "../../utils/theme";
 import "./style.scss";
 import classNames from "classnames";
@@ -13,19 +13,10 @@ import HeaderControls, { ControlsProps } from "./HeaderControls/HeaderControls";
 import useDeviceDetect from "../../hooks/useDeviceDetect";
 import useWindowSize from "../../hooks/useWindowSize";
 import { ComponentType } from "react";
-import { APP_TYPE, AppType } from "../../constants/appType";
 
 export interface HeaderProps {
   controlsComponent: ComponentType<ControlsProps>
 }
-const Logo = () => {
-  switch (APP_TYPE) {
-    case AppType.vmanomaly:
-      return <LogoAnomalyIcon/>;
-    default:
-      return <LogoIcon/>;
-  }
-};
 
 const Header: FC<HeaderProps> = ({ controlsComponent }) => {
   const { isMobile } = useDeviceDetect();
@@ -75,7 +66,7 @@ const Header: FC<HeaderProps> = ({ controlsComponent }) => {
       onClick={onClickLogo}
       style={{ color }}
     >
-      {<Logo/>}
+      {<LogoIcon/>}
     </div>
 
     {displaySidebar ? (

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -23,7 +23,6 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   totalSeries = 0,
   totalSeriesPrev = 0,
   totalSeriesAll = 0,
-  seriesCountByMetricName = [],
   metricNameStats,
   isPrometheus,
 }) => {
@@ -34,7 +33,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   const focusLabel = searchParams.get("focusLabel");
   const isMetric = /__name__/.test(match || "");
 
-  const progress = seriesCountByMetricName[0]?.value / totalSeriesAll * 100;
+  const progress = totalSeries / totalSeriesAll * 100;
   const diff = totalSeries - totalSeriesPrev;
   const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
 

app/vmui/packages/vmui/src/pages/CustomPanel/CustomPanelTabs/GraphTab.tsx

@@ -13,10 +13,9 @@ type Props = {
   isHistogram: boolean;
   graphData: MetricResult[];
   controlsRef: RefObject<HTMLDivElement>;
-  isAnomalyView?: boolean;
 }
 
-const GraphTab: FC<Props> = ({ isHistogram, graphData, controlsRef, isAnomalyView }) => {
+const GraphTab: FC<Props> = ({ isHistogram, graphData, controlsRef }) => {
   const { isMobile } = useDeviceDetect();
 
   const { customStep, yaxis, spanGaps, showAllPoints } = useGraphState();
@@ -74,7 +73,6 @@ const GraphTab: FC<Props> = ({ isHistogram, graphData, controlsRef, isAnomalyVie
         setPeriod={setPeriod}
         height={isMobile ? window.innerHeight * 0.5 : 500}
         isHistogram={isHistogram}
-        isAnomalyView={isAnomalyView}
         spanGaps={spanGaps}
         showAllPoints={showAllPoints}
       />

app/vmui/packages/vmui/src/pages/CustomPanel/QueryConfigurator/QueryConfigurator.tsx

@@ -26,7 +26,6 @@ import useSearchParamsFromObject from "../../../hooks/useSearchParamsFromObject"
 import { QueryStats } from "../../../api/types";
 import { usePrettifyQuery } from "./hooks/usePrettifyQuery";
 import QueryHistory from "../../../components/QueryHistory/QueryHistory";
-import AnomalyConfig from "../../../components/ExploreAnomaly/AnomalyConfig";
 import QueryEditorAutocomplete from "../../../components/Configurators/QueryEditor/QueryEditorAutocomplete";
 import { getUpdatedHistory } from "../../../components/QueryHistory/utils";
 
@@ -46,7 +45,6 @@ export interface QueryConfiguratorProps {
     prettify?: boolean;
     autocomplete?: boolean;
     traceQuery?: boolean;
-    anomalyConfig?: boolean;
     disableCache?: boolean;
     reduceMemUsage?: boolean;
   }
@@ -278,7 +276,6 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
           handleSelectQuery={handleSelectHistory}
           historyKey={"METRICS_QUERY_HISTORY"}
         />
-        {hideButtons?.anomalyConfig && <AnomalyConfig/>}
         {!hideButtons?.addQuery && stateQuery.length < MAX_QUERY_FIELDS && (
           <Button
             variant="outlined"

app/vmui/packages/vmui/src/pages/ExploreAnomaly/ExploreAnomaly.tsx

@@ -1,135 +0,0 @@
-import { FC, useMemo, useRef, useState } from "preact/compat";
-import classNames from "classnames";
-import useDeviceDetect from "../../hooks/useDeviceDetect";
-import { ForecastType } from "../../types";
-import { useSetQueryParams } from "../CustomPanel/hooks/useSetQueryParams";
-import QueryConfigurator from "../CustomPanel/QueryConfigurator/QueryConfigurator";
-import "../CustomPanel/style.scss";
-import { useQueryState } from "../../state/query/QueryStateContext";
-import { useFetchQuery } from "../../hooks/useFetchQuery";
-import { useGraphState } from "../../state/graph/GraphStateContext";
-import Spinner from "../../components/Main/Spinner/Spinner";
-import Alert from "../../components/Main/Alert/Alert";
-import WarningLimitSeries from "../CustomPanel/WarningLimitSeries/WarningLimitSeries";
-import GraphTab from "../CustomPanel/CustomPanelTabs/GraphTab";
-import { extractFields, isForecast } from "../../utils/uplot";
-import { MetricResult } from "../../api/types";
-import { promValueToNumber } from "../../utils/metric";
-
-// Hardcoded to 1.0 for now; consider adding a UI slider for threshold adjustment in the future.
-const ANOMALY_SCORE_THRESHOLD = 1;
-
-const ExploreAnomaly: FC = () => {
-  useSetQueryParams();
-  const { isMobile } = useDeviceDetect();
-
-  const { query } = useQueryState();
-  const { customStep } = useGraphState();
-
-  const controlsRef = useRef<HTMLDivElement>(null);
-
-  const [hideQuery] = useState<number[]>([]);
-  const [hideError, setHideError] = useState(!query[0]);
-  const [showAllSeries, setShowAllSeries] = useState(false);
-
-  const {
-    isLoading,
-    graphData,
-    error,
-    queryErrors,
-    setQueryErrors,
-    queryStats,
-    warning,
-  } = useFetchQuery({
-    visible: true,
-    customStep,
-    hideQuery,
-    showAllSeries
-  });
-
-  const data = useMemo(() => {
-    if (!graphData) return [];
-    const detectedData = graphData.map(d => ({ ...isForecast(d.metric), ...d }));
-    const realData = detectedData.filter(d => d.value === ForecastType.actual);
-    const anomalyScoreData = detectedData.filter(d => d.value === ForecastType.anomaly);
-    const anomalyData: MetricResult[] = realData.map((d) => {
-      const id = extractFields(d.metric);
-      const anomalyScoreDataByLabels = anomalyScoreData.find(du => extractFields(du.metric) === id);
-
-      return {
-        group: 1,
-        metric: { ...d.metric, __name__: ForecastType.anomaly },
-        values: d.values.filter(([t]) => {
-          if (!anomalyScoreDataByLabels) return false;
-          const anomalyScore = anomalyScoreDataByLabels.values.find(([tMax]) => tMax === t) as [number, string];
-          return anomalyScore && promValueToNumber(anomalyScore[1]) > ANOMALY_SCORE_THRESHOLD;
-        })
-      };
-    });
-    const filterData = detectedData.filter(d => (d.value !== ForecastType.anomaly) && d.value) as MetricResult[];
-    return filterData.concat(anomalyData);
-  }, [graphData]);
-
-  const handleRunQuery = () => {
-    setHideError(false);
-  };
-
-  return (
-    <div
-      className={classNames({
-        "vm-custom-panel": true,
-        "vm-custom-panel_mobile": isMobile,
-      })}
-    >
-      <QueryConfigurator
-        queryErrors={!hideError ? queryErrors : []}
-        setQueryErrors={setQueryErrors}
-        setHideError={setHideError}
-        stats={queryStats}
-        onRunQuery={handleRunQuery}
-        hideButtons={{
-          addQuery: true,
-          prettify: false,
-          autocomplete: false,
-          traceQuery: true,
-          anomalyConfig: true,
-          reduceMemUsage: true,
-        }}
-      />
-      {isLoading && <Spinner/>}
-      {(!hideError && error) && <Alert variant="error">{error}</Alert>}
-      {warning && (
-        <WarningLimitSeries
-          warning={warning}
-          query={query}
-          onChange={setShowAllSeries}
-        />
-      )}
-      <div
-        className={classNames({
-          "vm-custom-panel-body": true,
-          "vm-custom-panel-body_mobile": isMobile,
-          "vm-block": true,
-          "vm-block_mobile": isMobile,
-        })}
-      >
-        <div
-          className="vm-custom-panel-body-header"
-          ref={controlsRef}
-        >
-          <div/>
-        </div>
-        {data && (
-          <GraphTab
-            graphData={data}
-            isHistogram={false}
-            controlsRef={controlsRef}
-            isAnomalyView={true}
-          />
-        )}
-      </div>
-    </div>
-  );
-};
-
-export default ExploreAnomaly;

app/vmui/packages/vmui/src/pages/PredefinedPanels/hooks/useFetchDashboards.ts

@@ -3,7 +3,6 @@ import { DashboardSettings, ErrorTypes } from "../../../types";
 import { useAppState } from "../../../state/common/StateContext";
 import { useDashboardsDispatch } from "../../../state/dashboards/DashboardsStateContext";
 import { getAppModeEnable } from "../../../utils/app-mode";
-import { APP_TYPE_VM } from "../../../constants/appType";
 
 const importModule = async (filename: string) => {
   const data = await fetch(`./dashboards/${filename}`);
@@ -35,7 +34,7 @@ export const useFetchDashboards = (): {
   };
 
   const fetchRemoteDashboards = async () => {
-    if (!serverUrl || !APP_TYPE_VM) return;
+    if (!serverUrl) return;
     setError("");
     setIsLoading(true);
 

app/vmui/packages/vmui/src/router/index.ts

@@ -1,4 +1,3 @@
-import { APP_TYPE, AppType } from "../constants/appType";
 
 const router = {
   home: "/",
@@ -12,7 +11,6 @@ const router = {
   activeQueries: "/active-queries",
   queryAnalyzer: "/query-analyzer",
   icons: "/icons",
-  anomaly: "/anomaly",
   query: "/query",
   rawQuery: "/raw-query",
   downsamplingDebug: "/downsampling-filters-debug",
@@ -52,23 +50,11 @@ const routerOptionsDefault = {
   },
 };
 
-const getDefaultOptions = (appType: AppType) => {
-  switch (appType) {
-    case AppType.vmanomaly:
-      return {
-        title: "Anomaly exploration",
-        ...routerOptionsDefault,
-      };
-    default:
-      return {
-        title: "Query",
-        ...routerOptionsDefault,
-      };
-  }
-};
-
 export const routerOptions: { [key: string]: RouterOptions } = {
-  [router.home]: getDefaultOptions(APP_TYPE),
+  [router.home]: {
+    title: "Query",
+    ...routerOptionsDefault,
+  },
   [router.rawQuery]: {
     title: "Raw query",
     header: {
@@ -148,7 +134,6 @@ export const routerOptions: { [key: string]: RouterOptions } = {
     title: "Icons",
     header: {},
   },
-  [router.anomaly]: getDefaultOptions(AppType.vmanomaly),
   [router.query]: {
     title: "Query",
     ...routerOptionsDefault,

app/vmui/packages/vmui/src/router/navigation.ts

@@ -1,4 +1,4 @@
-import router, { routerOptions } from "./index";
+import router from "./index";
 
 export enum NavigationItemType {
   internalLink,
@@ -66,13 +66,3 @@ export const getDefaultNavigation = ({
   { value: router.dashboards, hide: !showPredefinedDashboards },
   { value: "Alerting", submenu: getAlertingNav(), hide: !showAlerting },
 ];
-
-/**
- * vmanomaly navigation menu
- */
-export const getAnomalyNavigation = (): NavigationItem[] => [
-  {
-    label: routerOptions[router.anomaly].title,
-    value: router.home,
-  },
-];

app/vmui/packages/vmui/src/router/useNavigationMenu.ts

@@ -3,8 +3,7 @@ import { useDashboardsState } from "../state/dashboards/DashboardsStateContext";
 import { useAppState } from "../state/common/StateContext";
 import { useMemo } from "preact/compat";
 import { processNavigationItems } from "./utils";
-import { getAnomalyNavigation, getDefaultNavigation } from "./navigation";
-import { APP_TYPE, AppType } from "../constants/appType";
+import { getDefaultNavigation } from "./navigation";
 
 const useNavigationMenu = () => {
   const appModeEnable = getAppModeEnable();
@@ -23,12 +22,7 @@ const useNavigationMenu = () => {
 
 
   const menu = useMemo(() => {
-    switch (APP_TYPE) {
-      case AppType.vmanomaly:
-        return getAnomalyNavigation();
-      default:
-        return getDefaultNavigation(navigationConfig);
-    }
+    return getDefaultNavigation(navigationConfig);
   }, [navigationConfig]);
 
   return processNavigationItems(menu);

app/vmui/packages/vmui/src/types/uplot.ts

@@ -1,15 +1,5 @@
 import { Axis, Series } from "uplot";
 
-export enum ForecastType {
-    yhat = "yhat",
-    yhatUpper = "yhat_upper",
-    yhatLower = "yhat_lower",
-    anomaly = "vmui_anomalies_points",
-    training = "vmui_training_data",
-    actual = "actual",
-    anomalyScore = "anomaly_score",
-}
-
 export interface SeriesItemStatsFormatted {
     min: string,
     max: string,
@@ -20,8 +10,6 @@ export interface SeriesItem extends Series {
     freeFormFields: {[key: string]: string};
     statsFormatted: SeriesItemStatsFormatted;
     median: number;
-    forecast?: ForecastType | null;
-    forecastGroup?: string;
     hasAlias?: boolean;
 }
 
@@ -30,7 +18,6 @@ export interface HideSeriesArgs {
     legend: LegendItemType,
     metaKey: boolean,
     series: Series[],
-    isAnomalyView?: boolean,
 }
 
 export type MinMax = { min: number, max: number }

app/vmui/packages/vmui/src/utils/array.ts

@@ -2,21 +2,6 @@ export const arrayEquals = (a: (string | number)[], b: (string | number)[]) => {
   return a.length === b.length && a.every((val, index) => val === b[index]);
 };
 
-export function groupByMultipleKeys<T>(items: T[], keys: (keyof T)[]): { keys: string[], values: T[] }[] {
-  const groups = items.reduce((result, item) => {
-    const compositeKey = keys.map(key => `${String(key)}: ${item[key] || "-"}`).join("|");
-
-    (result[compositeKey] = result[compositeKey] || []).push(item);
-
-    return result;
-  }, {} as { [key: string]: T[] });
-
-  return Object.entries(groups).map(([keyString, values]) => ({
-    keys: keyString.split("|"),
-    values
-  }));
-}
-
 export const isDecreasing = (arr: number[]): boolean => {
   if (arr.length < 2) return false;
 

app/vmui/packages/vmui/src/utils/color.ts

@@ -1,4 +1,4 @@
-import { ArrayRGB, ForecastType } from "../types";
+import { ArrayRGB } from "../types";
 
 export const baseContrastColors = [
   "#e54040",
@@ -21,16 +21,6 @@ export const hexToRGB = (hex: string): string => {
   return `${r}, ${g}, ${b}`;
 };
 
-export const anomalyColors: Record<ForecastType, string> = {
-  [ForecastType.yhatUpper]: "#7126a1",
-  [ForecastType.yhatLower]: "#7126a1",
-  [ForecastType.yhat]: "#da42a6",
-  [ForecastType.anomaly]: "#da4242",
-  [ForecastType.anomalyScore]: "#7126a1",
-  [ForecastType.actual]: "#203ea9",
-  [ForecastType.training]: `rgba(${hexToRGB("#203ea9")}, 0.2)`,
-};
-
 export const getColorFromString = (text: string): string => {
   const SEED = 16777215;
   const FACTOR = 49979693;

app/vmui/packages/vmui/src/utils/default-server-url.ts

@@ -1,5 +1,4 @@
 import { getAppModeParams } from "./app-mode";
-import { APP_TYPE, AppType } from "../constants/appType";
 import { getFromStorage } from "./storage";
 
 export const getDefaultURL = (u: string) => {
@@ -9,14 +8,6 @@ export const getDefaultURL = (u: string) => {
 export const getDefaultServer = (): string => {
   const { serverURL } = getAppModeParams();
   const storageURL = getFromStorage("SERVER_URL") as string;
-  const anomalyURL = `${window.location.origin}${window.location.pathname.replace(/^\/vmui/, "")}`;
   const defaultURL = getDefaultURL(window.location.href);
-  const url = serverURL || storageURL || defaultURL;
-
-  switch (APP_TYPE) {
-    case AppType.vmanomaly:
-      return storageURL || anomalyURL;
-    default:
-      return url;
-  }
+  return serverURL || storageURL || defaultURL;
 };

app/vmui/packages/vmui/src/utils/uplot/bands.ts

@@ -1,41 +0,0 @@
-import uPlot, { Series as uPlotSeries } from "uplot";
-import { ForecastType, SeriesItem } from "../../types";
-import { anomalyColors, hexToRGB } from "../color";
-
-export const setBand = (plot: uPlot, series: uPlotSeries[]) => {
-  // First, remove any existing bands
-  plot.delBand();
-
-  // If there aren't at least two series, we can't create a band
-  if (series.length < 2) return;
-
-  // Cast and enrich each series item with its index
-  const seriesItems = (series as SeriesItem[]).map((s, index) => ({ ...s, index }));
-
-  const upperSeries = seriesItems.filter(s => s.forecast === ForecastType.yhatUpper);
-  const lowerSeries = seriesItems.filter(s => s.forecast === ForecastType.yhatLower);
-
-  // Create bands by matching upper and lower series based on their freeFormFields
-  const bands = upperSeries.map((upper) => {
-    const correspondingLower = lowerSeries.find(lower => lower.forecastGroup === upper.forecastGroup);
-    if (!correspondingLower) return null;
-    return {
-      series: [upper.index, correspondingLower.index] as [number, number],
-      fill: createBandFill(ForecastType.yhatUpper),
-    };
-  }).filter(band => band !== null) as uPlot.Band[]; // Filter out any nulls from failed matches
-
-  // If there are no bands to add, exit the function
-  if (!bands.length) return;
-
-  // Add each band to the plot
-  bands.forEach(band => {
-    plot.addBand(band);
-  });
-};
-
-// Helper function to create the fill color for a band
-function createBandFill(forecastType: ForecastType): string {
-  const rgb = hexToRGB(anomalyColors[forecastType]);
-  return `rgba(${rgb}, 0.05)`;
-}

app/vmui/packages/vmui/src/utils/uplot/heatmap.ts

@@ -23,7 +23,18 @@ export const countsToFills = (u: uPlot, seriesIdx: number) => {
     }
   }
 
+  // no valid counts
+  if (!isFinite(minCount) || !isFinite(maxCount)) {
+    return counts.map(() => -1);
+  }
+
   const range = maxCount - minCount;
+
+  // all counts are the same
+  if (range === 0) {
+    return counts.map(c => (c > hideThreshold ? 0 : -1));
+  }
+
   const paletteSize = palette.length;
   const indexedFills = Array(counts.length);
 
@@ -40,9 +51,9 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
   const cellGap = Math.round(devicePixelRatio);
 
   uPlot.orient(u, seriesIdx, (
-    series,
-    dataX,
-    dataY,
+    _series,
+    _dataX,
+    _dataY,
     scaleX,
     scaleY,
     valToPosX,
@@ -51,8 +62,8 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
     yOff,
     xDim,
     yDim,
-    moveTo,
-    lineTo,
+    _moveTo,
+    _lineTo,
     rect
   ) => {
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -80,7 +91,7 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
     const cys = ys.slice(0, yBinQty).map((y: number) => {
       return Math.round(valToPosY(y, scaleY, yDim, yOff) - ySize / 2);
     });
-    const cxs = Array.from({ length: xBinQty }, (v, i) => {
+    const cxs = Array.from({ length: xBinQty }, (_v, i) => {
       return Math.round(valToPosX(xs[i * yBinQty], scaleX, xDim, xOff) - xSize);
     });
 
@@ -114,7 +125,7 @@ export const heatmapPaths = () => (u: uPlot, seriesIdx: number) => {
 export const convertPrometheusToVictoriaMetrics = (buckets: MetricResult[]): MetricResult[] => {
   if (!buckets.every(a => a.metric.le)) return buckets;
 
-  const sortedBuckets = buckets.sort((a,b) => parseFloat(a.metric.le) - parseFloat(b.metric.le));
+  const sortedBuckets = buckets.sort((a, b) => parseFloat(a.metric.le) - parseFloat(b.metric.le));
   const group = buckets[0]?.group || 1;
   let prevBucket: MetricResult = { metric: { le: "" }, values: [], group };
   const result: MetricResult[] = [];
@@ -169,5 +180,29 @@ export const normalizeData = (buckets: MetricResult[], isHistogram?: boolean): M
     return { ...bucket, values };
   }) as MetricResult[];
 
-  return result.filter(r => !r.values.every(v => v[1] === "0"));
+  // Indices of buckets that have any non-zero values
+  const idxsWithData = result
+    .map((r, i) => (r.values.every(v => v[1] === "0") ? -1 : i))
+    .filter(i => i !== -1);
+
+  const countWithData = idxsWithData.length;
+
+  // No data at all, or too few buckets to bother slicing
+  if (countWithData === 0 || result.length <= 3) {
+    return result;
+  }
+
+  // More than one non-empty bucket: keep only buckets with data
+  if (countWithData > 1) {
+    return result.filter((_, i) => idxsWithData.includes(i));
+  }
+
+  // Keep the only non-empty bucket plus its adjacent buckets (if available)
+  const idx = idxsWithData[0];
+  const keep = new Set<number>([idx]);
+
+  if (idx - 1 >= 0) keep.add(idx - 1);
+  if (idx + 1 < result.length) keep.add(idx + 1);
+
+  return result.filter((_, i) => keep.has(i));
 };

app/vmui/packages/vmui/src/utils/uplot/index.ts

@@ -5,4 +5,3 @@ export * from "./hooks";
 export * from "./instance";
 export * from "./scales";
 export * from "./series";
-export * from "./bands";

app/vmui/packages/vmui/src/utils/uplot/scales.ts

@@ -1,8 +1,7 @@
 import uPlot, { Range, Scale, Scales } from "uplot";
 import { getMinMaxBuffer } from "./axes";
 import { YaxisState } from "../../state/graph/reducer";
-import { ForecastType, MinMax, SetMinMax } from "../../types";
-import { anomalyColors } from "../color";
+import { MinMax, SetMinMax } from "../../types";
 
 export const getRangeX = ({ min, max }: MinMax): Range.MinMax => [min, max];
 
@@ -25,80 +24,3 @@ export const setSelect = (setPlotScale: SetMinMax) => (u: uPlot) => {
   const max = u.posToVal(u.select.left + u.select.width, "x");
   setPlotScale({ min, max });
 };
-
-export const scaleGradient = (
-  scaleKey: string,
-  ori: number,
-  scaleStops: [number, string][],
-  discrete = false
-) => (u: uPlot): CanvasGradient | string => {
-  const can = document.createElement("canvas");
-  const ctx = can.getContext("2d");
-  if (!ctx) return "";
-
-  const scale = u.scales[scaleKey];
-
-  // we want the stop below or at the scaleMax
-  // and the stop below or at the scaleMin, else the stop above scaleMin
-  let minStopIdx = 0;
-  let maxStopIdx = 1;
-
-  for (let i = 0; i < scaleStops.length; i++) {
-    const stopVal = scaleStops[i][0];
-
-    if (stopVal <= (scale.min || 0) || minStopIdx == null)
-      minStopIdx = i;
-
-    maxStopIdx = i;
-
-    if (stopVal >= (scale.max || 1))
-      break;
-  }
-
-  if (minStopIdx == maxStopIdx)
-    return scaleStops[minStopIdx][1];
-
-  let minStopVal = scaleStops[minStopIdx][0];
-  let maxStopVal = scaleStops[maxStopIdx][0];
-
-  if (minStopVal == -Infinity)
-    minStopVal = scale.min || 0;
-
-  if (maxStopVal == Infinity)
-    maxStopVal = scale.max || 1;
-
-  const minStopPos = u.valToPos(minStopVal, scaleKey, true) || 0;
-  const maxStopPos = u.valToPos(maxStopVal, scaleKey, true) || 1;
-
-  const range = minStopPos - maxStopPos;
-
-  let x0, y0, x1, y1;
-
-  if (ori == 1) {
-    x0 = x1 = 0;
-    y0 = minStopPos;
-    y1 = maxStopPos;
-  } else {
-    y0 = y1 = 0;
-    x0 = minStopPos;
-    x1 = maxStopPos;
-  }
-
-  const grd = ctx.createLinearGradient(x0, y0, x1, y1);
-
-  let prevColor = anomalyColors[ForecastType.actual];
-
-  for (let i = minStopIdx; i <= maxStopIdx; i++) {
-    const s = scaleStops[i];
-
-    const stopPos = i == minStopIdx ? minStopPos : i == maxStopIdx ? maxStopPos : u.valToPos(s[0], scaleKey, true) | 1;
-    const pct = Math.min(1, Math.max(0, (minStopPos - stopPos) / range));
-    if (discrete && i > minStopIdx) {
-      grd.addColorStop(pct, prevColor);
-    }
-
-    grd.addColorStop(pct, prevColor = s[1]);
-  }
-
-  return grd;
-};

app/vmui/packages/vmui/src/utils/uplot/series.ts

@@ -1,8 +1,8 @@
 import { MetricBase, MetricResult } from "../../api/types";
 import uPlot, { Series as uPlotSeries } from "uplot";
 import { getNameForMetric, promValueToNumber } from "../metric";
-import { ForecastType, HideSeriesArgs, LegendItemType, SeriesItem } from "../../types";
-import { anomalyColors, baseContrastColors, getColorFromString } from "../color";
+import { HideSeriesArgs, LegendItemType, SeriesItem } from "../../types";
+import { baseContrastColors, getColorFromString } from "../color";
 import { getMathStats } from "../math";
 import { formatPrettyNumber } from "./helpers";
 import { drawPoints } from "./scatter";
@@ -15,47 +15,26 @@ export const extractFields = (metric: MetricBase["metric"]): string => {
     .map(([key, value]) => `${key}: ${value}`).join(",");
 };
 
-type ForecastMetricInfo = {
-  value: ForecastType | null;
-  group: string;
-}
-
-export const isForecast = (metric: MetricBase["metric"]): ForecastMetricInfo => {
-  const metricName = metric?.__name__ || "";
-  const forecastRegex = new RegExp(`(${Object.values(ForecastType).join("|")})$`);
-  const match = metricName.match(forecastRegex);
-  const value = match && match[0] as ForecastType;
-  const isY = /(?:^|[^a-zA-Z0-9_])y(?:$|[^a-zA-Z0-9_])/.test(metricName);
-  return {
-    value: isY ? ForecastType.actual : value,
-    group: extractFields(metric)
-  };
-};
-
-export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[], alias: string[], showPoints?: boolean, isAnomalyUI?: boolean, isRawQuery?: boolean) => {
+export const getSeriesItemContext = (data: MetricResult[], hideSeries: string[], alias: string[], showPoints?: boolean, isRawQuery?: boolean) => {
   const colorState: {[key: string]: string} = {};
-  const maxColors = isAnomalyUI ? 0 : Math.min(data.length, baseContrastColors.length);
+  const maxColors = Math.min(data.length, baseContrastColors.length);
 
   for (let i = 0; i < maxColors; i++) {
     const label = getNameForMetric(data[i], alias[data[i].group - 1]);
     colorState[label] = baseContrastColors[i];
   }
 
-  return (d: MetricResult, i: number): SeriesItem => {
-    const metricInfo = isAnomalyUI ? isForecast(data[i].metric) : null;
+  return (d: MetricResult): SeriesItem => {
     const aliasValue = alias[d.group - 1];
-    const label = isAnomalyUI ? metricInfo?.group || "" : getNameForMetric(d, aliasValue);
+    const label = getNameForMetric(d, aliasValue);
 
     return {
       label,
       hasAlias: Boolean(aliasValue),
-      dash: getDashSeries(metricInfo),
-      width: getWidthSeries(metricInfo),
-      stroke: getStrokeSeries({ metricInfo, label, isAnomalyUI, colorState }),
-      points: getPointsSeries(metricInfo, showPoints, isRawQuery),
+      width: 1.4,
+      stroke: colorState[label] || getColorFromString(label),
+      points: getPointsSeries(showPoints, isRawQuery),
       spanGaps: false,
-      forecast: metricInfo?.value,
-      forecastGroup: metricInfo?.group,
       freeFormFields: d.metric,
       show: !includesHideSeries(label, hideSeries),
       scale: "1",
@@ -91,16 +70,11 @@ export const getLegendItem = (s: SeriesItem, group: number): LegendItemType => (
   hasAlias: s.hasAlias || false,
 });
 
-export const getHideSeries = ({ hideSeries, legend, metaKey, series, isAnomalyView }: HideSeriesArgs): string[] => {
+export const getHideSeries = ({ hideSeries, legend, metaKey, series }: HideSeriesArgs): string[] => {
   const { label } = legend;
   const include = includesHideSeries(label, hideSeries);
   const labels = series.map(getLabelForSeries);
 
-  // if anomalyView is true, always return all series except the one specified by `label`
-  if (isAnomalyView) {
-    return labels.filter(l => l !== label);
-  }
-
   if (metaKey) {
     return include ? hideSeries.filter(l => l !== label) : [...hideSeries, label];
   } else if (hideSeries.length) {
@@ -128,43 +102,7 @@ export const addSeries = (u: uPlot, series: uPlotSeries[], spanGaps = false, sho
   });
 };
 
-// Helpers
-
-const getDashSeries = (metricInfo: ForecastMetricInfo | null): number[] => {
-  const isLower = metricInfo?.value === ForecastType.yhatLower;
-  const isUpper = metricInfo?.value === ForecastType.yhatUpper;
-  const isYhat = metricInfo?.value === ForecastType.yhat;
-
-  if (isLower || isUpper) {
-    return [10, 5];
-  } else if (isYhat) {
-    return [10, 2];
-  }
-  return [];
-};
-
-const getWidthSeries = (metricInfo: ForecastMetricInfo | null): number => {
-  const isLower = metricInfo?.value === ForecastType.yhatLower;
-  const isUpper = metricInfo?.value === ForecastType.yhatUpper;
-  const isYhat = metricInfo?.value === ForecastType.yhat;
-  const isAnomalyMetric = metricInfo?.value === ForecastType.anomaly;
-
-  if (isUpper || isLower) {
-    return 0.7;
-  } else if (isYhat) {
-    return 1;
-  } else if (isAnomalyMetric) {
-    return 0;
-  }
-  return 1.4;
-};
-
-const getPointsSeries = (metricInfo: ForecastMetricInfo | null, showPoints: boolean = false, isRawQuery?: boolean): uPlotSeries.Points => {
-  const isAnomalyMetric = metricInfo?.value === ForecastType.anomaly;
-
-  if (isAnomalyMetric) {
-    return { size: 8, width: 4, space: 0 };
-  }
+const getPointsSeries = (showPoints: boolean = false, isRawQuery?: boolean): uPlotSeries.Points => {
   return {
     size: isRawQuery ? 0 : 4,
     width: 0,
@@ -187,31 +125,3 @@ const filterPoints = (self: uPlot, seriesIdx: number): number[] | null => {
 
   return indices;
 };
-
-type GetStrokeSeriesArgs = {
-  metricInfo: ForecastMetricInfo | null,
-  label: string,
-  colorState: {[p: string]: string},
-  isAnomalyUI?: boolean
-}
-
-const getStrokeSeries = ({ metricInfo, label, isAnomalyUI, colorState }: GetStrokeSeriesArgs): uPlotSeries.Stroke => {
-  const stroke: uPlotSeries.Stroke = colorState[label] || getColorFromString(label);
-  const isAnomalyMetric = metricInfo?.value === ForecastType.anomaly;
-
-  if (isAnomalyUI && isAnomalyMetric) {
-    return anomalyColors[ForecastType.anomaly];
-  } else if (isAnomalyUI && !isAnomalyMetric && !metricInfo?.value) {
-    // TODO add stroke for training data
-    // const hzGrad: [number, string][] = [
-    //   [time, anomalyColors[ForecastType.actual]],
-    //   [time, anomalyColors[ForecastType.training]],
-    //   [time, anomalyColors[ForecastType.actual]],
-    // ];
-    // stroke = scaleGradient("x", 0, hzGrad, true);
-    return anomalyColors[ForecastType.actual];
-  } else if (metricInfo?.value) {
-    return metricInfo?.value ? anomalyColors[metricInfo?.value] : stroke;
-  }
-  return colorState[label] || getColorFromString(label);
-};

app/vmui/packages/vmui/vite.config.ts

@@ -2,44 +2,40 @@ import * as path from "path";
 
 import { defineConfig, ProxyOptions } from "vite";
 import preact from "@preact/preset-vite";
-import dynamicIndexHtmlPlugin from "./config/plugins/dynamicIndexHtml";
 
 const getProxy = (): Record<string, ProxyOptions> | undefined => {
-  const playground = process.env.PLAYGROUND;
+  const playground = process.env.PLAYGROUND?.toLowerCase();
 
-  switch (playground) {
-    case "METRICS": {
-      return {
-        "^/(api|vmalert)/.*": {
-          target: "https://play.victoriametrics.com/select/0/prometheus",
-          changeOrigin: true,
-          configure: (proxy) => {
-            proxy.on("error", (err) => {
-              console.error("[proxy error]", err.message);
-            });
-          },
-        },
-        "/vmui/config.json": {
-          target: "https://play.victoriametrics.com/select/0",
-          changeOrigin: true,
-          configure: (proxy) => {
-            proxy.on("error", (err) => {
-              console.error("[proxy error]", err.message);
-            });
-          },
-        },
-      };
-    }
-    default: {
-      return undefined;
-    }
+  if (playground !== "true") {
+    return undefined;
   }
+
+  return {
+    "^/(api|vmalert)/.*": {
+      target: "https://play.victoriametrics.com/select/0/prometheus",
+      changeOrigin: true,
+      configure: (proxy) => {
+        proxy.on("error", (err) => {
+          console.error("[proxy error]", err.message);
+        });
+      },
+    },
+    "/prometheus/vmui/config.json": {
+      target: "https://play.victoriametrics.com/select/0",
+      changeOrigin: true,
+      configure: (proxy) => {
+        proxy.on("error", (err) => {
+          console.error("[proxy error]", err.message);
+        });
+      },
+    },
+  };
 };
 
-export default defineConfig(({ mode }) => {
+export default defineConfig(() => {
   return {
     base: "",
-    plugins: [preact(), dynamicIndexHtmlPlugin({ mode })],
+    plugins: [preact()],
     assetsInclude: ["**/*.md"],
     server: {
       open: true,

apptest/tests/dedup_test.go

@@ -193,9 +193,10 @@ func testDeduplication(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier,
 				}},
 				{Metric: map[string]string{"__name__": "metric4"}, Samples: []*apptest.Sample{
 					// If multiple raw samples have the same timestamp on the
-					// given -dedup.minScrapeInterval discrete interval, then
-					// stale markers are preferred over any other value.
-					{Timestamp: ts10, Value: decimal.StaleNaN},
+					// given -dedup.minScrapeInterval discrete interval,
+					// always prefer a non-decimal.StaleNaN value,
+					// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10196
+					{Timestamp: ts10, Value: 50},
 				}},
 			},
 		},

apptest/tests/prometheus_mock_storage.go

@@ -22,7 +22,17 @@ func NewPrometheusMockStorage(series []*prompb.TimeSeries) *PrometheusMockStorag
 	return &PrometheusMockStorage{store: series}
 }
 
-// Read implements the storage.Storage interface for reading time series data.
+// ReadMultiple implemnets the storage.ReadClient interface for reading time series data.
+func (ms *PrometheusMockStorage) ReadMultiple(ctx context.Context, queries []*prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
+	if len(queries) != 1 {
+		panic(fmt.Errorf("reading multiple queries isn't implemented"))
+	}
+
+	query := queries[0]
+	return ms.Read(ctx, query, sortSeries)
+}
+
+// Read implements the storage.ReadClient interface for reading time series data.
 func (ms *PrometheusMockStorage) Read(_ context.Context, query *prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
 	if ms.query != nil {
 		return nil, fmt.Errorf("expected only one call to remote client got: %v", query)

apptest/tests/remote_read_server.go

@@ -162,7 +162,7 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 			var matchers []*labels.Matcher
 			cb := func() (int64, error) { return 0, nil }
 
-			c := remote.NewSampleAndChunkQueryableClient(rrs.storage, nil, matchers, true, cb)
+			c := remote.NewSampleAndChunkQueryableClient(rrs.storage, labels.New(), matchers, true, cb)
 
 			q, err := c.ChunkQuerier(startTs, endTs)
 			if err != nil {
@@ -317,13 +317,13 @@ func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64)
 	return samples
 }
 
-func labelsToLabelsProto(labels labels.Labels) []prompb.Label {
-	result := make([]prompb.Label, 0, len(labels))
-	for _, l := range labels {
+func labelsToLabelsProto(ls labels.Labels) []prompb.Label {
+	result := make([]prompb.Label, 0, ls.Len())
+	ls.Range(func(l labels.Label) {
 		result = append(result, prompb.Label{
-			Name:  l.Name,
-			Value: l.Value,
+			Name:  strings.Clone(l.Name),
+			Value: strings.Clone(l.Value),
 		})
-	}
+	})
 	return result
 }

dashboards/vm/vmagent.json

@@ -627,7 +627,7 @@
           "refId": "A"
         }
       ],
-      "title": "Log errors (30m)",
+      "title": "Log messages (30m)",
       "type": "stat"
     },
     {
@@ -8413,4 +8413,4 @@
   "title": "VictoriaMetrics - vmagent (VM)",
   "uid": "G7Z9GzMGz_vm",
   "version": 1
-}
+}

dashboards/vm/vmauth.json

@@ -2573,13 +2573,13 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
               "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
           ],
-          "title": "Log errors",
+          "title": "Logging rate",
           "type": "timeseries"
         },
         {

dashboards/vmagent.json

@@ -626,7 +626,7 @@
           "refId": "A"
         }
       ],
-      "title": "Log errors (30m)",
+      "title": "Log messages (30m)",
       "type": "stat"
     },
     {
@@ -8412,4 +8412,4 @@
   "title": "VictoriaMetrics - vmagent",
   "uid": "G7Z9GzMGz",
   "version": 1
-}
+}

dashboards/vmauth.json

@@ -2572,13 +2572,13 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
               "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
           ],
-          "title": "Log errors",
+          "title": "Logging rate",
           "type": "timeseries"
         },
         {

deployment/docker/Makefile

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.23.2
 ROOT_IMAGE_SCRATCH ?= scratch
 CERTS_IMAGE := alpine:3.23.2
 
-GO_BUILDER_IMAGE := golang:1.25.5
+GO_BUILDER_IMAGE := golang:1.25.6
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.133.0
+    image: victoriametrics/vmagent:v1.134.0
     depends_on:
       - "vmauth"
     ports:
@@ -37,14 +37,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.133.0-cluster
+    image: victoriametrics/vmstorage:v1.134.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.133.0-cluster
+    image: victoriametrics/vmstorage:v1.134.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -54,7 +54,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.133.0-cluster
+    image: victoriametrics/vminsert:v1.134.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -63,7 +63,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.133.0-cluster
+    image: victoriametrics/vminsert:v1.134.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -75,7 +75,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.133.0-cluster
+    image: victoriametrics/vmselect:v1.134.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -85,7 +85,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.133.0-cluster
+    image: victoriametrics/vmselect:v1.134.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -100,7 +100,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.133.0
+    image: victoriametrics/vmauth:v1.134.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -114,7 +114,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.133.0
+    image: victoriametrics/vmalert:v1.134.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.133.0
+    image: victoriametrics/vmagent:v1.134.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.133.0
+    image: victoriametrics/victoria-metrics:v1.134.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -54,7 +54,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.133.0
+    image: victoriametrics/vmalert:v1.134.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts-health.yml

@@ -71,15 +71,16 @@ groups:
             the service could work unreliably with delays in processing.
 
       - alert: TooManyLogs
-        expr: sum(increase(vm_log_messages_total{level="error"}[5m])) without (app_version, location) > 0
+        expr: sum(increase(vm_log_messages_total{level!="info"}[5m])) without (app_version, location, is_printed) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+          summary: "Too many logs are generated for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
           description: >
-            Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.
-            Worth to check logs for specific error messages.
+            The job \"{{ $labels.job }}\" ({{ $labels.instance }}) generated {{ $value }} log messages with the level higher than info for the last 5 minutes.
+            Check the logs for the given target. Check also the \"location\" label at the vm_log_messages_total metric if -loggerLevel command-line flag is set to value other than INFO.
+            This label contains code locations responsible for generating log messages suppressed by -loggerLevel.
 
       - alert: TooManyTSIDMisses
         expr: increase(vm_missing_tsids_for_metric_id_total[5m]) > 0