lib/{fs,filestream}: use single ParallelExecutor for fs and filestream tasks

.github/workflows/vmui.yml

@@ -34,39 +34,33 @@ jobs:
       - name: Code checkout
         uses: actions/checkout@v6
 
-      - name: Cache node_modules
-        id: cache
-        uses: actions/cache@v5
+      - name: Setup Node
+        uses: actions/setup-node@v6
         with:
-          path: app/vmui/packages/vmui/node_modules
-          key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
-          restore-keys: |
-            vmui-deps-${{ runner.os }}-
+          node-version: '24.x'
 
-      - name: Install dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make vmui-install
+      - name: Cache node-modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            app/vmui/packages/vmui/node_modules
+          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
+          restore-keys: vmui-artifacts-${{ runner.os }}-
 
       - name: Run lint
         id: lint
         run: make vmui-lint
         continue-on-error: true
-        env:
-          VMUI_SKIP_INSTALL: true
 
       - name: Run tests
         id: test
         run: make vmui-test
         continue-on-error: true
-        env:
-          VMUI_SKIP_INSTALL: true
 
       - name: Run typecheck
         id: typecheck
         run: make vmui-typecheck
         continue-on-error: true
-        env:
-          VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
         uses: ataylorme/eslint-annotate-action@v3

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2026 VictoriaMetrics, Inc.
+   Copyright 2019-2025 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

app/victoria-metrics/main.go

@@ -134,7 +134,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>Single-node VictoriaMetrics</h2></br>")
-		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/'>https://docs.victoriametrics.com/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/victoria-metrics/self_scraper.go

@@ -29,9 +29,11 @@ var selfScraperWG sync.WaitGroup
 
 func startSelfScraper() {
 	selfScraperStopCh = make(chan struct{})
-	selfScraperWG.Go(func() {
+	selfScraperWG.Add(1)
+	go func() {
+		defer selfScraperWG.Done()
 		selfScraper(*selfScrapeInterval)
-	})
+	}()
 }
 
 func stopSelfScraper() {

app/vmagent/main.go

@@ -245,7 +245,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
-		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/vmagent/remotewrite/client.go

@@ -202,10 +202,14 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
 	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(concurrency)
+		return float64(*queues)
 	})
-	for range concurrency {
-		c.wg.Go(c.runWorker)
+	for i := 0; i < concurrency; i++ {
+		c.wg.Add(1)
+		go func() {
+			defer c.wg.Done()
+			c.runWorker()
+		}()
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 }

app/vmagent/remotewrite/pendingseries.go

@@ -48,7 +48,11 @@ func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Boo
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
-	ps.periodicFlusherWG.Go(ps.periodicFlusher)
+	ps.periodicFlusherWG.Add(1)
+	go func() {
+		defer ps.periodicFlusherWG.Done()
+		ps.periodicFlusher()
+	}()
 	return &ps
 }
 

app/vmagent/remotewrite/relabel.go

@@ -9,14 +9,14 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/VictoriaMetrics/metrics"
-	"gopkg.in/yaml.v2"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"go.yaml.in/yaml/v3"
+
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -139,7 +139,6 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		remoteWriteRelabelConfigData.Store(&rawCfg)
 		rcs.global = global
 	}
-
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
@@ -177,9 +176,19 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
-// isSet indicates whether (global or per-URL) command-line flags is set
 func (rcs *relabelConfigs) isSet() bool {
-	return *relabelConfigPathGlobal != "" || len(*relabelConfigPaths) > 0
+	if rcs == nil {
+		return false
+	}
+	if rcs.global.Len() > 0 {
+		return true
+	}
+	for _, pc := range rcs.perURL {
+		if pc.Len() > 0 {
+			return true
+		}
+	}
+	return false
 }
 
 // initLabelsGlobal must be called after parsing command-line flags.

app/vmagent/remotewrite/remotewrite.go

@@ -59,7 +59,7 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. "+
 		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -176,6 +176,13 @@ func Init() {
 		})
 	}
 
+	if *queues > maxQueues {
+		*queues = maxQueues
+	}
+	if *queues <= 0 {
+		*queues = 1
+	}
+
 	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
@@ -208,7 +215,9 @@ func Init() {
 	dropDanglingQueues()
 
 	// Start config reloader.
-	configReloaderWG.Go(func() {
+	configReloaderWG.Add(1)
+	go func() {
+		defer configReloaderWG.Done()
 		for {
 			select {
 			case <-configReloaderStopCh:
@@ -218,7 +227,7 @@ func Init() {
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
 		}
-	})
+	}()
 }
 
 func dropDanglingQueues() {
@@ -258,6 +267,17 @@ func initRemoteWriteCtxs(urls []string) {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks / *queues > 100 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 100 * *queues
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
 	rwctxIdx := make([]int, len(urls))
 	if retryMaxTime.String() != "" {
@@ -272,7 +292,7 @@ func initRemoteWriteCtxs(urls []string) {
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 		rwctxIdx[i] = i
 	}
 
@@ -538,9 +558,11 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 	// Push metadata to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
+	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		wg.Go(func() {
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
 			if !rwctx.tryPushMetadataInternal(mms) {
 				rwctx.pushFailures.Inc()
 				if forceDropSamplesOnFailure {
@@ -549,7 +571,7 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 				}
 				anyPushFailed.Store(true)
 			}
-		})
+		}(rwctx)
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -581,13 +603,15 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 	// Push tssBlock to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
+	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		wg.Go(func() {
+		go func(rwctx *remoteWriteCtx) {
+			defer wg.Done()
 			if !rwctx.TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		})
+		}(rwctx)
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -609,11 +633,13 @@ func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock
 		if len(shard) == 0 {
 			continue
 		}
-		wg.Go(func() {
-			if !rwctx.TryPushTimeSeries(shard, forceDropSamplesOnFailure) {
+		wg.Add(1)
+		go func(rwctx *remoteWriteCtx, tss []prompb.TimeSeries) {
+			defer wg.Done()
+			if !rwctx.TryPushTimeSeries(tss, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		})
+		}(rwctx, shard)
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -822,7 +848,7 @@ type remoteWriteCtx struct {
 	rowsDroppedOnPushFailure     *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
@@ -837,23 +863,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	}
 
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
-	queuesSize := queues.GetOptionalArg(argIdx)
-	if queuesSize > maxQueues {
-		queuesSize = maxQueues
-	} else if queuesSize <= 0 {
-		queuesSize = 1
-	}
-
-	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / *maxRowsPerBlock / 100
-	if maxInmemoryBlocks/queuesSize > 100 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 100 * queuesSize
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
@@ -871,16 +880,16 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
-		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, queuesSize)
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
 	default:
 		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
 	}
-	c.init(argIdx, queuesSize, sanitizedURL)
+	c.init(argIdx, *queues, sanitizedURL)
 
 	// Initialize pss
 	sf := significantFigures.GetOptionalArg(argIdx)
 	rd := roundDigits.GetOptionalArg(argIdx)
-	pssLen := queuesSize
+	pssLen := *queues
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
 		// since every pendingSeries can saturate up to a single CPU.

app/vmalert/config/types.go

@@ -76,14 +76,11 @@ func (t *Type) ValidateExpr(expr string) error {
 		if err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
-		labels, err := q.GetStatsLabels()
-		if err != nil {
-			return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
-		}
-		for i := range labels {
+		fields, _ := q.GetStatsByFields()
+		for i := range fields {
 			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
 			// making the result meaningless and may lead to cardinality issues.
-			if labels[i] == "_time" {
+			if fields[i] == "_time" {
 				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
 			}
 		}

app/vmalert/main.go

@@ -81,7 +81,9 @@ absolute path to all .tpl files in root.
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
-var extURL *url.URL
+var (
+	extURL *url.URL
+)
 
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
@@ -159,7 +161,7 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	manager, err := newManager(ctx)
 	if err != nil {
-		logger.Fatalf("failed to create manager: %s", err)
+		logger.Fatalf("failed to init: %s", err)
 	}
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
 	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)

app/vmalert/manager_test.go

@@ -65,9 +65,11 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 
 	const workers = 500
 	const iterations = 10
-	var wg sync.WaitGroup
-	for n := range workers {
-		wg.Go(func() {
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func(n int) {
+			defer wg.Done()
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
@@ -77,7 +79,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 				}
 				_ = m.update(context.Background(), cfg, false)
 			}
-		})
+		}(i)
 	}
 	wg.Wait()
 }

app/vmalert/notifier/alert.go

@@ -80,15 +80,14 @@ func (as AlertState) String() string {
 
 // AlertTplData is used to execute templating
 type AlertTplData struct {
-	Type      string
-	Labels    map[string]string
-	Value     float64
-	Expr      string
-	AlertID   uint64
-	GroupID   uint64
-	ActiveAt  time.Time
-	For       time.Duration
-	IsPartial bool
+	Type     string
+	Labels   map[string]string
+	Value    float64
+	Expr     string
+	AlertID  uint64
+	GroupID  uint64
+	ActiveAt time.Time
+	For      time.Duration
 }
 
 var tplHeaders = []string{
@@ -102,7 +101,6 @@ var tplHeaders = []string{
 	"{{ $groupID := .GroupID }}",
 	"{{ $activeAt := .ActiveAt }}",
 	"{{ $for := .For }}",
-	"{{ $isPartial := .IsPartial }}",
 }
 
 // ExecTemplate executes the Alert template for given

app/vmalert/notifier/alertmanager.go

@@ -14,6 +14,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -171,6 +172,11 @@ const alertManagerPath = "/api/v2/alerts"
 func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
 	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration,
 ) (*AlertManager, error) {
+
+	if err := httputil.CheckURL(alertManagerURL); err != nil {
+		return nil, fmt.Errorf("invalid alertmanager URL: %w", err)
+	}
+
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig

app/vmalert/notifier/config_watcher_test.go

@@ -212,16 +212,18 @@ consul_sd_configs:
 
 	const workers = 500
 	const iterations = 10
-	var wg sync.WaitGroup
-	for n := range workers {
-		wg.Go(func() {
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func(n int) {
+			defer wg.Done()
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
 				_ = cw.reload(paths[rnd]) // update can fail and this is expected
 				_ = cw.notifiers()
 			}
-		})
+		}(i)
 	}
 	wg.Wait()
 }

app/vmalert/notifier/init.go

@@ -11,8 +11,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -229,9 +229,6 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
-		if err := httputil.CheckURL(addr); err != nil {
-			return nil, fmt.Errorf("invalid notifier.url %q: %w", addr, err)
-		}
 		addr = strings.TrimSuffix(addr, "/")
 		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, sendTimeout.GetOptionalArg(i))
 		if err != nil {
@@ -269,7 +266,7 @@ func GetTargets() map[TargetType][]Target {
 	if getActiveNotifiers == nil {
 		return nil
 	}
-	targets := make(map[TargetType][]Target)
+	var targets = make(map[TargetType][]Target)
 	// use cached targets from configWatcher instead of getActiveNotifiers for the extra target labels
 	if cw != nil {
 		cw.targetsMu.RLock()
@@ -290,7 +287,7 @@ func GetTargets() map[TargetType][]Target {
 }
 
 // Send sends alerts to all active notifiers
-func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) chan error {
+func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) *vmalertutil.ErrGroup {
 	alertsToSend := make([]Alert, 0, len(alerts))
 	lblss := make([][]prompb.Label, 0, len(alerts))
 	// apply global relabel config first without modifying original alerts in alerts
@@ -303,18 +300,17 @@ func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string
 		lblss = append(lblss, lbls)
 	}
 
+	errGr := new(vmalertutil.ErrGroup)
 	wg := sync.WaitGroup{}
 	activeNotifiers := getActiveNotifiers()
-	errCh := make(chan error, len(activeNotifiers))
-	defer close(errCh)
 	for i := range activeNotifiers {
 		nt := activeNotifiers[i]
 		wg.Go(func() {
 			if err := nt.Send(ctx, alertsToSend, lblss, notifierHeaders); err != nil {
-				errCh <- fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err)
+				errGr.Add(fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err))
 			}
 		})
 	}
 	wg.Wait()
-	return errCh
+	return errGr
 }

app/vmalert/notifier/init_test.go

@@ -55,9 +55,9 @@ func TestInitNegative(t *testing.T) {
 		*blackHole = oldBlackHole
 	}()
 
-	f := func(path string, addr []string, bh bool) {
+	f := func(path, addr string, bh bool) {
 		*configPath = path
-		*addrs = flagutil.ArrayString(addr)
+		*addrs = flagutil.ArrayString{addr}
 		*blackHole = bh
 		if err := Init(nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
@@ -65,12 +65,9 @@ func TestInitNegative(t *testing.T) {
 	}
 
 	// *configPath, *addrs and *blackhole are mutually exclusive
-	f("/dummy/path", []string{"127.0.0.1"}, false)
-	f("/dummy/path", []string{}, true)
-	f("", []string{"127.0.0.1"}, true)
-	// addr cannot be ""
-	f("", []string{""}, false)
-	f("", []string{"127.0.0.1", ""}, false)
+	f("/dummy/path", "127.0.0.1", false)
+	f("/dummy/path", "", true)
+	f("", "127.0.0.1", true)
 }
 
 func TestBlackHole(t *testing.T) {
@@ -205,9 +202,7 @@ alert_relabel_configs:
 		},
 	}
 	errG := Send(context.Background(), firingAlerts, nil)
-	for err := range errG {
-		if err != nil {
-			t.Errorf("unexpected error when sending alerts: %s", err)
-		}
+	if errG.Err() != nil {
+		t.Fatalf("unexpected error when sending alerts: %s", err)
 	}
 }

app/vmalert/rule/alerting.go

@@ -346,8 +346,6 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		ls.processed[l.Name] = l.Value
 	}
 
-	// labels only support limited templating variables,
-	// including `labels`, `value` and `expr`, to avoid breaking alert states or causing cardinality issue with results
 	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
@@ -389,7 +387,11 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 			return nil, err
 		}
 		alertID := hash(ls.processed)
-		a := ar.newAlert(s, time.Time{}, ls.processed, nil) // initial alert
+		as, err := ar.expandAnnotationTemplates(s, qFn, time.Time{}, ls)
+		if err != nil {
+			return nil, err
+		}
+		a := ar.newAlert(s, time.Time{}, ls.processed, as) // initial alert
 
 		prevT := time.Time{}
 		for i := range s.Values {
@@ -405,6 +407,8 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
 				// reset to Pending if there are gaps > EvalInterval between DPs
 				a.State = notifier.StatePending
 				a.ActiveAt = at
+				// re-template the annotations as active timestamp is changed
+				a.Annotations, _ = ar.expandAnnotationTemplates(s, qFn, at, ls)
 				a.Start = time.Time{}
 			} else if at.Sub(a.ActiveAt) >= ar.For && a.State != notifier.StateFiring {
 				a.State = notifier.StateFiring
@@ -459,8 +463,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
 	}
 
-	isPartial := isPartialResponse(res)
-	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartial)
+	ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
 	qFn := func(query string) ([]datasource.Metric, error) {
 		res, _, err := ar.q.Query(ctx, query, ts)
 		return res.Data, err
@@ -486,7 +489,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				at = a.ActiveAt
 			}
 		}
-		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls, isPartial)
+		as, err := ar.expandAnnotationTemplates(m, qFn, at, ls)
 		if err != nil {
 			// only set error in current state, but do not break alert processing
 			curState.Err = err
@@ -604,17 +607,16 @@ func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.
 	return ls, nil
 }
 
-func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet, isPartial bool) (map[string]string, error) {
+func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templates.QueryFn, activeAt time.Time, ls *labelSet) (map[string]string, error) {
 	tplData := notifier.AlertTplData{
-		Value:     m.Values[0],
-		Type:      ar.Type.String(),
-		Labels:    ls.origin,
-		Expr:      ar.Expr,
-		AlertID:   hash(ls.processed),
-		GroupID:   ar.GroupID,
-		ActiveAt:  activeAt,
-		For:       ar.For,
-		IsPartial: isPartial,
+		Value:    m.Values[0],
+		Type:     ar.Type.String(),
+		Labels:   ls.origin,
+		Expr:     ar.Expr,
+		AlertID:  hash(ls.processed),
+		GroupID:  ar.GroupID,
+		ActiveAt: activeAt,
+		For:      ar.For,
 	}
 	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
 	if err != nil {
@@ -818,9 +820,7 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 	expr := fmt.Sprintf("default_rollup(%s{%s%s}[%ds])",
 		alertForStateMetricName, nameStr, labelsFilter, int(lookback.Seconds()))
 
-	// query ALERTS_FOR_STATE at `ts-1s` instead `ts` to avoid retrieving data written in the current run,
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10335
-	res, _, err := q.Query(ctx, expr, ts.Add(-1*time.Second))
+	res, _, err := q.Query(ctx, expr, ts)
 	if err != nil {
 		return fmt.Errorf("failed to execute restore query %q: %w ", expr, err)
 	}

app/vmalert/rule/alerting_test.go

@@ -664,7 +664,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-pending"},
-			Annotations: map[string]string{},
+			Annotations: map[string]string{"activeAt": "5000"},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -684,7 +684,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-firing",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-firing"},
-			Annotations: map[string]string{},
+			Annotations: map[string]string{"activeAt": "1000"},
 			State:       notifier.StateFiring,
 			ActiveAt:    time.Unix(1, 0),
 			Start:       time.Unix(5, 0),
@@ -705,7 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 			Name:        "for-hold-pending",
 			Type:        config.NewPrometheusType().String(),
 			Labels:      map[string]string{"alertname": "for-hold-pending"},
-			Annotations: map[string]string{},
+			Annotations: map[string]string{"activeAt": "5000"},
 			State:       notifier.StatePending,
 			ActiveAt:    time.Unix(5, 0),
 			Value:       1,
@@ -1120,7 +1120,7 @@ func TestAlertingRuleLimit_Success(t *testing.T) {
 }
 
 func TestAlertingRule_Template(t *testing.T) {
-	f := func(rule *AlertingRule, metrics []datasource.Metric, isResponsePartial bool, alertsExpected map[uint64]*notifier.Alert) {
+	f := func(rule *AlertingRule, metrics []datasource.Metric, alertsExpected map[uint64]*notifier.Alert) {
 		t.Helper()
 
 		fakeGroup := Group{
@@ -1133,7 +1133,6 @@ func TestAlertingRule_Template(t *testing.T) {
 			entries: make([]StateEntry, 10),
 		}
 		fq.Add(metrics...)
-		fq.SetPartialResponse(isResponsePartial)
 
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
@@ -1164,7 +1163,7 @@ func TestAlertingRule_Template(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 1, "instance", "foo"),
 		metricWithValueAndLabels(t, 1, "instance", "bar"),
-	}, false, map[uint64]*notifier.Alert{
+	}, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
 			Annotations: map[string]string{
 				"summary": `common: Too high connection number for "foo"`,
@@ -1193,14 +1192,14 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}".{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
+			"summary":     `{{ $labels.__name__ }}: Too high connection number for "{{ $labels.instance }}"`,
 			"description": `{{ $labels.alertname}}: It is {{ $value }} connections for "{{ $labels.instance }}"`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 2, "__name__", "first", "instance", "foo", alertNameLabel, "override"),
 		metricWithValueAndLabels(t, 10, "__name__", "second", "instance", "bar", alertNameLabel, "override"),
-	}, false, map[uint64]*notifier.Alert{
+	}, map[uint64]*notifier.Alert{
 		hash(map[string]string{alertNameLabel: "override label", "exported_alertname": "override", "instance": "foo"}): {
 			Labels: map[string]string{
 				alertNameLabel:       "override label",
@@ -1208,7 +1207,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "foo",
 			},
 			Annotations: map[string]string{
-				"summary":     `first: Too high connection number for "foo".`,
+				"summary":     `first: Too high connection number for "foo"`,
 				"description": `override: It is 2 connections for "foo"`,
 			},
 		},
@@ -1219,7 +1218,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":           "bar",
 			},
 			Annotations: map[string]string{
-				"summary":     `second: Too high connection number for "bar".`,
+				"summary":     `second: Too high connection number for "bar"`,
 				"description": `override: It is 10 connections for "bar"`,
 			},
 		},
@@ -1232,7 +1231,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			"instance": "{{ $labels.instance }}",
 		},
 		Annotations: map[string]string{
-			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}.{{ if $isPartial }} WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.{{ end }}`,
+			"summary": `Alert "{{ $labels.alertname }}({{ $labels.alertgroup }})" for instance {{ $labels.instance }}`,
 		},
 		alerts: make(map[uint64]*notifier.Alert),
 	}, []datasource.Metric{
@@ -1240,7 +1239,7 @@ func TestAlertingRule_Template(t *testing.T) {
 			alertNameLabel, "originAlertname",
 			alertGroupNameLabel, "originGroupname",
 			"instance", "foo"),
-	}, true, map[uint64]*notifier.Alert{
+	}, map[uint64]*notifier.Alert{
 		hash(map[string]string{
 			alertNameLabel:        "OriginLabels",
 			"exported_alertname":  "originAlertname",
@@ -1256,7 +1255,7 @@ func TestAlertingRule_Template(t *testing.T) {
 				"instance":            "foo",
 			},
 			Annotations: map[string]string{
-				"summary": `Alert "originAlertname(originGroupname)" for instance foo. WARNING: Partial response detected - this alert may be incomplete. Please verify the results manually.`,
+				"summary": `Alert "originAlertname(originGroupname)" for instance foo`,
 			},
 		},
 	})
@@ -1386,7 +1385,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"group":         "vmalert",
 		"alertname":     "ConfigurationReloadFailure",
 		"alertgroup":    "vmalert",
-		"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label": `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	expectedProcessedLabels := map[string]string{
@@ -1396,7 +1395,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
 		"exported_alertname": "ConfigurationReloadFailure",
 		"group":              "vmalert",
 		"alertgroup":         "vmalert",
-		"invalid_label":      `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
+		"invalid_label":      `error evaluating template: template: :1:268: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
 	}
 
 	ls, err := ar.toLabels(metric, nil)

app/vmalert/rule/group.go

@@ -18,7 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -375,7 +374,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 
 	g.infof("started")
 
-	eval := func(ctx context.Context, ts time.Time) time.Time {
+	eval := func(ctx context.Context, ts time.Time) {
 		g.metrics.iterationTotal.Inc()
 
 		start := time.Now()
@@ -383,7 +382,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 		if len(g.Rules) < 1 {
 			g.metrics.iterationDuration.UpdateDuration(start)
 			g.LastEvaluation = start
-			return ts
+			return
 		}
 
 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
@@ -397,7 +396,6 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 		}
 		g.metrics.iterationDuration.UpdateDuration(start)
 		g.LastEvaluation = start
-		return ts
 	}
 
 	evalCtx, cancel := context.WithCancel(ctx)
@@ -406,7 +404,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	g.mu.Unlock()
 	defer g.evalCancel()
 
-	realEvalTS := eval(evalCtx, evalTS)
+	eval(evalCtx, evalTS)
 
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
@@ -414,7 +412,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	// restore the rules state after the first evaluation
 	// so only active alerts can be restored.
 	if rr != nil {
-		err := g.restore(ctx, rr, realEvalTS, *remoteReadLookBack)
+		err := g.restore(ctx, rr, evalTS, *remoteReadLookBack)
 		if err != nil {
 			logger.Errorf("error while restoring ruleState for group %q: %s", g.Name, err)
 		}
@@ -757,7 +755,6 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		return fmt.Errorf("rule %q: failed to execute: %w", r, err)
 	}
 
-	var errG vmalertutil.ErrGroup
 	if e.Rw != nil {
 		pushToRW := func(tss []prompb.TimeSeries) error {
 			var lastErr error
@@ -769,26 +766,20 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 			return lastErr
 		}
 		if err := pushToRW(tss); err != nil {
-			errG.Add(err)
+			return err
 		}
 	}
 
 	ar, ok := r.(*AlertingRule)
 	if !ok {
-		return errG.Err()
+		return nil
 	}
 
 	alerts := ar.alertsToSend(resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
-		return errG.Err()
+		return nil
 	}
 
-	notifierErr := notifier.Send(ctx, alerts, e.notifierHeaders)
-	for err := range notifierErr {
-		if err != nil {
-			errG.Add(fmt.Errorf("rule %q: notifier failure: %w", r, err))
-		}
-	}
-
-	return errG.Err()
+	errGr := notifier.Send(ctx, alerts, e.notifierHeaders)
+	return errGr.Err()
 }

app/vmalert/rule/rule_test.go

@@ -65,15 +65,17 @@ func TestRule_stateConcurrent(_ *testing.T) {
 	r := &AlertingRule{state: &ruleState{entries: make([]StateEntry, 20)}}
 	const workers = 50
 	const iterations = 100
-	var wg sync.WaitGroup
-	for range workers {
-		wg.Go(func() {
+	wg := sync.WaitGroup{}
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func() {
+			defer wg.Done()
 			for i := 0; i < iterations; i++ {
 				r.state.add(StateEntry{At: time.Now()})
 				r.state.getAll()
 				r.state.getLast()
 			}
-		})
+		}()
 	}
 	wg.Wait()
 }

app/vmalert/vmalertutil/err_group.go

@@ -45,7 +45,7 @@ func (eg *ErrGroup) Error() string {
 		return ""
 	}
 	var b strings.Builder
-	fmt.Fprintf(&b, "errors(%d): \n", len(eg.errs))
+	fmt.Fprintf(&b, "errors(%d): ", len(eg.errs))
 	for i, err := range eg.errs {
 		b.WriteString(err.Error())
 		if i != len(eg.errs)-1 {

app/vmalert/vmalertutil/err_group_test.go

@@ -30,8 +30,8 @@ func TestErrGroup(t *testing.T) {
 	}
 
 	f(nil, "")
-	f([]error{errors.New("timeout")}, "errors(1): \ntimeout")
-	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): \ntimeout\ndeadline")
+	f([]error{errors.New("timeout")}, "errors(1): timeout")
+	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): timeout\ndeadline")
 }
 
 // TestErrGroupConcurrent supposed to test concurrent

app/vmalert/web.qtpl

@@ -9,7 +9,6 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
-    "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 ) %}
 
 {% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
@@ -79,8 +78,6 @@
 {% func Welcome(r *http.Request) %}
     {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
     <p>
-        Version {%s buildinfo.Version %} <br>
-
         API:<br>
         {% for _, p := range apiLinks  %}
             {%code p, doc := p[0], p[1] %}

app/vmauth/auth_config.go

@@ -113,8 +113,10 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
-		// The number of concurrently executed requests for the given user equals the limt.
-		// Wait until some of the currently executed requests are finished, so the current request could be executed.
+		ui.concurrencyLimitReached.Inc()
+
+		// The per-user limit for the number of concurrent requests is reached.
+		// Wait until the currently executed requests are finished, so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case ui.concurrencyLimitCh <- struct{}{}:
@@ -122,8 +124,6 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 		case <-ctx.Done():
 			err := ctx.Err()
 			if errors.Is(err, context.DeadlineExceeded) {
-				// The current request couldn't be executed until the request timeout.
-				ui.concurrencyLimitReached.Inc()
 				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
 					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
 			}
@@ -150,22 +150,12 @@ func (ui *UserInfo) stopHealthChecks() {
 	if ui == nil {
 		return
 	}
+	if ui.URLPrefix == nil {
+		return
+	}
 
-	if ui.URLPrefix != nil {
-		bus := ui.URLPrefix.bus.Load()
-		bus.stopHealthChecks()
-	}
-	if ui.DefaultURL != nil {
-		bus := ui.DefaultURL.bus.Load()
-		bus.stopHealthChecks()
-	}
-	for i := range ui.URLMaps {
-		um := &ui.URLMaps[i]
-		if um.URLPrefix != nil {
-			bus := um.URLPrefix.bus.Load()
-			bus.stopHealthChecks()
-		}
-	}
+	bus := ui.URLPrefix.bus.Load()
+	bus.stopHealthChecks()
 }
 
 // Header is `Name: Value` http header, which must be added to the proxied request.
@@ -373,10 +363,12 @@ func (bu *backendURL) isBroken() bool {
 
 func (bu *backendURL) setBroken() {
 	if bu.broken.CompareAndSwap(false, true) {
-		bu.healthCheckWG.Go(func() {
+		bu.healthCheckWG.Add(1)
+		go func() {
+			defer bu.healthCheckWG.Done()
 			bu.runHealthCheck()
 			bu.broken.Store(false)
-		})
+		}()
 	}
 }
 
@@ -402,7 +394,7 @@ func (bu *backendURL) runHealthCheck() {
 				if errors.Is(bu.healthCheckContext.Err(), context.Canceled) {
 					return
 				}
-				logger.Warnf("ignoring the backend at %s for %s because of dial error: %s", addr, *failTimeout, err)
+				logger.Warnf("ignoring the backend at %s for %s becasue of dial error: %s", addr, *failTimeout, err)
 				continue
 			}
 
@@ -741,9 +733,11 @@ func initAuthConfig() {
 	configTimestamp.Set(fasttime.UnixTimestamp())
 
 	stopCh = make(chan struct{})
-	authConfigWG.Go(func() {
+	authConfigWG.Add(1)
+	go func() {
+		defer authConfigWG.Done()
 		authConfigReloader(sighupCh)
-	})
+	}()
 }
 
 func stopAuthConfig() {
@@ -815,7 +809,7 @@ func reloadAuthConfig() (bool, error) {
 
 	ok, err := reloadAuthConfigData(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
 	}
 	if !ok {
 		return false, nil

app/vmauth/main.go

@@ -24,7 +24,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ioutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -41,38 +40,27 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
-	idleConnTimeout        = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
+	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
+		"See also -maxConcurrentRequests")
+	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
 	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 
-	requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
-		"This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. "+
-		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
-	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
-		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
-		"See also -requestBufferSize")
-
-	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process simultaneously. "+
-		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
-		"This protects vmauth itself from overloading and out-of-memory (OOM) failures. See also -maxConcurrentPerUserRequests "+
-		"and https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
-	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 100, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
-		"This provides fairness and isolation between users, preventing a single user from consuming all the available resources. "+
-		"It works in conjunction with -maxConcurrentRequests, which sets the global limit across all users. "+
-		"This default can be overridden for individual users via max_concurrent_requests option in per-user config. "+
-		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
-	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration to wait before rejecting incoming requests if concurrency limit "+
-		"specified via -maxConcurrentRequests or -maxConcurrentPerUserRequests command-line flags is reached. "+
-		"Requests are rejected with '429 Too Many Requests' http status code if the limit is still reached after the -maxQueueDuration duration. "+
-		"This allows graceful handling of short spikes in concurrent requests. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
+		"'429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
+	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
+		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options "+
+		"and max_concurrent_requests option in per-user config")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration the request waits for execution when the number of concurrently executed "+
+		"requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. "+
+		"This allows graceful handling of short spikes in the number of concurrent requests")
 
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
-	failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
-
+	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
+	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
+		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -227,121 +215,48 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
 	defer cancel()
 
-	// Acquire global concurrency limit.
-	if err := beginConcurrencyLimit(ctx); err != nil {
-		handleConcurrencyLimitError(w, r, err)
-		return
-	}
-	defer endConcurrencyLimit()
-
-	// Set read deadline for reading the initial chunk for the request body.
-	rc := http.NewResponseController(w)
-	deadline, ok := ctx.Deadline()
-	if !ok {
-		logger.Panicf("BUG: expecting valid deadline for the context")
-	}
-	if err := rc.SetReadDeadline(deadline); err != nil {
-		logger.Panicf("BUG: cannot set read deadline: %s", err)
-	}
-
-	// Read the initial chunk for the request body.
-	userName := ui.name()
-	if userName == "" {
-		userName = "unauthorized"
-	}
-	bb, err := bufferRequestBody(ctx, r.Body, userName)
-	if err != nil {
-		httpserver.Errorf(w, r, "%s", err)
-		return
-	}
-	r.Body = bb
-
-	// Disable the read deadline for the rest of the request body.
-	if err := rc.SetReadDeadline(time.Time{}); err != nil {
-		logger.Panicf("BUG: cannot reset read deadline: %s", err)
-	}
-
-	// Acquire concurrency limit for the given user.
-	if err := ui.beginConcurrencyLimit(ctx); err != nil {
-		handleConcurrencyLimitError(w, r, err)
-		return
-	}
-	defer ui.endConcurrencyLimit()
-
-	// Process the request.
-	processRequest(w, r, ui)
-}
-
-func beginConcurrencyLimit(ctx context.Context) error {
+	// Limit the concurrency of requests to backends
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		return nil
+		if err := ui.beginConcurrencyLimit(ctx); err != nil {
+			handleConcurrencyLimitError(w, r, err)
+			<-concurrencyLimitCh
+			return
+		}
 	default:
 		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
 		// so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case concurrencyLimitCh <- struct{}{}:
-			return nil
+			if err := ui.beginConcurrencyLimit(ctx); err != nil {
+				handleConcurrencyLimitError(w, r, err)
+				<-concurrencyLimitCh
+				return
+			}
 		case <-ctx.Done():
 			err := ctx.Err()
+
+			concurrentRequestsLimitReached.Inc()
+
 			if errors.Is(err, context.DeadlineExceeded) {
-				// The current request couldn't be executed until the request timeout.
-				concurrentRequestsLimitReached.Inc()
-				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+				err = fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
 					*maxQueueDuration, cap(concurrencyLimitCh))
+				handleConcurrencyLimitError(w, r, err)
+				return
 			}
-			return fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
+
+			err = fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
+			handleConcurrencyLimitError(w, r, err)
+			return
 		}
 	}
-}
-
-func endConcurrencyLimit() {
+	processRequest(w, r, ui)
+	ui.endConcurrencyLimit()
 	<-concurrencyLimitCh
 }
 
-func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (io.ReadCloser, error) {
-	if r == nil {
-		// This is a GET request with nil reader.
-		return nil, nil
-	}
-
-	maxBufSize := max(requestBufferSize.IntN(), maxRequestBodySizeToRetry.IntN())
-	if maxBufSize <= 0 {
-		return r, nil
-	}
-
-	lr := ioutil.GetLimitedReader(r, int64(maxBufSize))
-	defer ioutil.PutLimitedReader(lr)
-
-	start := time.Now()
-	buf, err := io.ReadAll(lr)
-	bufferRequestBodyDuration.UpdateDuration(start)
-
-	if err != nil {
-		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
-			rejectSlowClientRequests.Inc()
-
-			d := time.Since(start)
-
-			return nil, &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
-					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
-				StatusCode: http.StatusBadRequest,
-			}
-		}
-
-		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf("cannot read request body: %w", err),
-			StatusCode: http.StatusBadRequest,
-		}
-	}
-
-	bb := newBufferedBody(r, buf, maxBufSize)
-	return bb, nil
-}
-
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
@@ -367,6 +282,9 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		isDefault = true
 	}
 
+	rtb := newReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
+	r.Body = rtb
+
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getBackendURL()
@@ -374,19 +292,18 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			break
 		}
 		targetURL := bu.url
+		// Don't change path and add request_path query param for default route.
 		if isDefault {
-			// Don't change path and add request_path query param for default route.
 			query := targetURL.Query()
 			query.Set("request_path", u.String())
 			targetURL.RawQuery = query.Encode()
-		} else {
-			// Update path for regular routes.
+		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		}
 
 		wasLocalRetry := false
 	again:
-		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui, bu)
+		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
 		if needLocalRetry && !wasLocalRetry {
 			wasLocalRetry = true
 			goto again
@@ -396,7 +313,6 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if ok {
 			return
 		}
-
 		bu.setBroken()
 		ui.backendErrors.Inc()
 	}
@@ -408,7 +324,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ui.requestErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo, bu *backendURL) (bool, bool) {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
 	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
@@ -423,19 +339,27 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		}
 	}
 
-	bb, bbOK := req.Body.(*bufferedBody)
-	canRetry := !bbOK || bb.canRetry()
-
+	rtb, rtbOK := req.Body.(*readTrackingBody)
 	res, err := ui.rt.RoundTrip(req)
 
-	if errors.Is(r.Context().Err(), context.Canceled) {
-		// Do not retry canceled requests.
-		clientCanceledRequests.Inc()
-		return true, false
+	if ctxErr := r.Context().Err(); ctxErr != nil {
+		// Override the error returned by the RoundTrip with the context error if it isn't non-nil
+		// This makes sure the proper logging for canceled and timed out requests - log the real cause of the error
+		// instead of the random error, which could be returned from RoundTrip because of canceled or timed out request.
+		err = ctxErr
 	}
-
 	if err != nil {
-		if !canRetry {
+		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+			// Do not retry canceled or timed out requests
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			requestURI := httpserver.GetRequestURI(r)
+			if errors.Is(err, context.DeadlineExceeded) {
+				// Timed out request must be counted as errors, since this usually means that the backend is slow.
+				logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
+			}
+			return false, false
+		}
+		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
@@ -444,32 +368,27 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
 			ui.requestErrors.Inc()
-			bu.setBroken()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
 			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
-			if bbOK {
-				bb.resetReader()
-			}
 			return false, true
 		}
 
-		// Retry the request at another backend
+		// Request body wasn't read yet, this usually means that the backend isn't reachable; retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		requestURI := httpserver.GetRequestURI(r)
-		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, requestURI, targetURL, err)
-		if bbOK {
-			bb.resetReader()
-		}
+		// NOTE: do not use httpserver.GetRequestURI
+		// it explicitly reads request body, which may fail retries.
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, req.URL, targetURL, err)
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
-		if !canRetry {
+		_ = res.Body.Close()
+		if !rtbOK || !rtb.canRetry() {
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request body has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
@@ -478,16 +397,13 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			ui.requestErrors.Inc()
 			return true, false
 		}
-
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		requestURI := httpserver.GetRequestURI(r)
+		// NOTE: do not use httpserver.GetRequestURI
+		// it explicitly reads request body, which may fail retries.
 		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
-			remoteAddr, requestURI, targetURL, res.StatusCode, retryStatusCodes)
-		if bbOK {
-			bb.resetReader()
-		}
+			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
 		return false, false
 	}
 	removeHopHeaders(res.Header)
@@ -497,16 +413,10 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-
-	if errors.Is(r.Context().Err(), context.Canceled) {
-		// Do not retry canceled requests.
-		clientCanceledRequests.Inc()
-		return true, false
-	}
-
-	if err != nil && !netutil.IsTrivialNetworkError(err) {
+	if err != nil && !netutil.IsTrivialNetworkError(err) && !errors.Is(err, context.Canceled) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
+
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
 		ui.requestErrors.Inc()
 		return true, false
@@ -636,10 +546,6 @@ var (
 	configReloadRequests     = metrics.NewCounter(`vmauth_http_requests_total{path="/-/reload"}`)
 	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
-	clientCanceledRequests   = metrics.NewCounter(`vmauth_http_request_errors_total{reason="client_canceled"}`)
-	rejectSlowClientRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="reject_slow_client"}`)
-
-	bufferRequestBodyDuration = metrics.NewSummary(`vmauth_buffer_request_body_duration_seconds`)
 )
 
 func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
@@ -723,10 +629,10 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
-	if errors.Is(r.Context().Err(), context.Canceled) {
+	ctx := r.Context()
+	if errors.Is(ctx.Err(), context.Canceled) {
 		// Do not return any response for the request canceled by the client,
 		// since the connection to the client is already closed.
-		clientCanceledRequests.Inc()
 		return
 	}
 
@@ -738,76 +644,121 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-// bufferedBody serves two purposes:
-//  1. Enables request retries when the body size does not exceed maxBodySize
-//     by fully buffering the body in memory.
-//  2. Prevents slow clients from reducing effective server capacity by
-//     buffering the request body before acquiring a per-user concurrency slot.
-//
-// See bufferRequestBody for details on how bufferedBody is used.
-type bufferedBody struct {
-	// r contains reader for reading the data after buf is read.
+// readTrackingBody must be obtained via getReadTrackingBody()
+type readTrackingBody struct {
+	// maxBodySize is the maximum body size to cache in buf.
 	//
-	// r is nil if buf contains all the data.
+	// Bigger bodies cannot be retried.
+	maxBodySize int
+
+	// r contains reader for initial data reading
 	r io.ReadCloser
 
-	// buf contains the initial buffer read from r.
+	// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
+	// If more than maxBodySize is read from r, then cannotRetry is set to true.
 	buf []byte
 
-	// bufOffset is the offset at buf for already read bytes.
-	bufOffset int
+	// readBuf points to the cached data at buf, which must be read in the next call to Read().
+	readBuf []byte
 
-	// cannotRetry is set to true after Close() call on non-nil r.
+	// cannotRetry is set to true when more than maxBodySize bytes are read from r.
+	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
 	cannotRetry bool
+
+	// bufComplete is set to true when buf contains complete request body read from r.
+	bufComplete bool
 }
 
-func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody {
-	// Do not use sync.Pool here, since http.RoundTrip may still use request body after return.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
-
-	if len(buf) < maxBufSize {
-		// Read the full request body into buf.
-		r = nil
+func newReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
+	// do not use sync.Pool there
+	// since http.RoundTrip may still use request body after return
+	// See this issue for details https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
+	rtb := &readTrackingBody{}
+	if maxBodySize < 0 {
+		maxBodySize = 0
 	}
+	rtb.maxBodySize = maxBodySize
 
-	return &bufferedBody{
-		r:   r,
-		buf: buf,
+	if r == nil {
+		// This is GET request without request body
+		r = (*zeroReader)(nil)
 	}
+	rtb.r = r
+	return rtb
 }
 
-// Read implements io.Reader interface.
-func (bb *bufferedBody) Read(p []byte) (int, error) {
-	if bb.cannotRetry {
-		return 0, fmt.Errorf("cannot read already closed body")
-	}
-	if bb.bufOffset < len(bb.buf) {
-		n := copy(p, bb.buf[bb.bufOffset:])
-		bb.bufOffset += n
-		return n, nil
-	}
-	if bb.r == nil {
-		return 0, io.EOF
-	}
-	return bb.r.Read(p)
+type zeroReader struct{}
+
+func (r *zeroReader) Read(_ []byte) (int, error) {
+	return 0, io.EOF
 }
 
-func (bb *bufferedBody) canRetry() bool {
-	return bb.r == nil
-}
-
-// Close implements io.Closer interface.
-func (bb *bufferedBody) Close() error {
-	bb.resetReader()
-	if bb.r != nil {
-		bb.cannotRetry = true
-		return bb.r.Close()
-	}
+func (r *zeroReader) Close() error {
 	return nil
 }
 
-func (bb *bufferedBody) resetReader() {
-	bb.bufOffset = 0
+// Read implements io.Reader interface.
+func (rtb *readTrackingBody) Read(p []byte) (int, error) {
+	if len(rtb.readBuf) > 0 {
+		n := copy(p, rtb.readBuf)
+		rtb.readBuf = rtb.readBuf[n:]
+		return n, nil
+	}
+
+	if rtb.r == nil {
+		if rtb.bufComplete {
+			return 0, io.EOF
+		}
+		return 0, fmt.Errorf("cannot read client request body after closing client reader")
+	}
+
+	n, err := rtb.r.Read(p)
+	if rtb.cannotRetry {
+		return n, err
+	}
+
+	if len(rtb.buf)+n > rtb.maxBodySize {
+		rtb.cannotRetry = true
+		return n, err
+	}
+	rtb.buf = append(rtb.buf, p[:n]...)
+	if err == io.EOF {
+		rtb.bufComplete = true
+	}
+	return n, err
+}
+
+func (rtb *readTrackingBody) canRetry() bool {
+	if rtb.cannotRetry {
+		return false
+	}
+	if rtb.bufComplete {
+		return true
+	}
+	return rtb.r != nil
+}
+
+// Close implements io.Closer interface.
+func (rtb *readTrackingBody) Close() error {
+	if !rtb.cannotRetry {
+		rtb.readBuf = rtb.buf
+	} else {
+		rtb.readBuf = nil
+	}
+
+	// Close rtb.r only if the request body is completely read or if it is too big.
+	// http.Roundtrip performs body.Close call even without any Read calls,
+	// so this hack allows us to reuse request body.
+	if rtb.bufComplete || rtb.cannotRetry {
+		if rtb.r == nil {
+			return nil
+		}
+		err := rtb.r.Close()
+		rtb.r = nil
+		return err
+	}
+
+	return nil
 }
 
 func debugInfo(u *url.URL, r *http.Request) string {

app/vmauth/main_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"io"
 	"net"
@@ -11,7 +10,6 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
@@ -548,300 +546,28 @@ func (w *fakeResponseWriter) WriteHeader(statusCode int) {
 	}
 }
 
-// This is needed for net/http.ResponseController
-func (w *fakeResponseWriter) SetReadDeadline(deadline time.Time) error {
-	return nil
-}
-
-func TestBufferRequestBody_Success(t *testing.T) {
-	defaultRequestBufferSize := requestBufferSize.String()
-	defer func() {
-		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
-			t.Fatalf("cannot reset requestBufferSize: %s", err)
-		}
-	}()
-
-	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
-	defer func() {
-		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
-			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
-		}
-	}()
-
-	f := func(body *bytes.Buffer, requestBufferSizeFlag, maxRequestBodySizeToRetryFlag string) {
-		t.Helper()
-
-		expectedResponse := "statusCode=200"
-		if body.Len() > 0 {
-			expectedResponse += "\n" + body.String()
-		}
-
-		if err := requestBufferSize.Set(requestBufferSizeFlag); err != nil {
-			t.Fatalf("cannot set requestBufferSize: %s", err)
-		}
-		if err := maxRequestBodySizeToRetry.Set(maxRequestBodySizeToRetryFlag); err != nil {
-			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
-		}
-
-		var backendCalled bool
-		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			backendCalled = true
-
-			b, err := io.ReadAll(r.Body)
-			if err != nil {
-				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
-				return
-			}
-			if _, err := w.Write(b); err != nil {
-				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
-				return
-			}
-		}))
-		defer ts.Close()
-
-		// regular url_prefix
-		cfgStr := strings.ReplaceAll(`
-unauthorized_user:
-  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
-
-		cfgOrigP := authConfigData.Load()
-		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
-			t.Fatalf("cannot load config data: %s", err)
-		}
-		defer func() {
-			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
-			if cfgOrigP != nil {
-				cfgOrig = *cfgOrigP
-			}
-			_, err := reloadAuthConfigData(cfgOrig)
-			if err != nil {
-				t.Fatalf("cannot load the original config: %s", err)
-			}
-		}()
-
-		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
-		if err != nil {
-			t.Fatalf("cannot initialize http request: %s", err)
-		}
-
-		w := &fakeResponseWriter{}
-		if !requestHandlerWithInternalRoutes(w, r) {
-			t.Fatalf("unexpected false is returned from requestHandler")
-		}
-
-		response := w.getResponse()
-		response = strings.ReplaceAll(response, "\r\n", "\n")
-		response = strings.TrimSpace(response)
-
-		if response != expectedResponse {
-			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
-		}
-		if !backendCalled {
-			t.Fatalf("backend is not called")
-		}
-	}
-
-	// no body, no buffering, no retry
-	f(bytes.NewBuffer(nil), "0", "0")
-
-	// no body, buffering on, no retry
-	f(bytes.NewBuffer(nil), "100", "0")
-
-	// no body, no buffering, retry on
-	f(bytes.NewBuffer(nil), "0", "100")
-
-	// no body, buffering on, retry on
-	f(bytes.NewBuffer(nil), "100", "100")
-
-	// body smaller than buffer, retry max on
-	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "101", "101")
-
-	// body smaller than buffer
-	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "501", "0")
-
-	// body same size as buffer
-	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "500", "0")
-
-	// body bigger than a buffer
-	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "499", "0")
-
-	// body bigger than tmpBuf 8KiB used in buffering
-	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16384", "")
-
-	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16385", "")
-
-	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16383", "")
-}
-
-func TestBufferRequestBody_Failure(t *testing.T) {
-	defaultRequestBufferSize := requestBufferSize.String()
-	defer func() {
-		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
-			t.Fatalf("cannot reset requestBufferSize: %s", err)
-		}
-	}()
-
-	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
-	defer func() {
-		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
-			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
-		}
-	}()
-
-	defaultMaxQueueDuration := *maxQueueDuration
-	defer func() {
-		*maxQueueDuration = defaultMaxQueueDuration
-	}()
-
-	f := func(body *mockBody, expectedResponse string) {
-		t.Helper()
-
-		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
-			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
-		}
-		if err := requestBufferSize.Set("2048"); err != nil {
-			t.Fatalf("cannot set requestBufferSize: %s", err)
-		}
-		*maxQueueDuration = 100 * time.Millisecond
-
-		var backendCalled bool
-		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			backendCalled = true
-
-			b, err := io.ReadAll(r.Body)
-			if err != nil {
-				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
-				return
-			}
-			if _, err := w.Write(b); err != nil {
-				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
-				return
-			}
-		}))
-		defer ts.Close()
-
-		// regular url_prefix
-		cfgStr := strings.ReplaceAll(`
-unauthorized_user:
-  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
-
-		cfgOrigP := authConfigData.Load()
-		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
-			t.Fatalf("cannot load config data: %s", err)
-		}
-		defer func() {
-			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
-			if cfgOrigP != nil {
-				cfgOrig = *cfgOrigP
-			}
-			_, err := reloadAuthConfigData(cfgOrig)
-			if err != nil {
-				t.Fatalf("cannot load the original config: %s", err)
-			}
-		}()
-
-		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
-		if err != nil {
-			t.Fatalf("cannot initialize http request: %s", err)
-		}
-
-		w := &fakeResponseWriter{}
-		if !requestHandlerWithInternalRoutes(w, r) {
-			t.Fatalf("unexpected false is returned from requestHandler")
-		}
-
-		response := w.getResponse()
-		response = strings.ReplaceAll(response, "\r\n", "\n")
-		response = strings.TrimSpace(response)
-
-		if response != expectedResponse {
-			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
-		}
-		if backendCalled {
-			t.Fatalf("backend is called")
-		}
-	}
-
-	// an error at the beginning of reading
-	f(&mockBody{err: fmt.Errorf("an error")}, `statusCode=400
-cannot read request body: an error`)
-
-	// an error after reading 1024 bytes, buffer size is 2048 bytes
-	f(&mockBody{head: make([]byte, 1024), err: fmt.Errorf("an error")}, `statusCode=400
-cannot read request body: an error`)
-}
-
-type mockBody struct {
-	head []byte
-	err  error
-	tail []byte
-}
-
-func (r *mockBody) Read(p []byte) (n int, err error) {
-	if len(r.head) > 0 {
-		n = copy(p, r.head)
-		r.head = r.head[n:]
-		return n, nil
-	}
-
-	if r.err != nil {
-		return 0, r.err
-	}
-
-	if len(r.tail) > 0 {
-		n = copy(p, r.tail)
-		r.tail = r.tail[n:]
-		return n, nil
-	}
-
-	return 0, io.EOF
-}
-
-func TestBufferedBody_RetrySuccess(t *testing.T) {
+func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		defaultRequestBufferSize := requestBufferSize.String()
-		defer func() {
-			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
-				t.Fatalf("cannot reset requestBufferSize: %s", err)
-			}
-		}()
-		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
-			t.Fatalf("cannot set requestBufferSize: %s", err)
-		}
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
-		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
-		defer func() {
-			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
-				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
-			}
-		}()
-		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
-			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
-		}
-
-		ctx := context.Background()
-		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		bb, ok := rb.(*bufferedBody)
-		canRetry := !ok || bb.canRetry()
-
-		if !canRetry {
+		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		for i := 0; i < 5; i++ {
-			data, err := io.ReadAll(rb)
+			data, err := io.ReadAll(rtb)
 			if err != nil {
 				t.Fatalf("unexpected error when reading all the data at iteration %d: %s", i, err)
 			}
 			if string(data) != s {
 				t.Fatalf("unexpected data read at iteration %d\ngot\n%s\nwant\n%s", i, data, s)
 			}
-			if err := rb.Close(); err != nil {
-				t.Fatalf("unexpected error when closing bufferedBody at iteration %d: %s", i, err)
+			if err := rtb.Close(); err != nil {
+				t.Fatalf("unexpected error when closing readTrackingBody at iteration %d: %s", i, err)
+			}
+			if !rtb.canRetry() {
+				t.Fatalf("canRetry() must return true at iteration %d", i)
 			}
 		}
 	}
@@ -851,48 +577,19 @@ func TestBufferedBody_RetrySuccess(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1001)
+	f(newTestString(1000), 1000)
 }
 
-func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
+func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		// Check the case with partial read
-		defaultRequestBufferSize := requestBufferSize.String()
-		defer func() {
-			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
-				t.Fatalf("cannot reset requestBufferSize: %s", err)
-			}
-		}()
-		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
-			t.Fatalf("cannot set requestBufferSize: %s", err)
-		}
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
-		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
-		defer func() {
-			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
-				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
-			}
-		}()
-		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
-			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
-		}
-
-		ctx := context.Background()
-		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		bb, ok := rb.(*bufferedBody)
-		canRetry := !ok || bb.canRetry()
-
-		if !canRetry {
-			t.Fatalf("canRetry must return true")
-		}
 		for i := 0; i < len(s); i++ {
 			buf := make([]byte, i)
-			n, err := io.ReadFull(rb, buf)
+			n, err := io.ReadFull(rtb, buf)
 			if err != nil {
 				t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
 			}
@@ -902,20 +599,26 @@ func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 			if string(buf) != s[:i] {
 				t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
 			}
-			if err := rb.Close(); err != nil {
+			if err := rtb.Close(); err != nil {
 				t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
 			}
+			if !rtb.canRetry() {
+				t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
+			}
 		}
 
-		data, err := io.ReadAll(rb)
+		data, err := io.ReadAll(rtb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
+		if err := rtb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		}
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
 		}
 	}
 
@@ -924,53 +627,30 @@ func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1001)
+	f(newTestString(1000), 1000)
 }
 
-func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
+func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		defaultRequestBufferSize := requestBufferSize.String()
-		defer func() {
-			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
-				t.Fatalf("cannot reset requestBufferSize: %s", err)
-			}
-		}()
-		if err := requestBufferSize.Set("0"); err != nil {
-			t.Fatalf("cannot set requestBufferSize: %s", err)
-		}
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
-		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
-		defer func() {
-			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
-				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
-			}
-		}()
-		if err := maxRequestBodySizeToRetry.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
-			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
-		}
-
-		ctx := context.Background()
-		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		bb, ok := rb.(*bufferedBody)
-		canRetry := !ok || bb.canRetry()
-
-		if canRetry {
-			t.Fatalf("canRetry() must return false because of too big request body")
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		buf := make([]byte, 1)
-		n, err := io.ReadFull(rb, buf)
+		n, err := io.ReadFull(rtb, buf)
 		if err != nil {
 			t.Fatalf("unexpected error when reading a single byte: %s", err)
 		}
 		if n != 1 {
 			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
 		}
-		data, err := io.ReadAll(rb)
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true after reading one byte")
+		}
+		data, err := io.ReadAll(rtb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
@@ -978,11 +658,14 @@ func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 		if dataRead != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
 		}
-		if err := rb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
+		if err := rtb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		}
+		if rtb.canRetry() {
+			t.Fatalf("canRetry() must return false after closing the reader")
 		}
 
-		data, err = io.ReadAll(rb)
+		data, err = io.ReadAll(rtb)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -996,48 +679,35 @@ func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 	f(newTestString(2*maxBodySize), maxBodySize)
 }
 
-func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
+func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		defaultRequestBufferSize := requestBufferSize.String()
-		defer func() {
-			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
-				t.Fatalf("cannot reset requestBufferSize: %s", err)
-			}
-		}()
-		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
-			t.Fatalf("cannot set requestBufferSize: %s", err)
-		}
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
-		ctx := context.Background()
-		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		bb, ok := rb.(*bufferedBody)
-		canRetry := !ok || bb.canRetry()
-
-		if !canRetry {
+		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
-		data, err := io.ReadAll(rb)
+		data, err := io.ReadAll(rtb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
+		if err := rtb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
 		}
 
-		data, err = io.ReadAll(rb)
-		if err != nil {
-			t.Fatalf("unexpected error in io.ReadAll: %s", err)
+		if rtb.canRetry() {
+			t.Fatalf("canRetry() must return false after closing the reader")
 		}
-		if string(data) != s {
-			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
+		data, err = io.ReadAll(rtb)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if len(data) != 0 {
+			t.Fatalf("unexpected non-empty data read: %q", data)
 		}
 	}
 

app/vmctl/backoff/backoff.go

@@ -7,8 +7,6 @@ import (
 	"math"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -57,7 +55,6 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
 			return attempt, err // fail fast if not recoverable
 		}
 		attempt++
-		retriesTotal.Inc()
 		backoff := float64(b.minDuration) * math.Pow(b.factor, float64(i))
 		dur := time.Duration(backoff)
 		logger.Errorf("got error: %s on attempt: %d; will retry in %v", err, attempt, dur)
@@ -77,7 +74,3 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
 	}
 	return attempt, fmt.Errorf("execution failed after %d retry attempts", b.retries)
 }
-
-var (
-	retriesTotal = metrics.NewCounter(`vmctl_backoff_retries_total`)
-)

app/vmctl/flags.go

@@ -14,12 +14,6 @@ const (
 	globalSilent             = "s"
 	globalVerbose            = "verbose"
 	globalDisableProgressBar = "disable-progress-bar"
-
-	globalPushMetricsURL         = "pushmetrics.url"
-	globalPushMetricsInterval    = "pushmetrics.interval"
-	globalPushExtraLabels        = "pushmetrics.extraLabel"
-	globalPushHeaders            = "pushmetrics.header"
-	globalPushDisableCompression = "pushmetrics.disableCompression"
 )
 
 var (
@@ -39,29 +33,6 @@ var (
 			Value: false,
 			Usage: "Whether to disable progress bar during the import.",
 		},
-		&cli.StringSliceFlag{
-			Name:  globalPushMetricsURL,
-			Usage: "Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics",
-		},
-		&cli.DurationFlag{
-			Name:  globalPushMetricsInterval,
-			Value: 10 * time.Second,
-			Usage: "Interval for pushing metrics to every -pushmetrics.url",
-		},
-		&cli.StringSliceFlag{
-			Name: globalPushExtraLabels,
-			Usage: "Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. " +
-				"Flag can be set multiple times, to add few additional labels. " +
-				"For example, -pushmetrics.extraLabel='instance=\"foo\"' adds instance=\"foo\" label to all the metrics pushed to every -pushmetrics.url",
-		},
-		&cli.StringSliceFlag{
-			Name:  globalPushHeaders,
-			Usage: "Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.",
-		},
-		&cli.BoolFlag{
-			Name:  globalPushDisableCompression,
-			Usage: "Whether to disable compression when pushing metrics.",
-		},
 	}
 )
 
@@ -152,32 +123,32 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				" will have priority. Flag can be set multiple times, to add few additional labels.",
+				"will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
-				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination.",
+				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
 		},
 		&cli.StringFlag{
 			Name:  vmCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'",
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'",
 		},
 		&cli.StringFlag{
 			Name:  vmKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to '--vm-addr'",
+			Usage: "Optional path to client-side TLS key to use when connecting to '--vmAddr'",
 		},
 		&cli.StringFlag{
 			Name:  vmCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used",
+			Usage: "Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used",
 		},
 		&cli.StringFlag{
 			Name:  vmServerName,
-			Usage: "Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used",
+			Usage: "Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used",
 		},
 		&cli.BoolFlag{
 			Name:  vmInsecureSkipVerify,
-			Usage: "Whether to skip tls verification when connecting to '--vm-addr'",
+			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
 			Value: false,
 		},
 		&cli.IntFlag{
@@ -497,7 +468,7 @@ var (
 			Name: vmNativeFilterMatch,
 			Usage: "Time series selector to match series for export. For example, select {instance!=\"localhost\"} will " +
 				"match all series with \"instance\" label different to \"localhost\".\n" +
-				" See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format",
+				" See more details here https://github.com/VictoriaMetrics/VictoriaMetrics#how-to-export-data-in-native-format",
 			Value: `{__name__!=""}`,
 		},
 		&cli.StringFlag{
@@ -627,7 +598,7 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				" will have priority. Flag can be set multiple times, to add few additional labels.",
+				"will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
@@ -654,8 +625,8 @@ var (
 		&cli.BoolFlag{
 			Name: vmNativeDisableBinaryProtocol,
 			Usage: "Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format " +
-				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API. " +
-				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks. " +
+				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API." +
+				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},

app/vmctl/influx.go

@@ -7,8 +7,6 @@ import (
 	"log"
 	"sync"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
@@ -54,7 +52,6 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
-	influxSeriesTotal.Add(len(series))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing series"), len(series))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -66,18 +63,18 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 	ip.im.ResetStats()
 
 	var wg sync.WaitGroup
-	for range ip.cc {
-		wg.Go(func() {
+	wg.Add(ip.cc)
+	for i := 0; i < ip.cc; i++ {
+		go func() {
+			defer wg.Done()
 			for s := range seriesCh {
 				if err := ip.do(s); err != nil {
-					influxErrorsTotal.Inc()
 					errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
 					return
 				}
-				influxSeriesProcessed.Inc()
 				bar.Increment()
 			}
-		})
+		}()
 	}
 
 	// any error breaks the import
@@ -86,7 +83,6 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("influx error: %s", infErr)
 		case vmErr := <-ip.im.Errors():
-			influxErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
 		case seriesCh <- s:
 		}
@@ -99,7 +95,6 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 	// drain import errors channel
 	for vmErr := range ip.im.Errors() {
 		if vmErr.Err != nil {
-			influxErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
 		}
 	}
@@ -174,9 +169,3 @@ func (ip *influxProcessor) do(s *influx.Series) error {
 		}
 	}
 }
-
-var (
-	influxSeriesTotal     = metrics.NewCounter(`vmctl_influx_migration_series_total`)
-	influxSeriesProcessed = metrics.NewCounter(`vmctl_influx_migration_series_processed`)
-	influxErrorsTotal     = metrics.NewCounter(`vmctl_influx_migration_errors_total`)
-)

app/vmctl/limiter/limiter.go

@@ -4,8 +4,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 )
 
@@ -47,16 +45,9 @@ func (l *Limiter) Register(dataLen int) {
 			t := timerpool.Get(d)
 			<-t.C
 			timerpool.Put(t)
-			limiterThrottleEventsTotal.Inc()
 		}
 		l.budget += limit
 		l.deadline = time.Now().Add(time.Second)
 	}
 	l.budget -= int64(dataLen)
-	limiterBytesProcessed.Add(dataLen)
 }
-
-var (
-	limiterBytesProcessed      = metrics.NewCounter(`vmctl_limiter_bytes_processed_total`)
-	limiterThrottleEventsTotal = metrics.NewCounter(`vmctl_limiter_throttle_events_total`)
-)

app/vmctl/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"log"
 	"net/http"
@@ -20,9 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
@@ -44,20 +41,11 @@ func main() {
 	ctx, cancelCtx := context.WithCancel(context.Background())
 	start := time.Now()
 	beforeFn := func(c *cli.Context) error {
-		flag.Parse()
-		logger.Init()
 		isSilent = c.Bool(globalSilent)
 		if c.Bool(globalDisableProgressBar) {
 			barpool.Disable(true)
 		}
 		netutil.EnableIPv6()
-		pushmetrics.InitWith(&pushmetrics.Config{
-			URLs:               c.StringSlice(globalPushMetricsURL),
-			Interval:           c.Duration(globalPushMetricsInterval),
-			ExtraLabels:        c.StringSlice(globalPushExtraLabels),
-			DisableCompression: c.Bool(globalPushDisableCompression),
-			Headers:            c.StringSlice(globalPushHeaders),
-		})
 		return nil
 	}
 	app := &cli.App{
@@ -463,7 +451,6 @@ func main() {
 		log.Fatalln(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
-	pushmetrics.StopAndPush()
 }
 
 func initConfigVM(c *cli.Context) (vm.Config, error) {

app/vmctl/native/client.go

@@ -8,8 +8,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
 )
 
@@ -38,15 +36,12 @@ type Response struct {
 
 // Explore finds metric names by provided filter from api/v1/label/__name__/values
 func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start, end time.Time) ([]string, error) {
-	startTime := time.Now()
-	exploreRequestsTotal.Inc()
 	url := fmt.Sprintf("%s/%s", c.Addr, nativeMetricNamesAddr)
 	if tenantID != "" {
 		url = fmt.Sprintf("%s/select/%s/prometheus/%s", c.Addr, tenantID, nativeMetricNamesAddr)
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		exploreRequestsErrorsTotal.Inc()
 		return nil, fmt.Errorf("cannot create request to %q: %s", url, err)
 	}
 
@@ -58,53 +53,37 @@ func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start,
 
 	resp, err := c.do(req, http.StatusOK)
 	if err != nil {
-		exploreRequestsErrorsTotal.Inc()
-		exploreDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("series request failed: %s", err)
 	}
 
 	var response Response
 	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
-		exploreRequestsErrorsTotal.Inc()
-		exploreDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("cannot decode series response: %s", err)
 	}
-	exploreDuration.UpdateDuration(startTime)
 	return response.MetricNames, resp.Body.Close()
 }
 
 // ImportPipe uses pipe reader in request to process data
 func (c *Client) ImportPipe(ctx context.Context, dstURL string, pr *io.PipeReader) error {
-	startTime := time.Now()
-	importRequestsTotal.Inc()
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, dstURL, pr)
 	if err != nil {
-		importRequestsErrorsTotal.Inc()
 		return fmt.Errorf("cannot create import request to %q: %s", c.Addr, err)
 	}
 
 	importResp, err := c.do(req, http.StatusNoContent)
 	if err != nil {
-		importRequestsErrorsTotal.Inc()
-		importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("import request failed: %s", err)
 	}
 	if err := importResp.Body.Close(); err != nil {
-		importRequestsErrorsTotal.Inc()
-		importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("cannot close import response body: %s", err)
 	}
-	importDuration.UpdateDuration(startTime)
 	return nil
 }
 
 // ExportPipe makes request by provided filter and return io.ReadCloser which can be used to get data
 func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadCloser, error) {
-	startTime := time.Now()
-	exportRequestsTotal.Inc()
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		exportRequestsErrorsTotal.Inc()
 		return nil, fmt.Errorf("cannot create request to %q: %s", c.Addr, err)
 	}
 
@@ -123,11 +102,8 @@ func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadC
 
 	resp, err := c.do(req, http.StatusOK)
 	if err != nil {
-		exportRequestsErrorsTotal.Inc()
-		exportDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("export request failed: %w", err)
 	}
-	exportDuration.UpdateDuration(startTime)
 	return resp.Body, nil
 }
 
@@ -186,16 +162,3 @@ func (c *Client) do(req *http.Request, expSC int) (*http.Response, error) {
 	}
 	return resp, err
 }
-
-var (
-	importRequestsTotal        = metrics.NewCounter(`vmctl_vm_native_requests_total{type="import"}`)
-	exportRequestsTotal        = metrics.NewCounter(`vmctl_vm_native_requests_total{type="export"}`)
-	exploreRequestsTotal       = metrics.NewCounter(`vmctl_vm_native_requests_total{type="explore"}`)
-	importRequestsErrorsTotal  = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="import"}`)
-	exportRequestsErrorsTotal  = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="export"}`)
-	exploreRequestsErrorsTotal = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="explore"}`)
-
-	importDuration  = metrics.NewHistogram(`vmctl_vm_native_import_duration_seconds`)
-	exportDuration  = metrics.NewHistogram(`vmctl_vm_native_export_duration_seconds`)
-	exploreDuration = metrics.NewHistogram(`vmctl_vm_native_explore_duration_seconds`)
-)

app/vmctl/opentsdb.go

@@ -7,8 +7,6 @@ import (
 	"sync"
 	"time"
 
-	vmetrics "github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/cheggaaa/pb/v3"
@@ -59,7 +57,6 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 	if !prompt(ctx, question) {
 		return nil
 	}
-
 	op.im.ResetStats()
 	var startTime int64
 	if op.oc.HardTS != 0 {
@@ -87,24 +84,23 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 		seriesCh := make(chan queryObj, op.otsdbcc)
 		errCh := make(chan error)
 		// we're going to make serieslist * queryRanges queries, so we should represent that in the progress bar
-		otsdbSeriesTotal.Add(len(serieslist) * queryRanges)
 		bar := pb.StartNew(len(serieslist) * queryRanges)
 		defer func(bar *pb.ProgressBar) {
 			bar.Finish()
 		}(bar)
 		var wg sync.WaitGroup
-		for range op.otsdbcc {
-			wg.Go(func() {
+		wg.Add(op.otsdbcc)
+		for i := 0; i < op.otsdbcc; i++ {
+			go func() {
+				defer wg.Done()
 				for s := range seriesCh {
 					if err := op.do(s); err != nil {
-						otsdbErrorsTotal.Inc()
 						errCh <- fmt.Errorf("couldn't retrieve series for %s : %s", metric, err)
 						return
 					}
-					otsdbSeriesProcessed.Inc()
 					bar.Increment()
 				}
-			})
+			}()
 		}
 		/*
 			Loop through all series for this metric, processing all retentions and time ranges
@@ -121,7 +117,6 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 					case otsdbErr := <-errCh:
 						return fmt.Errorf("opentsdb error: %s", otsdbErr)
 					case vmErr := <-op.im.Errors():
-						otsdbErrorsTotal.Inc()
 						return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
 					case seriesCh <- queryObj{
 						Tr: tr, StartTime: startTime,
@@ -146,7 +141,6 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 	op.im.Close()
 	for vmErr := range op.im.Errors() {
 		if vmErr.Err != nil {
-			otsdbErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
 		}
 	}
@@ -177,9 +171,3 @@ func (op *otsdbProcessor) do(s queryObj) error {
 	}
 	return op.im.Input(&ts)
 }
-
-var (
-	otsdbSeriesTotal     = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_total`)
-	otsdbSeriesProcessed = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_processed`)
-	otsdbErrorsTotal     = vmetrics.NewCounter(`vmctl_opentsdb_migration_errors_total`)
-)

app/vmctl/prometheus.go

@@ -4,15 +4,11 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"strings"
 	"sync"
 
-	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
@@ -65,19 +61,19 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
-		var labelPairs []vm.LabelPair
+		var labels []vm.LabelPair
 		series := ss.At()
 
-		series.Labels().Range(func(label labels.Label) {
+		for _, label := range series.Labels() {
 			if label.Name == "__name__" {
 				name = label.Value
-				return
+				continue
 			}
-			labelPairs = append(labelPairs, vm.LabelPair{
-				Name:  strings.Clone(label.Name),
-				Value: strings.Clone(label.Value),
+			labels = append(labels, vm.LabelPair{
+				Name:  label.Name,
+				Value: label.Value,
 			})
-		})
+		}
 		if name == "" {
 			return fmt.Errorf("failed to find `__name__` label in labelset for block %v", b.Meta().ULID)
 		}
@@ -103,7 +99,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 		}
 		ts := vm.TimeSeries{
 			Name:       name,
-			LabelPairs: labelPairs,
+			LabelPairs: labels,
 			Timestamps: timestamps,
 			Values:     values,
 		}
@@ -115,7 +111,6 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 }
 
 func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
-	promBlocksTotal.Add(len(blocks))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -127,18 +122,18 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	pp.im.ResetStats()
 
 	var wg sync.WaitGroup
-	for range pp.cc {
-		wg.Go(func() {
+	wg.Add(pp.cc)
+	for i := 0; i < pp.cc; i++ {
+		go func() {
+			defer wg.Done()
 			for br := range blockReadersCh {
 				if err := pp.do(br); err != nil {
-					promErrorsTotal.Inc()
 					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
 					return
 				}
-				promBlocksProcessed.Inc()
 				bar.Increment()
 			}
-		})
+		}()
 	}
 	// any error breaks the import
 	for _, br := range blocks {
@@ -148,7 +143,6 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 			return fmt.Errorf("prometheus error: %s", promErr)
 		case vmErr := <-pp.im.Errors():
 			close(blockReadersCh)
-			promErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
 		case blockReadersCh <- br:
 		}
@@ -162,7 +156,6 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	// drain import errors channel
 	for vmErr := range pp.im.Errors() {
 		if vmErr.Err != nil {
-			promErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
 		}
 	}
@@ -172,9 +165,3 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 
 	return nil
 }
-
-var (
-	promBlocksTotal     = metrics.NewCounter(`vmctl_prometheus_migration_blocks_total`)
-	promBlocksProcessed = metrics.NewCounter(`vmctl_prometheus_migration_blocks_processed`)
-	promErrorsTotal     = metrics.NewCounter(`vmctl_prometheus_migration_errors_total`)
-)

app/vmctl/remoteread.go

@@ -7,8 +7,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
@@ -53,7 +51,6 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
-	remoteReadRangesTotal.Add(len(ranges))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing ranges"), len(ranges))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -69,18 +66,18 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	errCh := make(chan error)
 
 	var wg sync.WaitGroup
-	for range rrp.cc {
-		wg.Go(func() {
+	wg.Add(rrp.cc)
+	for i := 0; i < rrp.cc; i++ {
+		go func() {
+			defer wg.Done()
 			for r := range rangeC {
 				if err := rrp.do(ctx, r); err != nil {
-					remoteReadErrorsTotal.Inc()
 					errCh <- fmt.Errorf("request failed for: %s", err)
 					return
 				}
-				remoteReadRangesProcessed.Inc()
 				bar.Increment()
 			}
-		})
+		}()
 	}
 
 	for _, r := range ranges {
@@ -88,7 +85,6 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("remote read error: %s", infErr)
 		case vmErr := <-rrp.dst.Errors():
-			remoteReadErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
 		case rangeC <- &remoteread.Filter{
 			StartTimestampMs: r[0].UnixMilli(),
@@ -104,7 +100,6 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	// drain import errors channel
 	for vmErr := range rrp.dst.Errors() {
 		if vmErr.Err != nil {
-			remoteReadErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
 		}
 	}
@@ -125,9 +120,3 @@ func (rrp *remoteReadProcessor) do(ctx context.Context, filter *remoteread.Filte
 		return nil
 	})
 }
-
-var (
-	remoteReadRangesTotal     = metrics.NewCounter(`vmctl_remote_read_migration_ranges_total`)
-	remoteReadRangesProcessed = metrics.NewCounter(`vmctl_remote_read_migration_ranges_processed`)
-	remoteReadErrorsTotal     = metrics.NewCounter(`vmctl_remote_read_migration_errors_total`)
-)

app/vmctl/vm/vm.go

@@ -12,8 +12,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
@@ -82,12 +80,6 @@ type Importer struct {
 
 	s       *stats
 	backoff *backoff.Backoff
-
-	importRequestsTotal       *metrics.Counter
-	importRequestsErrorsTotal *metrics.Counter
-	importSamplesTotal        *metrics.Counter
-	importBytesTotal          *metrics.Counter
-	importDuration            *metrics.Histogram
 }
 
 // ResetStats resets im stats.
@@ -155,12 +147,6 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
 		backoff:    cfg.Backoff,
-
-		importRequestsTotal:       metrics.GetOrCreateCounter(`vmctl_importer_requests_total`),
-		importRequestsErrorsTotal: metrics.GetOrCreateCounter(`vmctl_importer_request_errors_total`),
-		importSamplesTotal:        metrics.GetOrCreateCounter(`vmctl_importer_samples_total`),
-		importBytesTotal:          metrics.GetOrCreateCounter(`vmctl_importer_bytes_total`),
-		importDuration:            metrics.GetOrCreateHistogram(`vmctl_importer_request_duration_seconds`),
 	}
 	if err := im.Ping(); err != nil {
 		return nil, fmt.Errorf("ping to %q failed: %s", addr, err)
@@ -170,13 +156,15 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		cfg.BatchSize = 1e5
 	}
 
-	for i := range int(cfg.Concurrency) {
+	im.wg.Add(int(cfg.Concurrency))
+	for i := 0; i < int(cfg.Concurrency); i++ {
 		pbPrefix := fmt.Sprintf(`{{ green "VM worker %d:" }}`, i)
 		bar := barpool.AddWithTemplate(pbPrefix+pbTpl, 0)
 
-		im.wg.Go(func() {
+		go func(bar barpool.Bar) {
+			defer im.wg.Done()
 			im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
-		})
+		}(bar)
 	}
 	im.ResetStats()
 	return im, nil
@@ -325,13 +313,9 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 		return nil
 	}
 
-	startTime := time.Now()
-	im.importRequestsTotal.Inc()
-
 	pr, pw := io.Pipe()
 	req, err := http.NewRequest(http.MethodPost, im.importPath, pr)
 	if err != nil {
-		im.importRequestsErrorsTotal.Inc()
 		return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
 	}
 	if im.user != "" {
@@ -351,7 +335,6 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	if im.compress {
 		zw, err := gzip.NewWriterLevel(w, 1)
 		if err != nil {
-			im.importRequestsErrorsTotal.Inc()
 			return fmt.Errorf("unexpected error when creating gzip writer: %s", err)
 		}
 		w = zw
@@ -363,39 +346,29 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	for _, ts := range tsBatch {
 		n, err := ts.write(bw)
 		if err != nil {
-			im.importRequestsErrorsTotal.Inc()
 			return fmt.Errorf("write err: %w", err)
 		}
 		totalBytes += n
 		totalSamples += len(ts.Values)
 	}
 	if err := bw.Flush(); err != nil {
-		im.importRequestsErrorsTotal.Inc()
 		return err
 	}
 	if closer, ok := w.(io.Closer); ok {
 		err := closer.Close()
 		if err != nil {
-			im.importRequestsErrorsTotal.Inc()
 			return err
 		}
 	}
 	if err := pw.Close(); err != nil {
-		im.importRequestsErrorsTotal.Inc()
 		return err
 	}
 
 	requestErr := <-errCh
 	if requestErr != nil {
-		im.importRequestsErrorsTotal.Inc()
-		im.importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("import request error for %q: %w", im.addr, requestErr)
 	}
 
-	im.importSamplesTotal.Add(totalSamples)
-	im.importBytesTotal.Add(totalBytes)
-	im.importDuration.UpdateDuration(startTime)
-
 	im.s.Lock()
 	im.s.bytes += uint64(totalBytes)
 	im.s.samples += uint64(totalSamples)

app/vmctl/vm_native.go

@@ -9,8 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
@@ -84,19 +82,13 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 		if !prompt(ctx, question) {
 			return nil
 		}
-		migrationTenantsTotal.Set(uint64(len(tenants)))
 	}
 
 	for _, tenantID := range tenants {
 		err := p.runBackfilling(ctx, tenantID, ranges)
 		if err != nil {
-			migrationErrorsTotal.Inc()
 			return fmt.Errorf("migration failed: %s", err)
 		}
-
-		if p.interCluster {
-			migrationTenantsProcessed.Inc()
-		}
 	}
 
 	log.Println("Import finished!")
@@ -164,7 +156,6 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
 	p.s.bytes += uint64(written)
 	p.s.requests++
 	p.s.Unlock()
-	migrationBytesTransferredTotal.AddInt64(written)
 
 	if err := pw.Close(); err != nil {
 		return err
@@ -208,7 +199,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var foundSeriesMsg string
 	var requestsToMake int
-	var metricsMap = map[string][][]time.Time{
+	var metrics = map[string][][]time.Time{
 		"": ranges,
 	}
 
@@ -220,11 +211,11 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	if !p.disablePerMetricRequests {
 		format = fmt.Sprintf(nativeWithBackoffTpl, barPrefix)
-		metricsMap, err = p.explore(ctx, p.src, tenantID, ranges)
+		metrics, err = p.explore(ctx, p.src, tenantID, ranges)
 		if err != nil {
 			return fmt.Errorf("failed to explore metric names: %s", err)
 		}
-		if len(metricsMap) == 0 {
+		if len(metrics) == 0 {
 			errMsg := "no metrics found"
 			if tenantID != "" {
 				errMsg = fmt.Sprintf("%s for tenant id: %s", errMsg, tenantID)
@@ -232,14 +223,10 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			log.Println(errMsg)
 			return nil
 		}
-		for _, m := range metricsMap {
+		for _, m := range metrics {
 			requestsToMake += len(m)
 		}
-		foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metricsMap), requestsToMake)
-
-		migrationMetricsTotal.Add(len(metricsMap))
-	} else {
-		requestsToMake = len(ranges)
+		foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metrics), requestsToMake)
 	}
 
 	if !p.interCluster {
@@ -253,7 +240,6 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		log.Print(foundSeriesMsg)
 	}
 
-	migrationRequestsPlanned.Add(requestsToMake)
 	bar := barpool.NewSingleProgress(format, requestsToMake)
 	bar.Start()
 	defer bar.Finish()
@@ -263,7 +249,9 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var wg sync.WaitGroup
 	for i := 0; i < p.cc; i++ {
-		wg.Go(func() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
 			for f := range filterCh {
 				if !p.disablePerMetricRequests {
 					if err := p.do(ctx, f, srcURL, dstURL, nil); err != nil {
@@ -277,13 +265,12 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 						return
 					}
 				}
-				migrationRequestsCompleted.Inc()
 			}
-		})
+		}()
 	}
 
 	// any error breaks the import
-	for mName, mRanges := range metricsMap {
+	for mName, mRanges := range metrics {
 		match, err := buildMatchWithFilter(p.filter.Match, mName)
 		if err != nil {
 			logger.Errorf("failed to build filter %q for metric name %q: %s", p.filter.Match, mName, err)
@@ -303,9 +290,6 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			}:
 			}
 		}
-		if !p.disablePerMetricRequests {
-			migrationMetricsProcessed.Inc()
-		}
 	}
 
 	close(filterCh)
@@ -414,18 +398,3 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) {
 	match := "{" + strings.Join(filters, " or ") + "}"
 	return match, nil
 }
-
-var (
-	migrationMetricsTotal     = metrics.NewCounter(`vmctl_vm_native_migration_metrics_total`)
-	migrationMetricsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_metrics_processed`)
-
-	migrationRequestsPlanned   = metrics.NewCounter(`vmctl_vm_native_migration_requests_planned`)
-	migrationRequestsCompleted = metrics.NewCounter(`vmctl_vm_native_migration_requests_completed`)
-
-	migrationErrorsTotal = metrics.NewCounter(`vmctl_vm_native_migration_errors_total`)
-
-	migrationTenantsTotal     = metrics.NewCounter(`vmctl_vm_native_migration_tenants_total`)
-	migrationTenantsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_tenants_processed`)
-
-	migrationBytesTransferredTotal = metrics.NewCounter(`vmctl_vm_native_migration_bytes_transferred_total`)
-)

app/vminsert/common/streamaggr.go

@@ -111,7 +111,9 @@ func InitStreamAggr() {
 	saCfgTimestamp.Set(fasttime.UnixTimestamp())
 
 	// Start config reloader.
-	saCfgReloaderWG.Go(func() {
+	saCfgReloaderWG.Add(1)
+	go func() {
+		defer saCfgReloaderWG.Done()
 		for {
 			select {
 			case <-sighupCh:
@@ -120,7 +122,7 @@ func InitStreamAggr() {
 			}
 			reloadStreamAggrConfig()
 		}
-	})
+	}()
 }
 
 func reloadStreamAggrConfig() {

app/vminsert/main.go

@@ -232,7 +232,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		firehose.WriteSuccessResponse(w, r)
 		return true
-	case "/zabbixconnector/api/v1/history":
+	case "zabbixconnector/api/v1/history":
 		zabbixconnectorHistoryRequests.Inc()
 		if err := zabbixconnector.InsertHandlerForHTTP(r); err != nil {
 			zabbixconnectorHistoryErrors.Inc()
@@ -241,7 +241,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			fmt.Fprintf(w, `{"error":%q}`, err.Error())
 			return true
 		}
-		w.WriteHeader(http.StatusOK)
+		w.WriteHeader(http.StatusAccepted)
 		return true
 	case "/newrelic":
 		newrelicCheckRequest.Inc()

app/vmselect/graphite/transform.go

@@ -3896,9 +3896,27 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 	seriesCh := make(chan *series, goroutines)
 	errCh := make(chan error, 1)
 	var wg sync.WaitGroup
+	wg.Add(goroutines)
+	go func() {
+		var err error
+		for {
+			s, e := nextSeries()
+			if e != nil || s == nil {
+				err = e
+				break
+			}
+			seriesCh <- s
+		}
+		close(seriesCh)
+		wg.Wait()
+		close(resultCh)
+		errCh <- err
+		close(errCh)
+	}()
 	var skipProcessing atomic.Bool
-	for range goroutines {
-		wg.Go(func() {
+	for i := 0; i < goroutines; i++ {
+		go func() {
+			defer wg.Done()
 			for s := range seriesCh {
 				if skipProcessing.Load() {
 					continue
@@ -3916,24 +3934,8 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 					}
 				}
 			}
-		})
+		}()
 	}
-	go func() {
-		var err error
-		for {
-			s, e := nextSeries()
-			if e != nil || s == nil {
-				err = e
-				break
-			}
-			seriesCh <- s
-		}
-		close(seriesCh)
-		wg.Wait()
-		close(resultCh)
-		errCh <- err
-		close(errCh)
-	}()
 	wrapper := func() (*series, error) {
 		r := <-resultCh
 		if r == nil {

app/vmselect/main.go

@@ -520,7 +520,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 			fmt.Fprintf(w, "%s", `{"status":"error","msg":"for accessing vmalert flag '-vmalert.proxyURL' must be configured"}`)
 			return true
 		}
-		proxyVMAlertRequests(w, r, path)
+		proxyVMAlertRequests(w, r)
 		return true
 	}
 
@@ -558,7 +558,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/rules", "/rules":
 		rulesRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r, path)
+			proxyVMAlertRequests(w, r)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
@@ -568,7 +568,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/alerts", "/alerts":
 		alertsRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r, path)
+			proxyVMAlertRequests(w, r)
 			return true
 		}
 		// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
@@ -578,7 +578,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	case "/api/v1/notifiers", "/notifiers":
 		notifiersRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
-			proxyVMAlertRequests(w, r, path)
+			proxyVMAlertRequests(w, r)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -725,7 +725,7 @@ var (
 	metricNamesStatsResetErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
 )
 
-func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request, path string) {
+func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		err := recover()
 		if err == nil || err == http.ErrAbortHandler {
@@ -736,10 +736,8 @@ func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request, path string) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
-	req := r.Clone(r.Context())
-	req.URL.Path = strings.TrimPrefix(path, "prometheus")
-	req.Host = vmalertProxyHost
-	vmalertProxy.ServeHTTP(w, req)
+	r.Host = vmalertProxyHost
+	vmalertProxy.ServeHTTP(w, r)
 }
 
 var (

app/vmselect/netstorage/netstorage.go

@@ -296,12 +296,14 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for workerID := range workChs {
-		qtChild := qt.NewChild("worker #%d", workerID)
-		wg.Go(func() {
-			timeseriesWorker(qtChild, workChs, uint(workerID))
+	for i := range workChs {
+		wg.Add(1)
+		qtChild := qt.NewChild("worker #%d", i)
+		go func(workerID uint) {
+			timeseriesWorker(qtChild, workChs, workerID)
 			qtChild.Done()
-		})
+			wg.Done()
+		}(uint(i))
 	}
 	wg.Wait()
 
@@ -512,10 +514,12 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for workerID := range workers {
-		wg.Go(func() {
-			unpackWorker(workChs, uint(workerID))
-		})
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func(workerID uint) {
+			unpackWorker(workChs, workerID)
+			wg.Done()
+		}(uint(i))
 	}
 	wg.Wait()
 
@@ -1016,10 +1020,12 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 		mustStop      atomic.Bool
 	)
 	var wg sync.WaitGroup
-	for workerID := range gomaxprocs {
-		wg.Go(func() {
+	wg.Add(gomaxprocs)
+	for i := 0; i < gomaxprocs; i++ {
+		go func(workerID uint) {
+			defer wg.Done()
 			for xw := range workCh {
-				if err := f(&xw.mn, &xw.b, tr, uint(workerID)); err != nil {
+				if err := f(&xw.mn, &xw.b, tr, workerID); err != nil {
 					errGlobalLock.Lock()
 					if errGlobal == nil {
 						errGlobal = err
@@ -1030,7 +1036,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 				xw.reset()
 				exportWorkPool.Put(xw)
 			}
-		})
+		}(uint(i))
 	}
 
 	// Feed workers with work

app/vmselect/promql/aggr_incremental_test.go

@@ -103,13 +103,15 @@ func testIncrementalParallelAggr(iafc *incrementalAggrFuncContext, tssSrc, tssEx
 	workersCount := netstorage.MaxWorkers()
 	tsCh := make(chan *timeseries)
 	var wg sync.WaitGroup
-	for workerID := range workersCount {
-		wg.Go(func() {
+	wg.Add(workersCount)
+	for i := 0; i < workersCount; i++ {
+		go func(workerID uint) {
+			defer wg.Done()
 			for ts := range tsCh {
 				runtime.Gosched() // allow other goroutines performing the work
-				iafc.updateTimeseries(ts, uint(workerID))
+				iafc.updateTimeseries(ts, workerID)
 			}
-		})
+		}(uint(i))
 	}
 	for _, ts := range tssSrc {
 		tsCh <- ts

app/vmselect/promql/binary_op.go

@@ -566,24 +566,15 @@ func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
 		return
 	}
 
-	// to avoid marshaling the same metric name multiple times, use a slice to store the results and an index to access them.
-	// todo: this may allocate a lot of []byte and could be further optimized.
-	nameRightSlice := make([][]byte, len(tssRight))
-	getRightName := func(idx int) []byte {
-		if nameRightSlice[idx] == nil {
-			nameRightSlice[idx] = marshalMetricNameSorted(nameRightSlice[idx], &tssRight[idx].MetricName)
-		}
-		return nameRightSlice[idx]
-	}
-
-	nameLeft := bbPool.Get()
+	nameLeft, nameRight := bbPool.Get(), bbPool.Get()
 	for _, tsLeft := range tssLeft {
 		valuesLeft := tsLeft.Values
 		nameLeft.B = marshalMetricNameSorted(nameLeft.B[:0], &tsLeft.MetricName)
 		for i, v := range valuesLeft {
 			leftIsNaN := math.IsNaN(v)
-			for rIdx, tsRight := range tssRight {
-				canBeMerged := bytes.Equal(nameLeft.B, getRightName(rIdx))
+			for _, tsRight := range tssRight {
+				nameRight.B = marshalMetricNameSorted(nameRight.B[:0], &tsRight.MetricName)
+				canBeMerged := bytes.Equal(nameLeft.B, nameRight.B)
 				valueRight := tsRight.Values[i]
 				if leftIsNaN && canBeMerged {
 					// fill NaNs with valueRight if labels match
@@ -598,6 +589,7 @@ func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
 		}
 	}
 	bbPool.Put(nameLeft)
+	bbPool.Put(nameRight)
 }
 
 func binaryOpIfnot(bfa *binaryOpFuncArg) ([]*timeseries, error) {

app/vmselect/promql/binary_op_timing_test.go

@@ -92,22 +92,6 @@ func BenchmarkBinaryOpOr(b *testing.B) {
 		}
 		benchmarkBinaryOpOr(b, bfa)
 	})
-
-	b.Run("tss:1000 or tss:40000: new", func(b *testing.B) {
-		left, right := make([]*timeseries, 1000), make([]*timeseries, 40000)
-		for i := range left {
-			left[i] = ts(fmt.Sprintf(`a{foo="%d"}`, i))
-		}
-		for i := range right {
-			right[i] = ts(fmt.Sprintf(`b{foo="%d"}`, i))
-		}
-		bfa := &binaryOpFuncArg{
-			be:    mustParseMetricsQL("a or b"),
-			left:  left,
-			right: right,
-		}
-		benchmarkBinaryOpOr(b, bfa)
-	})
 }
 
 func benchmarkBinaryOpOr(b *testing.B, bfa *binaryOpFuncArg) {

app/vmselect/promql/eval.go

@@ -477,18 +477,22 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		var tssFirst []*timeseries
 		var errFirst error
 		qtFirst := qt.NewChild("expr1")
-		wg.Go(func() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
 			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
 			qtFirst.Done()
-		})
+		}()
 
 		var tssSecond []*timeseries
 		var errSecond error
 		qtSecond := qt.NewChild("expr2")
-		wg.Go(func() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
 			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
 			qtSecond.Done()
-		})
+		}()
 
 		wg.Wait()
 		if errFirst != nil {
@@ -706,13 +710,17 @@ func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.
 	qt.Printf("eval function args in parallel")
 	var wg sync.WaitGroup
 	for i, e := range es {
+		wg.Add(1)
 		qtChild := qt.NewChild("eval arg %d", i)
-		wg.Go(func() {
-			defer qtChild.Done()
+		go func(e metricsql.Expr, i int) {
+			defer func() {
+				qtChild.Done()
+				wg.Done()
+			}()
 			rv, err := evalExpr(qtChild, ec, e)
 			rvs[i] = rv
 			errs[i] = err
-		})
+		}(e, i)
 	}
 	wg.Wait()
 	for _, err := range errs {
@@ -777,8 +785,7 @@ func getRollupExprArg(arg metricsql.Expr) *metricsql.RollupExpr {
 // - rollupFunc(m) if iafc is nil
 // - aggrFunc(rollupFunc(m)) if iafc isn't nil
 func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc, expr metricsql.Expr,
-	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
-) ([]*timeseries, error) {
+	re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
 	if re.At == nil {
 		return evalRollupFuncWithoutAt(qt, ec, funcName, rf, expr, re, iafc)
 	}
@@ -828,8 +835,7 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 }
 
 func evalRollupFuncWithoutAt(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext,
-) ([]*timeseries, error) {
+	expr metricsql.Expr, re *metricsql.RollupExpr, iafc *incrementalAggrFuncContext) ([]*timeseries, error) {
 	funcName = strings.ToLower(funcName)
 	ecNew := ec
 	var offset int64
@@ -1011,14 +1017,16 @@ func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, time
 	}
 
 	var wg sync.WaitGroup
-	for workerID := range workers {
-		wg.Go(func() {
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func(workerID uint) {
+			defer wg.Done()
 			var tmpValues []float64
 			var tmpTimestamps []int64
 			for ts := range workChs[workerID] {
-				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, uint(workerID))
+				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, workerID)
 			}
-		})
+		}(uint(i))
 	}
 	wg.Wait()
 }
@@ -1050,8 +1058,7 @@ func removeNanValues(dstValues []float64, dstTimestamps []int64, values []float6
 
 // evalInstantRollup evaluates instant rollup where ec.Start == ec.End.
 func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64,
-) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window int64) ([]*timeseries, error) {
 	if ec.Start != ec.End {
 		logger.Panicf("BUG: evalInstantRollup cannot be called on non-empty time range; got %s", ec.timeRangeString())
 	}
@@ -1076,12 +1083,10 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 		rollupResultCacheV.DeleteInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 	}
 	getCachedSeries := func(qt *querytracer.Tracer) ([]*timeseries, int64, error) {
-		rollupResultCacheV.rollupResultCacheRequests.Inc()
 	again:
 		offset := int64(0)
 		tssCached := rollupResultCacheV.GetInstantValues(qt, expr, window, ec.Step, ec.EnforcedTagFilterss)
 		if len(tssCached) == 0 {
-			rollupResultCacheV.rollupResultCacheMisses.Inc()
 			// Cache miss. Re-populate the missing data.
 			start := int64(fasttime.UnixTimestamp()*1000) - cacheTimestampOffset.Milliseconds()
 			offset = timestamp - start
@@ -1124,7 +1129,6 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			deleteCachedSeries(qt)
 			goto again
 		}
-		rollupResultCacheV.rollupResultCachePartialHits.Inc()
 		ec.QueryStats.addSeriesFetched(len(tssCached))
 		return tssCached, offset, nil
 	}
@@ -1533,11 +1537,16 @@ func assertInstantValues(tss []*timeseries) {
 	}
 }
 
-var memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
+var (
+	rollupResultCacheFullHits    = metrics.NewCounter(`vm_rollup_result_cache_full_hits_total`)
+	rollupResultCachePartialHits = metrics.NewCounter(`vm_rollup_result_cache_partial_hits_total`)
+	rollupResultCacheMiss        = metrics.NewCounter(`vm_rollup_result_cache_miss_total`)
+
+	memoryIntensiveQueries = metrics.NewCounter(`vm_memory_intensive_queries_total`)
+)
 
 func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr,
-) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
 	window, err := windowExpr.NonNegativeDuration(ec.Step)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", expr.AppendString(nil), err)
@@ -1573,20 +1582,19 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	}
 
 	// Search for cached results.
-	rollupResultCacheV.rollupResultCacheRequests.Inc()
 	tssCached, start := rollupResultCacheV.GetSeries(qt, ec, expr, window)
 	ec.QueryStats.addSeriesFetched(len(tssCached))
 	if start > ec.End {
 		qt.Printf("the result is fully cached")
-		rollupResultCacheV.rollupResultCacheFullHits.Inc()
+		rollupResultCacheFullHits.Inc()
 		return tssCached, nil
 	}
 	if start > ec.Start {
 		qt.Printf("partial cache hit")
-		rollupResultCacheV.rollupResultCachePartialHits.Inc()
+		rollupResultCachePartialHits.Inc()
 	} else {
 		qt.Printf("cache miss")
-		rollupResultCacheV.rollupResultCacheMisses.Inc()
+		rollupResultCacheMiss.Inc()
 	}
 
 	// Fetch missing results, which aren't cached yet.
@@ -1622,8 +1630,7 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 //
 // pointsPerSeries is used only for estimating the needed memory for query processing
 func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
-	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64,
-) ([]*timeseries, error) {
+	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, window, pointsPerSeries int64) ([]*timeseries, error) {
 	if qt.Enabled() {
 		qt = qt.NewChild("rollup %s: timeRange=%s, step=%d, window=%d", expr.AppendString(nil), ec.timeRangeString(), ec.Step, window)
 		defer qt.Done()
@@ -1713,7 +1720,6 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 		return nil, err
 	}
 	defer rml.Put(uint64(rollupMemorySize))
-	qs.addMemoryUsage(rollupMemorySize)
 	qt.Printf("the rollup evaluation needs an estimated %d bytes of RAM for %d series and %d points per series (summary %d points)",
 		rollupMemorySize, timeseriesLen, pointsPerSeries, rollupPoints)
 
@@ -1747,8 +1753,7 @@ func maxSilenceInterval() int64 {
 
 func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
 	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
-) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs)
 	defer qt.Done()
 	var samplesScannedTotal atomic.Uint64
@@ -1787,8 +1792,7 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string,
 }
 
 func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
-	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
-) ([]*timeseries, error) {
+	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
 
@@ -1828,8 +1832,7 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 }
 
 func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName,
-	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64,
-) uint64 {
+	valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) uint64 {
 	tsDst.MetricName.CopyFrom(mnSrc)
 	if len(rc.TagValue) > 0 {
 		tsDst.MetricName.AddTag("rollup", rc.TagValue)

app/vmselect/promql/exec.go

@@ -37,7 +37,7 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
 	if querystats.Enabled() {
 		startTime := time.Now()
 		defer func() {
-			querystats.RegisterQuery(q, ec.End-ec.Start, startTime, ec.QueryStats.memoryUsage())
+			querystats.RegisterQuery(q, ec.End-ec.Start, startTime)
 			ec.QueryStats.addExecutionTimeMsec(startTime)
 		}()
 	}

app/vmselect/promql/query_stats.go

@@ -13,8 +13,6 @@ type QueryStats struct {
 	ExecutionDuration atomic.Pointer[time.Duration]
 	// SeriesFetched contains the number of series fetched from storage or cache.
 	SeriesFetched atomic.Int64
-	// MemoryUsage contains the estimated memory consumption of the query
-	MemoryUsage atomic.Int64
 
 	at *auth.Token
 
@@ -55,17 +53,3 @@ func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
 	d := time.Since(startTime)
 	qs.ExecutionDuration.Store(&d)
 }
-
-func (qs *QueryStats) addMemoryUsage(memoryUsage int64) {
-	if qs == nil {
-		return
-	}
-	qs.MemoryUsage.Store(memoryUsage)
-}
-
-func (qs *QueryStats) memoryUsage() int64 {
-	if qs == nil {
-		return 0
-	}
-	return qs.MemoryUsage.Load()
-}

app/vmselect/promql/rollup.go

@@ -534,10 +534,7 @@ type rollupFuncArg struct {
 	timestamps []int64
 
 	// Real value preceding values.
-	// Is populated if the preceding sample falls within the rc.LookbackDelta range, or if rc.LookbackDelta is not set.
-	//
-	// It provides an additional check and value for rollup functions such as increase(), changes(),
-	// when the prevValue is NaN due to a gap or a small lookback window.
+	// Is populated if preceding value is within the rc.LookbackDelta.
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -716,11 +713,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	// Extend dstValues in order to remove mallocs below.
 	dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
 
-	// Set maxPrevInterval for subsequent rfa.prevValue calculations in rollupFunc:
-	// For instant queries, use rc.Step directly as maxPrevInterval.
-	// For range queries, rc.Step is typically too small to serve as the lookback window between two rollup points.
-	// Instead, estimate the scrape interval from raw sample timestamps (using the 0.6 quantile of the last 20 intervals)
-	// and slightly inflate the scrape interval to set maxPrevInterval, allowing for some tolerance to jitter.
+	// Use step as the scrape interval for instant queries (when start == end).
 	maxPrevInterval := rc.Step
 	if rc.Start < rc.End {
 		scrapeInterval := getScrapeInterval(timestamps, rc.Step)
@@ -736,21 +729,22 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		}
 	}
 	window := rc.Window
-	// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
-	// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
-	// when the window doesn't cover at least two raw samples (this is what most users expect).
-	//
-	// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
-	// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
-	//
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 	if window <= 0 {
 		window = rc.Step
 		if rc.MayAdjustWindow && window < maxPrevInterval {
+			// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
+			// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
+			// when the window doesn't cover at least two raw samples (this is what most users expect).
+			//
+			// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
+			// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
+			//
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 			window = maxPrevInterval
 		}
-		// Artificial window cannot exceed explicit rc.LookbackDelta, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 		if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
+			// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
+			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 			window = rc.LookbackDelta
 		}
 	}
@@ -875,17 +869,17 @@ func getScrapeInterval(timestamps []int64, defaultInterval int64) int64 {
 		return defaultInterval
 	}
 
-	// Estimate scrape interval as 0.6 quantile of the last 20 intervals.
-	tsPrev := timestamps[len(timestamps)-1]
-	timestamps = timestamps[:len(timestamps)-1]
+	// Estimate scrape interval as 0.6 quantile for the first 20 intervals.
+	tsPrev := timestamps[0]
+	timestamps = timestamps[1:]
 	if len(timestamps) > 20 {
-		timestamps = timestamps[len(timestamps)-20:]
+		timestamps = timestamps[:20]
 	}
 	a := getFloat64s()
 	intervals := a.A[:0]
-	for i := len(timestamps) - 1; i >= 0; i-- {
-		intervals = append(intervals, float64(tsPrev-timestamps[i]))
-		tsPrev = timestamps[i]
+	for _, ts := range timestamps {
+		intervals = append(intervals, float64(ts-tsPrev))
+		tsPrev = ts
 	}
 	scrapeInterval := int64(quantile(0.6, intervals))
 	a.A = intervals
@@ -2113,15 +2107,9 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		// Assume that the value didn't change during the current gap
-		// if realPrevValue exists.
-		if !math.IsNaN(rfa.realPrevValue) {
-			prevValue = rfa.realPrevValue
-		} else {
-			n++
-			prevValue = values[0]
-			values = values[1:]
-		}
+		prevValue = values[0]
+		values = values[1:]
+		n++
 	}
 	for _, v := range values {
 		if v != prevValue {

app/vmselect/promql/rollup_result_cache.go

@@ -83,11 +83,9 @@ func checkRollupResultCacheReset() {
 
 const checkRollupResultCacheResetInterval = 5 * time.Second
 
-var (
-	needRollupResultCacheReset       atomic.Bool
-	checkRollupResultCacheResetOnce  sync.Once
-	rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
-)
+var needRollupResultCacheReset atomic.Bool
+var checkRollupResultCacheResetOnce sync.Once
+var rollupResultResetMetricRowSample atomic.Pointer[storage.MetricRow]
 
 var rollupResultCacheV = &rollupResultCache{
 	c: workingsetcache.New(1024 * 1024), // This is a cache for testing.
@@ -180,12 +178,6 @@ func InitRollupResultCache(cachePath string) {
 
 	rollupResultCacheV = &rollupResultCache{
 		c: c,
-
-		rollupResultCacheRequests:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_requests_total`),
-		rollupResultCacheFullHits:    metrics.GetOrCreateCounter(`vm_rollup_result_cache_full_hits_total`),
-		rollupResultCachePartialHits: metrics.GetOrCreateCounter(`vm_rollup_result_cache_partial_hits_total`),
-		rollupResultCacheMisses:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_miss_total`),
-		rollupResultCacheResets:      metrics.GetOrCreateCounter(`vm_rollup_result_cache_resets_total`),
 	}
 }
 
@@ -201,18 +193,13 @@ func StopRollupResultCache() {
 
 type rollupResultCache struct {
 	c *workingsetcache.Cache
-
-	rollupResultCacheRequests    *metrics.Counter
-	rollupResultCacheFullHits    *metrics.Counter
-	rollupResultCachePartialHits *metrics.Counter
-	rollupResultCacheMisses      *metrics.Counter
-
-	rollupResultCacheResets *metrics.Counter
 }
 
+var rollupResultCacheResets = metrics.NewCounter(`vm_cache_resets_total{type="promql/rollupResult"}`)
+
 // ResetRollupResultCache resets rollup result cache.
 func ResetRollupResultCache() {
-	rollupResultCacheV.rollupResultCacheResets.Inc()
+	rollupResultCacheResets.Inc()
 	rollupResultCacheKeyPrefix.Add(1)
 	logger.Infof("rollupResult cache has been cleared")
 }

app/vmselect/promql/rollup_test.go

@@ -232,7 +232,6 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
 	}
 	var rfa rollupFuncArg
 	rfa.prevValue = nan
-	rfa.realPrevValue = nan
 	rfa.prevTimestamp = 0
 	rfa.values = append(rfa.values, testValues...)
 	rfa.timestamps = append(rfa.timestamps, testTimestamps...)
@@ -1655,7 +1654,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1675,7 +1674,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1795,7 +1794,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1815,7 +1814,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1889,126 +1888,3 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
-
-func TestRollupChangesWithStaleness(t *testing.T) {
-	// there is a gap between samples in the dataset below
-	timestamps := []int64{0, 15000, 30000, 70000}
-	values := []float64{1, 1, 1, 1}
-
-	// if step > gap, then changes will always respect value before gap
-	t.Run("step>gap", func(t *testing.T) {
-		rc := rollupConfig{
-			Func:               rollupChanges,
-			Start:              0,
-			End:                70000,
-			Step:               45000,
-			Window:             0,
-			MaxPointsPerSeries: 1e4,
-		}
-		rc.Timestamps = rc.getTimestamps()
-		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
-		if samplesScanned != 7 {
-			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
-		}
-		valuesExpected := []float64{1, 0}
-		timestampsExpected := []int64{0, 45e3}
-		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
-	})
-	// even if LookbackDelta < gap
-	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
-		rc := rollupConfig{
-			Func:               rollupChanges,
-			Start:              0,
-			End:                70000,
-			Step:               45000,
-			LookbackDelta:      10e3,
-			Window:             0,
-			MaxPointsPerSeries: 1e4,
-		}
-		rc.Timestamps = rc.getTimestamps()
-		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
-		if samplesScanned != 7 {
-			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
-		}
-		valuesExpected := []float64{1, 0}
-		timestampsExpected := []int64{0, 45e3}
-		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
-	})
-
-	// if step < gap and LookbackDelta>0 then changes will respect value before gap
-	// only if it is not stale according to LookbackDelta
-	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
-		rc := rollupConfig{
-			Func:               rollupChanges,
-			Start:              0,
-			End:                70000,
-			Step:               10000,
-			Window:             0,
-			MaxPointsPerSeries: 1e4,
-			LookbackDelta:      30e3,
-		}
-		rc.Timestamps = rc.getTimestamps()
-		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
-		if samplesScanned != 8 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
-		}
-		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 1}
-		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
-		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
-	})
-
-	// there is a staleness marker between samples in the dataset below
-	timestamps = []int64{0, 10000, 20000, 30000, 40000}
-	values = []float64{1, 1, 1, decimal.StaleNaN, 1}
-
-	t.Run("staleness marker", func(t *testing.T) {
-		rc := rollupConfig{
-			Func:               rollupChanges,
-			Start:              0,
-			End:                40000,
-			Step:               10000,
-			Window:             0,
-			MaxPointsPerSeries: 1e4,
-		}
-		rc.Timestamps = rc.getTimestamps()
-		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
-		if samplesScanned != 10 {
-			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
-		}
-		valuesExpected := []float64{1, 0, 0, 1, 1}
-		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3}
-		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
-	})
-
-	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280
-	//
-	// When there are gaps between samples that exceed maxPrevInterval,
-	// either due to changes in the scrape interval or missing scrapes.
-	// For example, if the scrape interval was initially 30s and later changed to 10s,
-	// the auto-calculated scrape interval is 10s, with maxPrevInterval inflated to 15s.
-	//
-	// At t=30s:
-	// prevValue is NaN, as the last sample at t=0s is considered stale for t=30s given the maxPrevInterval.
-	// realPrevValue is 1, taken from t=0s, since LookbackDelta=0 ignores staleness.
-	// the result should be `changes(1, 1) -> 0` instead of `changes(1, NaN)`.
-	// At t=100s:
-	// preValue is also NaN, as the last sample at t=70s is considered stale for t=100s.
-	// realPrevValue is 1, taken from t=70s,
-	// result should be `changes(2, 1) -> 1`.
-	timestamps = []int64{0, 30000, 40000, 50000, 60000, 70000, 100000}
-	values = []float64{1, 1, 1, 1, 1, 1, 2}
-	t.Run("issue-10280", func(t *testing.T) {
-		rc := rollupConfig{
-			Func:               rollupChanges,
-			Start:              0,
-			End:                100e3,
-			Step:               10e3,
-			MaxPointsPerSeries: 1e4,
-		}
-		rc.Timestamps = rc.getTimestamps()
-		gotValues, _ := rc.Do(nil, values, timestamps)
-		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
-		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3, 80e3, 90e3, 100e3}
-		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
-	})
-}

app/vmselect/querystats/querystats.go

@@ -8,7 +8,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 )
@@ -16,8 +15,7 @@ import (
 var (
 	lastQueriesCount = flag.Int("search.queryStats.lastQueriesCount", 20000, "Query stats for /api/v1/status/top_queries is tracked on this number of last queries. "+
 		"Zero value disables query stats tracking")
-	minQueryDuration    = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
-	minQueryMemoryUsage = flagutil.NewBytes("search.queryStats.minQueryMemoryUsage", 1024, "The minimum memory bytes consumption for queries to track in query stats at /api/v1/status/top_queries. Queries with lower memory bytes consumption are ignored in query stats")
+	minQueryDuration = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
 )
 
 var (
@@ -33,9 +31,9 @@ func Enabled() bool {
 // RegisterQuery registers the query on the given timeRangeMsecs, which has been started at startTime.
 //
 // RegisterQuery must be called when the query is finished.
-func RegisterQuery(query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
+func RegisterQuery(query string, timeRangeMsecs int64, startTime time.Time) {
 	initOnce.Do(initQueryStats)
-	qsTracker.registerQuery(query, timeRangeMsecs, startTime, memoryUsage)
+	qsTracker.registerQuery(query, timeRangeMsecs, startTime)
 }
 
 // WriteJSONQueryStats writes query stats to given writer in json format.
@@ -56,7 +54,6 @@ type queryStatRecord struct {
 	timeRangeSecs int64
 	registerTime  time.Time
 	duration      time.Duration
-	memoryUsage   int64
 }
 
 type queryStatKey struct {
@@ -69,8 +66,8 @@ func initQueryStats() {
 	if recordsCount <= 0 {
 		recordsCount = 1
 	} else {
-		logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s, -search.queryStats.minQueryMemoryUsage=%s",
-			*lastQueriesCount, *minQueryDuration, minQueryMemoryUsage)
+		logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s",
+			*lastQueriesCount, *minQueryDuration)
 	}
 	qsTracker = &queryStatsTracker{
 		a: make([]queryStatRecord, recordsCount),
@@ -81,7 +78,6 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
 	fmt.Fprintf(w, `{"topN":"%d","maxLifetime":"%s",`, topN, maxLifetime)
 	fmt.Fprintf(w, `"search.queryStats.lastQueriesCount":%d,`, *lastQueriesCount)
 	fmt.Fprintf(w, `"search.queryStats.minQueryDuration":"%s",`, *minQueryDuration)
-	fmt.Fprintf(w, `"search.queryStats.minQueryMemoryUsage":"%s",`, minQueryMemoryUsage)
 	fmt.Fprintf(w, `"topByCount":[`)
 	topByCount := qst.getTopByCount(topN, maxLifetime)
 	for i, r := range topByCount {
@@ -106,28 +102,15 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, maxLife
 			fmt.Fprintf(w, `,`)
 		}
 	}
-
-	fmt.Fprintf(w, `],"topByAvgMemoryUsage":[`)
-	topByAvgMemoryConsumption := qst.getTopByAvgMemoryUsage(topN, maxLifetime)
-	for i, r := range topByAvgMemoryConsumption {
-		fmt.Fprintf(w, `{"query":%s,"timeRangeSeconds":%d,"avgMemoryBytes":%d,"count":%d}`, stringsutil.JSONString(r.query), r.timeRangeSecs, r.memoryUsage, r.count)
-		if i+1 < len(topByAvgMemoryConsumption) {
-			fmt.Fprintf(w, `,`)
-		}
-	}
-
 	fmt.Fprintf(w, `]}`)
 }
 
-func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
+func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64, startTime time.Time) {
 	registerTime := time.Now()
 	duration := registerTime.Sub(startTime)
 	if duration < *minQueryDuration {
 		return
 	}
-	if memoryUsage < int64(minQueryMemoryUsage.IntN()) {
-		return
-	}
 
 	qst.mu.Lock()
 	defer qst.mu.Unlock()
@@ -143,7 +126,6 @@ func (qst *queryStatsTracker) registerQuery(query string, timeRangeMsecs int64,
 	r.timeRangeSecs = timeRangeMsecs / 1000
 	r.registerTime = registerTime
 	r.duration = duration
-	r.memoryUsage = memoryUsage
 }
 
 func (r *queryStatRecord) matches(currentTime time.Time, maxLifetime time.Duration) bool {
@@ -275,47 +257,3 @@ func (qst *queryStatsTracker) getTopBySumDuration(topN int, maxLifetime time.Dur
 	}
 	return a
 }
-
-type queryStatByMemory struct {
-	query         string
-	timeRangeSecs int64
-	memoryUsage   int64
-	count         int
-}
-
-func (qst *queryStatsTracker) getTopByAvgMemoryUsage(topN int, maxLifetime time.Duration) []queryStatByMemory {
-	currentTime := time.Now()
-	qst.mu.Lock()
-	type countSum struct {
-		count int
-		sum   int64
-	}
-	m := make(map[queryStatKey]countSum)
-	for _, r := range qst.a {
-		if r.matches(currentTime, maxLifetime) {
-			k := r.key()
-			ks := m[k]
-			ks.count++
-			ks.sum += r.memoryUsage
-			m[k] = ks
-		}
-	}
-	qst.mu.Unlock()
-
-	var a []queryStatByMemory
-	for k, ks := range m {
-		a = append(a, queryStatByMemory{
-			query:         k.query,
-			timeRangeSecs: k.timeRangeSecs,
-			memoryUsage:   ks.sum / int64(ks.count),
-			count:         ks.count,
-		})
-	}
-	sort.Slice(a, func(i, j int) bool {
-		return a[i].memoryUsage > a[j].memoryUsage
-	})
-	if len(a) > topN {
-		a = a[:topN]
-	}
-	return a
-}

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-C4RD5Sxk.js"></script>
-  <link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
+  <script type="module" crossorigin src="./assets/index-Clpj_g75.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-D5YL0cqB.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-D7CzMv1O.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-jEWkrqzO.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -29,8 +29,7 @@ import (
 )
 
 var (
-	retentionPeriod = flagutil.NewRetentionDuration("retentionPeriod", "1M", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. "+
-		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
+	retentionPeriod   = flagutil.NewRetentionDuration("retentionPeriod", "1", "Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See also -retentionFilter")
 	snapshotAuthKey   = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
 	forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
 	forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -389,23 +388,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	case "/create":
 		snapshotsCreateTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
-		snapshotName := Storage.MustCreateSnapshot()
-
-		// Verify whether the client already closed the connection.
-		// In this case it is better to drop the created snapshot, since the client isn't interested in it.
-		if err := r.Context().Err(); err != nil {
-			logger.Infof("deleting already created snapshot at %s because the client canceled the request", snapshotName)
-			if err := deleteSnapshot(snapshotName); err != nil {
-				logger.Infof("cannot delete just created snapshot: %s", err)
-				return true
-			}
-			return true
-		}
-
+		snapshotPath := Storage.MustCreateSnapshot()
 		if prometheusCompatibleResponse {
-			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotName))
+			fmt.Fprintf(w, `{"status":"success","data":{"name":%s}}`, stringsutil.JSONString(snapshotPath))
 		} else {
-			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotName))
+			fmt.Fprintf(w, `{"status":"ok","snapshot":%s}`, stringsutil.JSONString(snapshotPath))
 		}
 		return true
 	case "/list":
@@ -425,12 +412,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		snapshotsDeleteTotal.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		snapshotName := r.FormValue("snapshot")
-		if err := deleteSnapshot(snapshotName); err != nil {
-			jsonResponseError(w, err)
-			snapshotsDeleteErrorsTotal.Inc()
-			return true
+
+		snapshots := Storage.MustListSnapshots()
+		for _, snName := range snapshots {
+			if snName == snapshotName {
+				if err := Storage.DeleteSnapshot(snName); err != nil {
+					err = fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
+					jsonResponseError(w, err)
+					snapshotsDeleteErrorsTotal.Inc()
+					return true
+				}
+				fmt.Fprintf(w, `{"status":"ok"}`)
+				return true
+			}
 		}
-		fmt.Fprintf(w, `{"status":"ok"}`)
+
+		err := fmt.Errorf("cannot find snapshot %q", snapshotName)
+		jsonResponseError(w, err)
 		return true
 	case "/delete_all":
 		snapshotsDeleteAllTotal.Inc()
@@ -451,26 +449,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
-func deleteSnapshot(snapshotName string) error {
-	snapshots := Storage.MustListSnapshots()
-	for _, snName := range snapshots {
-		if snName == snapshotName {
-			if err := Storage.DeleteSnapshot(snName); err != nil {
-				return fmt.Errorf("cannot delete snapshot %q: %w", snName, err)
-			}
-			return nil
-		}
-	}
-	return fmt.Errorf("cannot find snapshot %q", snapshotName)
-}
-
 func initStaleSnapshotsRemover(strg *storage.Storage) {
 	staleSnapshotsRemoverCh = make(chan struct{})
 	if snapshotsMaxAge.Duration() <= 0 {
 		return
 	}
 	snapshotsMaxAgeDur := snapshotsMaxAge.Duration()
-	staleSnapshotsRemoverWG.Go(func() {
+	staleSnapshotsRemoverWG.Add(1)
+	go func() {
+		defer staleSnapshotsRemoverWG.Done()
 		d := timeutil.AddJitterToDuration(time.Second * 11)
 		t := time.NewTicker(d)
 		defer t.Stop()
@@ -482,7 +469,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 			}
 			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
 		}
-	})
+	}()
 }
 
 func stopStaleSnapshotsRemover() {

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.6 AS build-web-stage
+FROM golang:1.25.5 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/Makefile

@@ -1,23 +1,26 @@
 # All these commands must run from repository root.
+
 copy-metricsql-docs:
 	cp docs/victoriametrics/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
 
-vmui-run-npm-command: vmui-package-base-image
+vmui-package-base-image:
+	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
+
+vmui-build: copy-metricsql-docs vmui-package-base-image
 	docker run --rm \
 		--user $(shell id -u):$(shell id -g) \
 		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
 		-w /build/packages/vmui \
 		--entrypoint=/bin/bash \
-		vmui-builder-image -c "[ \"$$VMUI_SKIP_INSTALL\" = \"true\" ] || npm ci; $(NPM_COMMAND)"
+		vmui-builder-image -c "npm install && npm run build"
 
-vmui-install:
-	NPM_COMMAND="true" $(MAKE) vmui-run-npm-command
-
-vmui-package-base-image:
-	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
-
-vmui-build: copy-metricsql-docs
-	NPM_COMMAND="npm run build" $(MAKE) vmui-run-npm-command
+vmui-anomaly-build: vmui-package-base-image
+	docker run --rm \
+		--user $(shell id -u):$(shell id -g) \
+		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
+		-w /build/packages/vmui \
+		--entrypoint=/bin/bash \
+		vmui-builder-image -c "npm install && npm run build:anomaly"
 
 vmui-release: vmui-build
 	docker build -t ${DOCKER_NAMESPACE}/vmui:latest -f app/vmui/Dockerfile-web ./app/vmui/packages/vmui
@@ -35,11 +38,11 @@ vmui-update: vmui-build
 vmui-install-dependencies:
 	cd app/vmui/packages/vmui && npm ci
 
-vmui-lint:
-	NPM_COMMAND="npm run lint" $(MAKE) vmui-run-npm-command
+vmui-lint: vmui-install-dependencies
+	cd app/vmui/packages/vmui && npm run lint
 
-vmui-typecheck:
-	NPM_COMMAND="npm run typecheck" $(MAKE) vmui-run-npm-command
+vmui-typecheck: vmui-install-dependencies
+	cd app/vmui/packages/vmui && npm run typecheck
 
-vmui-test:
-	NPM_COMMAND="npm run test" $(MAKE) vmui-run-npm-command
+vmui-test: vmui-install-dependencies
+	cd app/vmui/packages/vmui && npm run test

app/vmui/packages/vmui/.env.vmanomaly

@@ -0,0 +1 @@
+VITE_APP_TYPE=vmanomaly

app/vmui/packages/vmui/config/plugins/dynamicIndexHtml.ts

@@ -0,0 +1,23 @@
+import { readFile } from "fs/promises";
+import { IndexHtmlTransform } from "vite";
+
+/**
+ * Vite plugin to dynamically load index.html based on the current mode.
+ * If a specific mode-based index file (e.g., index.vmanomaly.html) exists, it is used.
+ * Otherwise, the default index.html is loaded.
+ */
+export default function dynamicIndexHtmlPlugin({ mode }) {
+  return {
+    name: "vm-dynamic-index-html",
+    transformIndexHtml: {
+      order: "pre",
+      handler: async () => {
+        try {
+          return await readFile(`./index.${mode}.html`, "utf8");
+        } catch (error) {
+          return await readFile("./index.html", "utf8");
+        }
+      }
+    } as IndexHtmlTransform
+  };
+}

app/vmui/packages/vmui/eslint.config.js

@@ -46,7 +46,7 @@ export default [...compat.extends(
   settings: {
     react: {
       pragma: "React",
-      version: "19.0",
+      version: "detect",
     },
 
     linkComponents: ["Hyperlink", {
@@ -69,11 +69,10 @@ export default [...compat.extends(
       "varsIgnorePattern": "^_",
       "ignoreRestSiblings": true
     }],
-
+    
     "unused-imports/no-unused-imports": "error",
 
     "react/jsx-closing-bracket-location": [1, "line-aligned"],
-    "object-curly-spacing": [2, "always"],
 
     "react/jsx-max-props-per-line": [1, {
       maximum: 1,
@@ -82,23 +81,13 @@ export default [...compat.extends(
     "react/jsx-first-prop-new-line": [1, "multiline"],
 
     // Disable core indent rule due to recursion issues in ESLint 9; use JSX-specific rules instead
-    indent: ["error", 2, {
-      SwitchCase: 1,
-      ignoredNodes: [
-        "JSXElement",
-        "JSXElement *",
-        "JSXFragment",
-        "JSXFragment *",
-      ],
-    }],
+    indent: "off",
     "react/jsx-indent": ["error", 2],
     "react/jsx-indent-props": ["error", 2],
 
-    "linebreak-style": ["error", "unix"],
-    quotes: ["error", "double"],
-    semi: ["error", "always"],
     // Formatting rules moved out of ESLint core; omit here to avoid deprecation noise
     "react/prop-types": 0,
     "react/react-in-jsx-scope": "off",
+
   },
 }];

app/vmui/packages/vmui/index.vmanomaly.html

@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="/favicon.svg" />
+  <link rel="apple-touch-icon" href="/favicon.svg" />
+  <link rel="mask-icon" href="/favicon.svg" color="#000000">
+
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta name="description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI"/>
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="/manifest.json" crossorigin="use-credentials"/>
+  <!--
+    Notice the use of in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>UI for VictoriaMetrics Anomaly Detection</title>
+
+  <meta name="twitter:card" content="summary">
+  <meta name="twitter:title" content="UI for VictoriaMetrics Anomaly Detection">
+  <meta name="twitter:site" content="@https://victoriametrics.com/products/enterprise/anomaly-detection/">
+  <meta name="twitter:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
+  <meta name="twitter:image" content="/preview.jpg">
+
+  <meta property="og:type" content="website">
+  <meta property="og:title" content="UI for VictoriaMetrics Anomaly Detection">
+  <meta property="og:url" content="https://victoriametrics.com/products/enterprise/anomaly-detection/">
+  <meta property="og:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+<script type="module" src="/src/index.tsx"></script>
+</body>
+</html>

app/vmui/packages/vmui/package.json

@@ -7,8 +7,10 @@
   "scripts": {
     "prestart": "npm run copy-metricsql-docs",
     "start": "vite",
-    "start:playground": "cross-env PLAYGROUND=true npm run start",
+    "start:playground": "cross-env PLAYGROUND=METRICS npm run start",
+    "start:anomaly": "vite --mode vmanomaly",
     "build": "vite build",
+    "build:anomaly": "vite build --mode vmanomaly",
     "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
     "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
@@ -16,48 +18,47 @@
     "preview": "vite preview",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
-    "test:dev": "vitest",
-    "precommit": "npm run lint:local && npm run typecheck && npm run test"
+    "test:dev": "vitest"
   },
   "dependencies": {
     "classnames": "^2.5.1",
-    "dayjs": "^1.11.19",
+    "dayjs": "^1.11.13",
     "lodash.debounce": "^4.0.8",
-    "marked": "^17.0.1",
-    "preact": "^10.28.2",
-    "qs": "^6.14.1",
+    "marked": "^16.0.0",
+    "preact": "^10.26.9",
+    "qs": "^6.14.0",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.12.0",
+    "react-router-dom": "^7.6.3",
     "uplot": "^1.6.32",
-    "vite": "^7.3.1",
-    "web-vitals": "^5.1.0"
+    "vite": "^7.1.11",
+    "web-vitals": "^5.0.3"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.3.3",
-    "@eslint/js": "^9.39.2",
+    "@eslint/eslintrc": "^3.3.1",
+    "@eslint/js": "^9.30.1",
     "@preact/preset-vite": "^2.10.2",
-    "@testing-library/jest-dom": "^6.9.1",
+    "@testing-library/jest-dom": "^6.6.3",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^25.0.8",
+    "@types/node": "^24.0.12",
     "@types/qs": "^6.14.0",
-    "@types/react": "^19.2.8",
+    "@types/react": "^19.1.8",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.53.0",
-    "@typescript-eslint/parser": "^8.53.0",
-    "cross-env": "^10.1.0",
-    "eslint": "^9.39.2",
+    "@typescript-eslint/eslint-plugin": "^8.36.0",
+    "@typescript-eslint/parser": "^8.36.0",
+    "cross-env": "^7.0.3",
+    "eslint": "^9.30.1",
     "eslint-plugin-react": "^7.37.5",
-    "eslint-plugin-unused-imports": "^4.3.0",
-    "globals": "^17.0.0",
+    "eslint-plugin-unused-imports": "^4.1.4",
+    "globals": "^16.3.0",
     "http-proxy-middleware": "^3.0.5",
-    "jsdom": "^27.4.0",
+    "jsdom": "^26.1.0",
     "postcss": "^8.5.6",
-    "rollup-plugin-visualizer": "^6.0.5",
-    "sass-embedded": "^1.97.2",
-    "typescript": "^5.9.3",
-    "vitest": "^4.0.17"
+    "rollup-plugin-visualizer": "^6.0.3",
+    "sass-embedded": "^1.89.2",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.4"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/AppAnomaly.tsx

@@ -0,0 +1,41 @@
+import { FC, useState } from "preact/compat";
+import { HashRouter, Route, Routes } from "react-router-dom";
+import AppContextProvider from "./contexts/AppContextProvider";
+import ThemeProvider from "./components/Main/ThemeProvider/ThemeProvider";
+import AnomalyLayout from "./layouts/AnomalyLayout/AnomalyLayout";
+import ExploreAnomaly from "./pages/ExploreAnomaly/ExploreAnomaly";
+import router from "./router";
+import CustomPanel from "./pages/CustomPanel";
+
+const AppAnomaly: FC = () => {
+  const [loadedTheme, setLoadedTheme] = useState(false);
+
+  return <>
+    <HashRouter>
+      <AppContextProvider>
+        <>
+          <ThemeProvider onLoaded={setLoadedTheme}/>
+          {loadedTheme && (
+            <Routes>
+              <Route
+                path={"/"}
+                element={<AnomalyLayout/>}
+              >
+                <Route
+                  path={"/"}
+                  element={<ExploreAnomaly/>}
+                />
+                <Route
+                  path={router.query}
+                  element={<CustomPanel/>}
+                />
+              </Route>
+            </Routes>
+          )}
+        </>
+      </AppContextProvider>
+    </HashRouter>
+  </>;
+};
+
+export default AppAnomaly;

app/vmui/packages/vmui/src/components/Chart/Line/Legend/Legend.tsx

@@ -14,11 +14,12 @@ export type QueryGroup = {
 interface LegendProps {
   labels: LegendItemType[];
   query: string[];
+  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
 
-const Legend: FC<LegendProps> = ({ labels, query, isPredefinedPanel, onChange }) => {
+const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, isPredefinedPanel, onChange }) => {
   const { groupByLabel } = useLegendGroup();
   const groupSeries = useGroupSeries({ labels, query, groupByLabel });
 
@@ -32,6 +33,7 @@ const Legend: FC<LegendProps> = ({ labels, query, isPredefinedPanel, onChange })
             key={group}
             labels={items}
             group={group}
+            isAnomalyView={isAnomalyView}
             onChange={onChange}
           />
         ))}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -13,6 +13,7 @@ import { getFromStorage } from "../../../../utils/storage";
 
 export type LegendProps = {
   labels: LegendItemType[];
+  isAnomalyView?: boolean;
   duplicateFields?: string[];
   onChange: (item: LegendItemType, metaKey: boolean) => void;
 }
@@ -21,7 +22,7 @@ interface LegendGroupProps extends LegendProps {
   group: string | number;
 }
 
-const LegendGroup: FC<LegendGroupProps> = ({ labels, group, onChange }) => {
+const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onChange }) => {
   const { isTableView } = useLegendView();
   const { groupByLabel } = useLegendGroup();
   const copyToClipboard = useCopyToClipboard();
@@ -38,14 +39,14 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, onChange }) => {
 
   const Content = isTableView ? LegendTable : LegendLines;
 
-  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false";
-  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT;
+  const disableAutoCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE") === "false"
+  const defaultExpanded = disableAutoCollapse ? true : sortedLabels.length <= LEGEND_COLLAPSE_SERIES_LIMIT
 
   const expandedWarning = (
     <span className="vm-legend-group-header__warning">
       Legend collapsed by default ({sortedLabels.length} series) — click to expand.
     </span>
-  );
+  )
 
   return (
     <div
@@ -80,6 +81,7 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, onChange }) => {
       >
         <Content
           labels={sortedLabels}
+          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendItem/LegendItem.tsx

@@ -13,10 +13,11 @@ import { getLabelAlias } from "../../../../../utils/metric";
 interface LegendItemProps {
   legend: LegendItemType;
   onChange?: (item: LegendItemType, metaKey: boolean) => void;
+  isAnomalyView?: boolean;
   duplicateFields?: string[];
 }
 
-const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields }) => {
+const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields, isAnomalyView }) => {
   const copyToClipboard = useCopyToClipboard();
   const { hideStats } = useShowStats();
 
@@ -51,10 +52,12 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange, duplicateFields })
       })}
       onClick={createHandlerClick(legend)}
     >
-      <div
-        className="vm-legend-item__marker"
-        style={{ backgroundColor: legend.color }}
-      />
+      {!isAnomalyView && (
+        <div
+          className="vm-legend-item__marker"
+          style={{ backgroundColor: legend.color }}
+        />
+      )}
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
           {legend.hasAlias && legend.label}

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendViews/LegendLines.tsx

@@ -2,7 +2,7 @@ import { FC } from "preact/compat";
 import LegendItem from "../LegendItem/LegendItem";
 import { LegendProps } from "../LegendGroup";
 
-const LegendLines: FC<LegendProps> = ({ labels, duplicateFields, onChange }) => {
+const LegendLines: FC<LegendProps> = ({ labels, isAnomalyView, duplicateFields, onChange }) => {
 
   return (
     <div className="vm-legend-item-container">
@@ -10,6 +10,7 @@ const LegendLines: FC<LegendProps> = ({ labels, duplicateFields, onChange }) =>
         <LegendItem
           key={legendItem.label}
           legend={legendItem}
+          isAnomalyView={isAnomalyView}
           duplicateFields={duplicateFields}
           onChange={onChange}
         />

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/LegendAnomaly.tsx

@@ -0,0 +1,82 @@
+import { FC, useMemo } from "preact/compat";
+import { ForecastType, SeriesItem } from "../../../../types";
+import { anomalyColors } from "../../../../utils/color";
+import "./style.scss";
+
+type Props = {
+  series: SeriesItem[];
+};
+
+const titles: Partial<Record<ForecastType, string>> = {
+  [ForecastType.yhat]: "yhat",
+  [ForecastType.yhatLower]: "yhat_upper - yhat_lower",
+  [ForecastType.yhatUpper]: "yhat_upper - yhat_lower",
+  [ForecastType.anomaly]: "anomalies",
+  [ForecastType.training]: "training data",
+  [ForecastType.actual]: "y"
+};
+
+const LegendAnomaly: FC<Props> = ({ series }) => {
+
+  const uniqSeriesStyles = useMemo(() => {
+    const uniqSeries = series.reduce((accumulator, currentSeries) => {
+      const hasForecast = Object.prototype.hasOwnProperty.call(currentSeries, "forecast");
+      const isNotUpper = currentSeries.forecast !== ForecastType.yhatUpper;
+      const isUniqForecast = !accumulator.find(s => s.forecast === currentSeries.forecast);
+      if (hasForecast && isUniqForecast && isNotUpper) {
+        accumulator.push(currentSeries);
+      }
+      return accumulator;
+    }, [] as SeriesItem[]);
+
+    const trainingSeries = {
+      ...uniqSeries[0],
+      forecast: ForecastType.training,
+      color: anomalyColors[ForecastType.training],
+    };
+    uniqSeries.splice(1, 0, trainingSeries);
+
+    return uniqSeries.map(s => ({
+      ...s,
+      color: typeof s.stroke === "string" ? s.stroke : anomalyColors[s.forecast || ForecastType.actual],
+    }));
+  }, [series]);
+
+  return <>
+    <div className="vm-legend-anomaly">
+      {/* TODO: remove .filter() after the correct training data has been added */}
+      {uniqSeriesStyles.filter(f => f.forecast !== ForecastType.training).map((s, i) => (
+        <div
+          key={`${i}_${s.forecast}`}
+          className="vm-legend-anomaly-item"
+        >
+          <svg>
+            {s.forecast === ForecastType.anomaly ? (
+              <circle
+                cx="15"
+                cy="7"
+                r="4"
+                fill={s.color}
+                stroke={s.color}
+                strokeWidth="1.4"
+              />
+            ) : (
+              <line
+                x1="0"
+                y1="7"
+                x2="30"
+                y2="7"
+                stroke={s.color}
+                strokeWidth={s.width || 1}
+                strokeDasharray={s.dash?.join(",")}
+              />
+            )}
+          </svg>
+          <div className="vm-legend-anomaly-item__title">{titles[s.forecast || ForecastType.actual]}</div>
+        </div>
+      ))}
+    </div>
+  </>;
+};
+
+export default LegendAnomaly;

app/vmui/packages/vmui/src/components/Chart/Line/LegendAnomaly/style.scss

@@ -0,0 +1,23 @@
+@use "src/styles/variables" as *;
+
+.vm-legend-anomaly {
+  position: relative;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  flex-wrap: wrap;
+  gap: calc($padding-large * 2);
+  cursor: default;
+
+  &-item {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    gap: $padding-small;
+
+    svg {
+      width: 30px;
+      height: 14px;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Chart/Line/LineChart/LineChart.tsx

@@ -13,6 +13,7 @@ import {
   getRangeY,
   getScales,
   handleDestroy,
+  setBand,
   setSelect
 } from "../../../../utils/uplot";
 import { MetricResult } from "../../../../api/types";
@@ -39,6 +40,7 @@ export interface LineChartProps {
   setPeriod: ({ from, to }: { from: Date, to: Date }) => void;
   layoutSize: ElementSize;
   height?: number;
+  isAnomalyView?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
 }
@@ -53,6 +55,7 @@ const LineChart: FC<LineChartProps> = ({
   setPeriod,
   layoutSize,
   height,
+  isAnomalyView,
   spanGaps = false,
   showAllPoints = false,
 }) => {
@@ -72,7 +75,7 @@ const LineChart: FC<LineChartProps> = ({
     seriesFocus,
     setCursor,
     resetTooltips
-  } = useLineTooltip({ u: uPlotInst, metrics, series, unit });
+  } = useLineTooltip({ u: uPlotInst, metrics, series, unit, isAnomalyView });
 
   const options: uPlotOptions = {
     ...getDefaultOptions({ width: layoutSize.width, height }),
@@ -108,6 +111,7 @@ const LineChart: FC<LineChartProps> = ({
     if (!uPlotInst) return;
     delSeries(uPlotInst);
     addSeries(uPlotInst, series, spanGaps, showAllPoints);
+    setBand(uPlotInst, series);
     uPlotInst.redraw();
   }, [series, spanGaps, showAllPoints]);
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -29,7 +29,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   const { seriesLimits } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
-  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
+  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE")
   const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
 
   const [limits, setLimits] = useState(seriesLimits);
@@ -58,7 +58,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   }, [limits]);
 
   useEffect(() => {
-    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
+    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`)
   }, [legendCollapse]);
 
   useImperativeHandle(ref, () => ({ handleApply }), [handleApply]);

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/ServerConfigurator/ServerConfigurator.tsx

@@ -9,6 +9,7 @@ import { getFromStorage, removeFromStorage, saveToStorage } from "../../../../ut
 import useBoolean from "../../../../hooks/useBoolean";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
+import { getTenantIdFromUrl } from "../../../../utils/tenants";
 
 interface ServerConfiguratorProps {
   onClose: () => void;
@@ -38,6 +39,10 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   };
 
   const handleApply = useCallback(() => {
+    const tenantIdFromUrl = getTenantIdFromUrl(serverUrl);
+    if (tenantIdFromUrl !== "") {
+      dispatch({ type: "SET_TENANT_ID", payload: tenantIdFromUrl });
+    }
     dispatch({ type: "SET_SERVER", payload: serverUrl });
     onClose();
   }, [serverUrl]);
@@ -55,6 +60,12 @@ const ServerConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     }
   }, [enabledStorage]);
 
+  useEffect(() => {
+    if (enabledStorage) {
+      saveToStorage("SERVER_URL", serverUrl);
+    }
+  }, [serverUrl]);
+
   useEffect(() => {
     // the tenant selector can change the serverUrl
     if (stateServerUrl === serverUrl) return;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration.tsx

@@ -1,4 +1,4 @@
-import { FC, useState, useRef, useMemo } from "preact/compat";
+import { FC, useState, useRef, useEffect, useMemo } from "preact/compat";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
 import { ArrowDownIcon, StorageIcon } from "../../../Main/Icons";
@@ -10,14 +10,14 @@ import { getAppModeEnable } from "../../../../utils/app-mode";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import TextField from "../../../Main/TextField/TextField";
-import { replaceTenantId } from "../../../../utils/tenants";
+import { getTenantIdFromUrl, replaceTenantId } from "../../../../utils/tenants";
 import useBoolean from "../../../../hooks/useBoolean";
 
 const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   const appModeEnable = getAppModeEnable();
   const { isMobile } = useDeviceDetect();
 
-  const { tenantId, serverUrl } = useAppState();
+  const { tenantId: tenantIdState, serverUrl } = useAppState();
   const dispatch = useAppDispatch();
   const timeDispatch = useTimeDispatch();
 
@@ -48,8 +48,10 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   }, [accountIds]);
 
   const createHandlerChange = (value: string) => () => {
+    const tenant = value;
+    dispatch({ type: "SET_TENANT_ID", payload: tenant });
     if (serverUrl) {
-      const updateServerUrl = replaceTenantId(serverUrl, value);
+      const updateServerUrl = replaceTenantId(serverUrl, tenant);
       if (updateServerUrl === serverUrl) return;
       dispatch({ type: "SET_SERVER", payload: updateServerUrl });
       timeDispatch({ type: "RUN_QUERY" });
@@ -57,6 +59,16 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
     handleCloseOptions();
   };
 
+  useEffect(() => {
+    const id = getTenantIdFromUrl(serverUrl);
+
+    if (tenantIdState && tenantIdState !== id) {
+      createHandlerChange(tenantIdState)();
+    } else {
+      createHandlerChange(id)();
+    }
+  }, [serverUrl]);
+
   if (!showTenantSelector) return null;
 
   return (
@@ -71,7 +83,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               <span className="vm-mobile-option__icon"><StorageIcon/></span>
               <div className="vm-mobile-option-text">
                 <span className="vm-mobile-option-text__label">Tenant ID</span>
-                <span className="vm-mobile-option-text__value">{tenantId}</span>
+                <span className="vm-mobile-option-text__value">{tenantIdState}</span>
               </div>
               <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
             </div>
@@ -94,7 +106,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               )}
               onClick={toggleOpenOptions}
             >
-              {tenantId}
+              {tenantIdState}
             </Button>
           )}
         </div>
@@ -126,7 +138,7 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               className={classNames({
                 "vm-list-item": true,
                 "vm-list-item_mobile": isMobile,
-                "vm-list-item_active": id === tenantId
+                "vm-list-item_active": id === tenantIdState
               })}
               key={id}
               onClick={createHandlerChange(id)}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/hooks/useFetchAccountIds.ts

@@ -3,18 +3,19 @@ import { useEffect, useMemo, useState } from "preact/compat";
 import { ErrorTypes } from "../../../../../types";
 import { getAccountIds } from "../../../../../api/accountId";
 import { getAppModeEnable, getAppModeParams } from "../../../../../utils/app-mode";
+import { getTenantIdFromUrl } from "../../../../../utils/tenants";
 
 export const useFetchAccountIds = () => {
   const { useTenantID } = getAppModeParams();
   const appModeEnable = getAppModeEnable();
-  const { tenantId, serverUrl } = useAppState();
+  const { serverUrl } = useAppState();
 
   const [isLoading, setIsLoading] = useState(false);
   const [error, setError] = useState<ErrorTypes | string>();
   const [accountIds, setAccountIds] = useState<string[]>([]);
 
   const fetchUrl = useMemo(() => getAccountIds(serverUrl), [serverUrl]);
-  const isServerUrlWithTenant = useMemo(() => !!tenantId, [tenantId]);
+  const isServerUrlWithTenant = useMemo(() => !!getTenantIdFromUrl(serverUrl), [serverUrl]);
   const preventFetch = appModeEnable ? !useTenantID : !isServerUrlWithTenant;
 
   useEffect(() => {

app/vmui/packages/vmui/src/components/ExploreAlerts/helpers.ts

@@ -17,4 +17,4 @@ export const formatDuration = (raw: number) => {
 export const formatEventTime = (raw: string) => {
   const t = dayjs(raw);
   return t.year() <= 1 ? "Never" : t.format("DD MMM YYYY HH:mm:ss");
-};
+}

app/vmui/packages/vmui/src/components/ExploreAnomaly/AnomalyConfig.tsx

@@ -0,0 +1,132 @@
+import { FC, useState } from "preact/compat";
+import Button from "../Main/Button/Button";
+import TextField from "../Main/TextField/TextField";
+import Modal from "../Main/Modal/Modal";
+import Spinner from "../Main/Spinner/Spinner";
+import { DownloadIcon, ErrorIcon } from "../Main/Icons";
+import useBoolean from "../../hooks/useBoolean";
+import useDeviceDetect from "../../hooks/useDeviceDetect";
+import { useAppState } from "../../state/common/StateContext";
+import classNames from "classnames";
+import "./style.scss";
+import { useQueryState } from "../../state/query/QueryStateContext";
+import { useTimeState } from "../../state/time/TimeStateContext";
+import { getStepFromDuration } from "../../utils/time";
+
+const AnomalyConfig: FC = () => {
+  const { serverUrl } = useAppState();
+  const { isMobile } = useDeviceDetect();
+
+  const {
+    value: isModalOpen,
+    setTrue: setOpenModal,
+    setFalse: setCloseModal,
+  } = useBoolean(false);
+
+  const { query } = useQueryState();
+  const { period } = useTimeState();
+  const [isLoading, setIsLoading] = useState(false);
+  const [textConfig, setTextConfig] = useState<string>("");
+  const [downloadUrl, setDownloadUrl] = useState<string>("");
+  const [error, setError] = useState<string>("");
+
+  const fetchConfig = async () => {
+    setIsLoading(true);
+    try {
+      const queryParam = encodeURIComponent(query[0] || "");
+      const stepParam = encodeURIComponent(period.step || getStepFromDuration(period.end - period.start, false));
+
+      const url = `${serverUrl}/api/vmanomaly/config.yaml?query=${queryParam}&step=${stepParam}`;
+      const response = await fetch(url);
+      const contentType = response.headers.get("Content-Type");
+      if (!response.ok) {
+        const bodyText = await response.text();
+        setError(` ${response.status} ${response.statusText}: ${bodyText}`);
+      } else if (contentType == "application/yaml") {
+        const blob = await response.blob();
+        const yamlAsString = await blob.text();
+        setTextConfig(yamlAsString);
+        setDownloadUrl(URL.createObjectURL(blob));
+      } else {
+        setError("Response Content-Type is not YAML, does `Server URL` point to VMAnomaly server?");
+      }
+    } catch (error) {
+      console.error(error);
+      setError(String(error));
+    }
+    setIsLoading(false);
+  };
+
+  const handleOpenModal = () => {
+    setOpenModal();
+    setError("");
+    URL.revokeObjectURL(downloadUrl);
+    setTextConfig("");
+    setDownloadUrl("");
+    return fetchConfig();
+  };
+
+  return (
+    <>
+      <Button
+        color="secondary"
+        variant="outlined"
+        onClick={handleOpenModal}
+      >
+        Open Config
+      </Button>
+      {isModalOpen && (
+        <Modal
+          title="Download config"
+          onClose={setCloseModal}
+        >
+          <div
+            className={classNames({
+              "vm-anomaly-config": true,
+              "vm-anomaly-config_mobile": isMobile,
+            })}
+          >
+            {isLoading && (
+              <Spinner
+                containerStyles={{ position: "relative" }}
+                message={"Loading config..."}
+              />
+            )}
+            {!isLoading && error && (
+              <div className="vm-anomaly-config-error">
+                <div className="vm-anomaly-config-error__icon"><ErrorIcon/></div>
+                <h3 className="vm-anomaly-config-error__title">Cannot download config</h3>
+                <p className="vm-anomaly-config-error__text">{error}</p>
+              </div>
+            )}
+            {!isLoading && textConfig && (
+              <TextField
+                value={textConfig}
+                label={"config.yaml"}
+                type="textarea"
+                disabled={true}
+              />
+            )}
+            <div className="vm-anomaly-config-footer">
+              {downloadUrl && (
+                <a
+                  href={downloadUrl}
+                  download={"config.yaml"}
+                >
+                  <Button
+                    variant="contained"
+                    startIcon={<DownloadIcon/>}
+                  >
+                    download
+                  </Button>
+                </a>
+              )}
+            </div>
+          </div>
+        </Modal>
+      )}
+    </>
+  );
+};
+
+export default AnomalyConfig;

app/vmui/packages/vmui/src/components/ExploreAnomaly/style.scss

@@ -0,0 +1,61 @@
+@use "src/styles/variables" as *;
+
+.vm-anomaly-config {
+  display: grid;
+  grid-template-rows: calc(($vh * 70) - 78px - ($padding-medium*3)) auto;
+  gap: $padding-global;
+  min-width: 400px;
+  max-width: 80vw;
+  min-height: 300px;
+
+  &_mobile {
+    width: 100%;
+    max-width: none;
+    min-height: 100%;
+    grid-template-rows: calc(($vh * 100) - 78px - ($padding-global*3)) auto;
+  }
+
+  textarea {
+    overflow: auto;
+    width: 100%;
+    height: 100%;
+    max-height: 900px;
+  }
+
+  &-error {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    width: 100%;
+    gap: $padding-small;
+    text-align: center;
+
+    &__icon {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      width: 30px;
+      height: 30px;
+      margin-bottom: $padding-small;
+      color: $color-error;
+    }
+
+    &__title {
+      font-size: $font-size-medium;
+      font-weight: bold;
+    }
+
+    &__text {
+      max-width: 700px;
+      line-height: 1.3;
+    }
+  }
+
+  &-footer {
+    display: flex;
+    align-items: center;
+    justify-content: flex-end;
+    gap: $padding-small;
+  }
+}

app/vmui/packages/vmui/src/components/Main/Button/style.scss

@@ -1,5 +1,4 @@
 @use "src/styles/variables" as *;
-@use 'sass:meta';
 
 $button-radius: 6px;
 
@@ -43,8 +42,6 @@ $button-radius: 6px;
 
     svg {
       width: 14px;
-      min-width: 14px;
-      max-width: 14px;
     }
   }
 
@@ -54,8 +51,6 @@ $button-radius: 6px;
 
     svg {
       width: 16px;
-      min-width: 16px;
-      max-width: 16px;
     }
   }
 
@@ -65,8 +60,6 @@ $button-radius: 6px;
 
     svg {
       width: 18px;
-      min-width: 18px;
-      max-width: 18px;
       line-height: 16px;
     }
   }
@@ -135,14 +128,8 @@ $button-radius: 6px;
   );
 
   @each $name, $color in $button-colors {
-    @if $name == white {
-      @include contained-button($name, $color, $color-black);
-      @include outlined-button($name, $color, $color-white);
-      @include text-button($name, $color-white);
-    } @else {
-      @include contained-button($name, $color, $color-white);
-      @include outlined-button($name, $color, $color);
-      @include text-button($name, $color);
-    }
+    @include contained-button($name, $color, if($name == white, $color-black, $color-white));
+    @include outlined-button($name, $color, if($name == white, $color-white, $color));
+    @include text-button($name, if($name == white, $color-white, $color));
   }
 }

app/vmui/packages/vmui/src/components/QueryHistory/utils.ts

@@ -1,8 +1,4 @@
-import {
-  getFromStorage,
-  saveToStorage,
-  StorageKeys,
-} from "../../utils/storage";
+import { getFromStorage, removeFromStorage, saveToStorage, StorageKeys } from "../../utils/storage";
 import { QueryHistoryType } from "../../state/query/reducer";
 import { MAX_QUERIES_HISTORY, MAX_QUERY_FIELDS } from "../../constants/graph";
 
@@ -77,3 +73,17 @@ export const getUpdatedHistory = (query: string, queryHistory?: QueryHistoryType
     values: newValues
   };
 };
+
+const migrateMetricsQueryHistoryToHistoryByKey = () => {
+  const migrateHistory = (type: HistoryType) => {
+    const queryList = getFromStorage(type) as string;
+    if (queryList) {
+      const queryHistory: string[][] = JSON.parse(queryList);
+      saveHistoryToStorage("METRICS_QUERY_HISTORY", type, queryHistory);
+      removeFromStorage([type]);
+    }
+  };
+  migrateHistory("QUERY_HISTORY");
+  migrateHistory("QUERY_FAVORITES");
+};
+migrateMetricsQueryHistoryToHistoryByKey();

app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx

@@ -12,7 +12,7 @@ import {
   getMinMaxBuffer,
   getTimeSeries,
 } from "../../../utils/uplot";
-import { TimeParams, LegendItemType } from "../../../types";
+import { TimeParams, SeriesItem, LegendItemType } from "../../../types";
 import { AxisRange, YaxisState } from "../../../state/graph/reducer";
 import { getMathStats } from "../../../utils/math";
 import classNames from "classnames";
@@ -23,6 +23,8 @@ import { promValueToNumber } from "../../../utils/metric";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useElementSize from "../../../hooks/useElementSize";
 import { ChartTooltipProps } from "../../Chart/ChartTooltip/ChartTooltip";
+import LegendAnomaly from "../../Chart/Line/LegendAnomaly/LegendAnomaly";
+import { groupByMultipleKeys } from "../../../utils/array";
 import { useGraphDispatch } from "../../../state/graph/GraphStateContext";
 import { sameTs } from "../../../utils/time";
 import { useLocation } from "react-router-dom";
@@ -42,6 +44,7 @@ export interface GraphViewProps {
   fullWidth?: boolean;
   height?: number;
   isHistogram?: boolean;
+  isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   spanGaps?: boolean;
   showAllPoints?: boolean;
@@ -61,6 +64,7 @@ const GraphView: FC<GraphViewProps> = ({
   fullWidth = true,
   height,
   isHistogram,
+  isAnomalyView,
   isPredefinedPanel,
   spanGaps,
   showAllPoints
@@ -85,8 +89,8 @@ const GraphView: FC<GraphViewProps> = ({
   const [legendValue, setLegendValue] = useState<ChartTooltipProps | null>(null);
 
   const getSeriesItem = useMemo(() => {
-    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isRawQuery);
-  }, [data, hideSeries, alias, showAllPoints, isRawQuery]);
+    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isAnomalyView, isRawQuery);
+  }, [data, hideSeries, alias, showAllPoints, isAnomalyView, isRawQuery]);
 
   const setLimitsYaxis = (minVal: number, maxVal: number) => {
     let min = Number.isFinite(minVal) ? minVal : 0;
@@ -98,7 +102,7 @@ const GraphView: FC<GraphViewProps> = ({
   };
 
   const onChangeLegend = (legend: LegendItemType, metaKey: boolean) => {
-    setHideSeries(getHideSeries({ hideSeries, legend, metaKey, series }));
+    setHideSeries(getHideSeries({ hideSeries, legend, metaKey, series, isAnomalyView }));
   };
 
   const prepareHistogramData = (data: (number | null)[][]) => {
@@ -123,6 +127,20 @@ const GraphView: FC<GraphViewProps> = ({
     return [null, [xs, ys, counts]];
   };
 
+  const prepareAnomalyLegend = (legend: LegendItemType[]): LegendItemType[] => {
+    if (!isAnomalyView) return legend;
+
+    // For vmanomaly: Only select the first series per group (due to API specs) and clear __name__ in freeFormFields.
+    const grouped = groupByMultipleKeys(legend, ["group", "label"]);
+    return grouped.map((group) => {
+      const firstEl = group.values[0];
+      return {
+        ...firstEl,
+        freeFormFields: { ...firstEl.freeFormFields, __name__: "" }
+      };
+    });
+  };
+
   useEffect(() => {
     const dLen = data.length;
 
@@ -137,7 +155,7 @@ const GraphView: FC<GraphViewProps> = ({
 
     for (let i = 0; i < dLen; i++) {
       const d = data[i];
-      const seriesItem = getSeriesItem(d);
+      const seriesItem = getSeriesItem(d, i);
       tempSeries[i + 1] = seriesItem;
       tempLegend[i] = getLegendItem(seriesItem, d.group);
 
@@ -188,7 +206,7 @@ const GraphView: FC<GraphViewProps> = ({
       const avg = Math.abs(Number(avgRaw));
       const range = getMinMaxBuffer(min, max);
       const rangeStep = Math.abs(range[1] - range[0]);
-      const needStabilize = (avg > rangeStep * 1e10);
+      const needStabilize = (avg > rangeStep * 1e10) && !isAnomalyView;
 
       return needStabilize ? results.fill(avg) : results;
     });
@@ -196,11 +214,13 @@ const GraphView: FC<GraphViewProps> = ({
     timeDataSeries.unshift(timeSeries);
 
     const result = isHistogram ? prepareHistogramData(timeDataSeries) : timeDataSeries;
+    const legend = prepareAnomalyLegend(tempLegend);
 
     setLimitsYaxis(minVal, maxVal);
     setDataChart(result as uPlotData);
     setSeries(tempSeries);
-    setLegend(tempLegend);
+    setLegend(legend);
+    isAnomalyView && setHideSeries(legend.map(s => s.label || "").slice(1));
   }, [data, timezone, isHistogram, currentStep, isRawQuery]);
 
   useEffect(() => {
@@ -212,13 +232,13 @@ const GraphView: FC<GraphViewProps> = ({
 
     for (let i = 0; i < dLen; i++) {
       const d = data[i];
-      const seriesItem = getSeriesItem(d);
+      const seriesItem = getSeriesItem(d, i);
       tempSeries[i + 1] = seriesItem;
       tempLegend[i] = getLegendItem(seriesItem, d.group);
     }
 
     setSeries(tempSeries);
-    setLegend(tempLegend);
+    setLegend(prepareAnomalyLegend(tempLegend));
   }, [hideSeries]);
 
   const hasTimeData = dataChart[0]?.length > 0;
@@ -261,6 +281,7 @@ const GraphView: FC<GraphViewProps> = ({
           setPeriod={setPeriod}
           layoutSize={containerSize}
           height={height}
+          isAnomalyView={isAnomalyView}
           spanGaps={spanGaps}
           showAllPoints={isRawQuery ? true : showAllPoints}
         />
@@ -277,10 +298,12 @@ const GraphView: FC<GraphViewProps> = ({
           onChangeLegend={setLegendValue}
         />
       )}
+      {isAnomalyView && showLegend && (<LegendAnomaly series={series as SeriesItem[]}/>)}
       {!isHistogram && showLegend && (
         <Legend
           labels={legend}
           query={query}
+          isAnomalyView={isAnomalyView}
           onChange={onChangeLegend}
           isPredefinedPanel={isPredefinedPanel}
         />

app/vmui/packages/vmui/src/components/WebStorageCheck/WebStorageCheck.tsx

@@ -1,69 +0,0 @@
-import { useEffect } from "react";
-import { StorageErrorCode } from "./types";
-import { useSnack } from "../../contexts/Snackbar";
-import { storageErrorInfo } from "./storageErrors";
-import "./style.scss";
-
-const classifyStorageException = (e: unknown): StorageErrorCode => {
-  if (!(e instanceof DOMException)) return StorageErrorCode.UNKNOWN;
-
-  switch (e.name) {
-    case "QuotaExceededError":
-      return StorageErrorCode.QUOTA_EXCEEDED;
-    case "SecurityError":
-      return StorageErrorCode.SECURITY_ERROR;
-    default:
-      return StorageErrorCode.UNKNOWN;
-  }
-};
-
-const getStorageError = (storage: Storage | null | undefined): StorageErrorCode | null => {
-  if (!storage) {
-    return StorageErrorCode.NO_STORAGE;
-  }
-
-  try {
-    const key = "__vmui_test__";
-    storage.setItem(key, "1");
-    storage.removeItem(key);
-    return null;
-  } catch (e) {
-    return classifyStorageException(e);
-  }
-};
-
-const WebStorageCheck = () => {
-  const { showInfoMessage } = useSnack();
-
-  useEffect(() => {
-    const error = getStorageError(window.localStorage);
-
-    if (error) {
-      const { title, description, fix } = storageErrorInfo[error];
-
-      const text = (
-        <div className="vm-storage-check">
-          <h3>{title}</h3>
-          <p>{description}</p>
-
-          {!!fix?.length && (
-            <div className="vm-storage-check__fix">
-              <div>Try this:</div>
-              <ul>
-                {fix.map((step, i) => (
-                  <li key={`${i}-${step}`}>{step}</li>
-                ))}
-              </ul>
-            </div>
-          )}
-        </div>
-      );
-
-      showInfoMessage({ text: text, type: "error", timeout: 600000 });
-    }
-  }, []);
-
-  return null;
-};
-
-export default WebStorageCheck;

app/vmui/packages/vmui/src/components/WebStorageCheck/storageErrors.ts

@@ -1,47 +0,0 @@
-import { StorageError, StorageErrorCode } from "./types";
-
-export const storageErrorInfo: Record<StorageErrorCode, StorageError> = {
-  [StorageErrorCode.NO_STORAGE]: {
-    title: "Storage unavailable",
-    description:
-      "Browser storage is not available for this website.",
-    fix: [
-      "Disable Private/Incognito mode and reload the page.",
-      "Disable privacy or ad-blocking extensions for this site and reload.",
-      "Open the site in another browser.",
-    ],
-  },
-
-  [StorageErrorCode.SECURITY_ERROR]: {
-    title: "Storage access blocked",
-    description:
-      "Browser settings or an extension are blocking access to browser storage.",
-    fix: [
-      "Disable Private/Incognito mode and reload the page.",
-      "Disable privacy or ad-blocking extensions for this site and reload.",
-      "Open the site in a regular browser tab (not embedded).",
-    ],
-  },
-
-  [StorageErrorCode.QUOTA_EXCEEDED]: {
-    title: "Storage quota exceeded",
-    description:
-      "The storage limit for this website has been reached.",
-    fix: [
-      "Clear this website’s stored data and reload the page.",
-      "Close other tabs for this website and try again.",
-      "Use another browser or browser profile.",
-    ],
-  },
-
-  [StorageErrorCode.UNKNOWN]: {
-    title: "Storage error",
-    description:
-      "An unexpected error occurred while accessing browser storage.",
-    fix: [
-      "Reload the page.",
-      "Update the browser and try again.",
-      "Disable browser extensions and reload.",
-    ],
-  },
-};

app/vmui/packages/vmui/src/components/WebStorageCheck/style.scss

@@ -1,11 +0,0 @@
-@use "src/styles/variables" as *;
-
-.vm-storage-check {
-  h3 {
-    font-weight: bold;
-  }
-
-  p {
-    margin-bottom: $padding-global
-  }
-}

app/vmui/packages/vmui/src/components/WebStorageCheck/types.ts

@@ -1,13 +0,0 @@
-
-export enum StorageErrorCode {
-  NO_STORAGE = "NO_STORAGE",
-  SECURITY_ERROR = "SECURITY_ERROR",
-  QUOTA_EXCEEDED = "QUOTA_EXCEEDED",
-  UNKNOWN = "UNKNOWN",
-}
-
-export type StorageError = {
-  title: string;
-  description: string;
-  fix: string[]
-}

app/vmui/packages/vmui/src/constants/appType.ts

@@ -0,0 +1,8 @@
+export enum AppType {
+  victoriametrics = "victoriametrics",
+  vmanomaly = "vmanomaly",
+}
+
+export const APP_TYPE = import.meta.env.VITE_APP_TYPE;
+export const APP_TYPE_VM = APP_TYPE === AppType.victoriametrics;
+export const APP_TYPE_ANOMALY = APP_TYPE === AppType.vmanomaly;

app/vmui/packages/vmui/src/hooks/uplot/useLineTooltip.ts

@@ -13,9 +13,10 @@ interface LineTooltipHook {
   metrics: MetricResult[];
   series: uPlotSeries[];
   unit?: string;
+  isAnomalyView?: boolean;
 }
 
-const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
+const useLineTooltip = ({ u, metrics, series, unit, isAnomalyView }: LineTooltipHook) => {
   const [showTooltip, setShowTooltip] = useState(false);
   const [tooltipIdx, setTooltipIdx] = useState({ seriesIdx: -1, dataIdx: -1 });
   const [stickyTooltips, setStickyToolTips] = useState<ChartTooltipProps[]>([]);
@@ -78,7 +79,7 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
       point,
       u: u,
       id: `${seriesIdx}_${dataIdx}`,
-      title: groups.size > 1 ? `Query ${group}` : "",
+      title: groups.size > 1 && !isAnomalyView ? `Query ${group}` : "",
       dates: [date ? dayjs(date * 1000).tz().format(DATE_FULL_TIMEZONE_FORMAT) : "-"],
       value: formatPrettyNumber(value, min, max),
       info: getMetricName(metricItem, seriesItem),
@@ -86,7 +87,7 @@ const useLineTooltip = ({ u, metrics, series, unit }: LineTooltipHook) => {
       marker: `${seriesItem?.stroke}`,
       duplicateCount,
     };
-  }, [u, tooltipIdx, metrics, series, unit]);
+  }, [u, tooltipIdx, metrics, series, unit, isAnomalyView]);
 
   const handleClick = useCallback(() => {
     if (!showTooltip) return;

app/vmui/packages/vmui/src/hooks/useFetchAppConfig.ts

@@ -1,6 +1,7 @@
 import { useAppDispatch, useAppState } from "../state/common/StateContext";
 import { useEffect, useState } from "preact/compat";
 import { ErrorTypes } from "../types";
+import { APP_TYPE_VM } from "../constants/appType";
 
 const useFetchAppConfig = () => {
   const { serverUrl } = useAppState();
@@ -11,6 +12,7 @@ const useFetchAppConfig = () => {
 
   useEffect(() => {
     const fetchAppConfig = async () => {
+      if (!APP_TYPE_VM) return;
       setError("");
       setIsLoading(true);
 

app/vmui/packages/vmui/src/hooks/useFetchDefaultTimezone.ts

@@ -5,6 +5,7 @@ import { useTimeDispatch } from "../state/time/TimeStateContext";
 import { getFromStorage } from "../utils/storage";
 import dayjs from "dayjs";
 import { getBrowserTimezone } from "../utils/time";
+import { APP_TYPE_VM } from "../constants/appType";
 
 const disabledDefaultTimezone = Boolean(getFromStorage("DISABLED_DEFAULT_TIMEZONE"));
 
@@ -28,7 +29,7 @@ const useFetchDefaultTimezone = () => {
   };
 
   const fetchDefaultTimezone = async () => {
-    if (!serverUrl) return;
+    if (!serverUrl || !APP_TYPE_VM) return;
     setError("");
     setIsLoading(true);
 

app/vmui/packages/vmui/src/hooks/useFetchQuery.ts

@@ -13,6 +13,7 @@ import { isHistogramData } from "../utils/metric";
 import { useGraphState } from "../state/graph/GraphStateContext";
 import { getStepFromDuration } from "../utils/time";
 import { getQueryStringValue } from "../utils/query-string";
+import { APP_TYPE_ANOMALY } from "../constants/appType";
 
 interface FetchQueryParams {
   predefinedQuery?: string[]
@@ -134,7 +135,7 @@ export const useFetchQuery = ({
           }
 
           const preventChangeType = !!getQueryStringValue("display_mode", null);
-          isHistogramResult = isDisplayChart && !preventChangeType && isHistogramData(resp.data.result);
+          isHistogramResult = !APP_TYPE_ANOMALY && isDisplayChart && !preventChangeType && isHistogramData(resp.data.result);
           seriesLimit = isHistogramResult ? Infinity : defaultLimit;
           const freeTempSize = Math.max(0, seriesLimit - tempData.length);
           resp.data.result.slice(0, freeTempSize).forEach((d: MetricBase) => {

app/vmui/packages/vmui/src/index.tsx

@@ -3,9 +3,20 @@ import "./constants/dayjsPlugins";
 import App from "./App";
 import reportWebVitals from "./reportWebVitals";
 import "./styles/style.scss";
+import { APP_TYPE, AppType } from "./constants/appType";
+import AppAnomaly from "./AppAnomaly";
+
+const getAppComponent = () => {
+  switch (APP_TYPE) {
+    case AppType.vmanomaly:
+      return <AppAnomaly/>;
+    default:
+      return <App/>;
+  }
+};
 
 const root = document.getElementById("root");
-if (root) render(<App/>, root);
+if (root) render(getAppComponent(), root);
 
 
 // If you want to start measuring performance in your app, pass a function

app/vmui/packages/vmui/src/layouts/AnomalyLayout/AnomalyLayout.tsx

@@ -0,0 +1,50 @@
+import Header from "../Header/Header";
+import { FC, useEffect } from "preact/compat";
+import { Outlet, useSearchParams } from "react-router-dom";
+import qs from "qs";
+import "../MainLayout/style.scss";
+import { getAppModeEnable } from "../../utils/app-mode";
+import classNames from "classnames";
+import Footer from "../Footer/Footer";
+import useFetchDefaultTimezone from "../../hooks/useFetchDefaultTimezone";
+import useDeviceDetect from "../../hooks/useDeviceDetect";
+import ControlsAnomalyLayout from "./ControlsAnomalyLayout";
+
+const AnomalyLayout: FC = () => {
+  const appModeEnable = getAppModeEnable();
+  const { isMobile } = useDeviceDetect();
+  const [searchParams, setSearchParams] = useSearchParams();
+
+  useFetchDefaultTimezone();
+
+  // for support old links with search params
+  const redirectSearchToHashParams = () => {
+    const { search, href } = window.location;
+    if (search) {
+      const query = qs.parse(search, { ignoreQueryPrefix: true });
+      Object.entries(query).forEach(([key, value]) => searchParams.set(key, value as string));
+      setSearchParams(searchParams);
+      window.location.search = "";
+    }
+    const newHref = href.replace(/\/\?#\//, "/#/");
+    if (newHref !== href) window.location.replace(newHref);
+  };
+
+  useEffect(redirectSearchToHashParams, []);
+
+  return <section className="vm-container">
+    <Header controlsComponent={ControlsAnomalyLayout}/>
+    <div
+      className={classNames({
+        "vm-container-body": true,
+        "vm-container-body_mobile": isMobile,
+        "vm-container-body_app": appModeEnable
+      })}
+    >
+      <Outlet/>
+    </div>
+    {!appModeEnable && <Footer/>}
+  </section>;
+};
+
+export default AnomalyLayout;