lib/storage: retruce number of shards in metricIDCache

This should reduce cpu utilization while still removing the storage connection saturation.

Follow-up for 6bc809813b (#10346)

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/vmui.yml

@@ -34,33 +34,39 @@ jobs:
       - name: Code checkout
         uses: actions/checkout@v6
 
-      - name: Setup Node
-        uses: actions/setup-node@v6
+      - name: Cache node_modules
+        id: cache
+        uses: actions/cache@v5
         with:
-          node-version: '24.x'
+          path: app/vmui/packages/vmui/node_modules
+          key: vmui-deps-${{ runner.os }}-${{ hashFiles('app/vmui/packages/vmui/package-lock.json', 'app/vmui/Dockerfile-build') }}
+          restore-keys: |
+            vmui-deps-${{ runner.os }}-
 
-      - name: Cache node-modules
-        uses: actions/cache@v4
-        with:
-          path: |
-            app/vmui/packages/vmui/node_modules
-          key: vmui-artifacts-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: vmui-artifacts-${{ runner.os }}-
+      - name: Install dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make vmui-install
 
       - name: Run lint
         id: lint
         run: make vmui-lint
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run tests
         id: test
         run: make vmui-test
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Run typecheck
         id: typecheck
         run: make vmui-typecheck
         continue-on-error: true
+        env:
+          VMUI_SKIP_INSTALL: true
 
       - name: Annotate Code Linting Results
         uses: ataylorme/eslint-annotate-action@v3

LICENSE

@@ -175,7 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2019-2025 VictoriaMetrics, Inc.
+   Copyright 2019-2026 VictoriaMetrics, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

app/vmagent/main.go

@@ -245,6 +245,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>vmagent</h2>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See docs at <a href='https://docs.victoriametrics.com/victoriametrics/vmagent/'>https://docs.victoriametrics.com/victoriametrics/vmagent/</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		httpserver.WriteAPIHelp(w, [][2]string{

app/vmagent/remotewrite/client.go

@@ -202,14 +202,10 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.retriesCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_retries_count_total{url=%q}`, c.sanitizedURL))
 	c.sendDuration = metrics.GetOrCreateFloatCounter(fmt.Sprintf(`vmagent_remotewrite_send_duration_seconds_total{url=%q}`, c.sanitizedURL))
 	metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_queues{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(*queues)
+		return float64(concurrency)
 	})
-	for i := 0; i < concurrency; i++ {
-		c.wg.Add(1)
-		go func() {
-			defer c.wg.Done()
-			c.runWorker()
-		}()
+	for range concurrency {
+		c.wg.Go(c.runWorker)
 	}
 	logger.Infof("initialized client for -remoteWrite.url=%q", c.sanitizedURL)
 }

app/vmagent/remotewrite/pendingseries.go

@@ -48,11 +48,7 @@ func newPendingSeries(fq *persistentqueue.FastQueue, isVMRemoteWrite *atomic.Boo
 	ps.wr.significantFigures = significantFigures
 	ps.wr.roundDigits = roundDigits
 	ps.stopCh = make(chan struct{})
-	ps.periodicFlusherWG.Add(1)
-	go func() {
-		defer ps.periodicFlusherWG.Done()
-		ps.periodicFlusher()
-	}()
+	ps.periodicFlusherWG.Go(ps.periodicFlusher)
 	return &ps
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -59,7 +59,7 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flag.Int("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
 		"isn't enough for sending high volume of collected data to remote storage. "+
 		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -176,13 +176,6 @@ func Init() {
 		})
 	}
 
-	if *queues > maxQueues {
-		*queues = maxQueues
-	}
-	if *queues <= 0 {
-		*queues = 1
-	}
-
 	if len(*shardByURLLabels) > 0 && len(*shardByURLIgnoreLabels) > 0 {
 		logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
 			"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
@@ -215,9 +208,7 @@ func Init() {
 	dropDanglingQueues()
 
 	// Start config reloader.
-	configReloaderWG.Add(1)
-	go func() {
-		defer configReloaderWG.Done()
+	configReloaderWG.Go(func() {
 		for {
 			select {
 			case <-configReloaderStopCh:
@@ -227,7 +218,7 @@ func Init() {
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
 		}
-	}()
+	})
 }
 
 func dropDanglingQueues() {
@@ -267,17 +258,6 @@ func initRemoteWriteCtxs(urls []string) {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")
 	}
-
-	maxInmemoryBlocks := memory.Allowed() / len(urls) / *maxRowsPerBlock / 100
-	if maxInmemoryBlocks / *queues > 100 {
-		// There is no much sense in keeping higher number of blocks in memory,
-		// since this means that the producer outperforms consumer and the queue
-		// will continue growing. It is better storing the queue to file.
-		maxInmemoryBlocks = 100 * *queues
-	}
-	if maxInmemoryBlocks < 2 {
-		maxInmemoryBlocks = 2
-	}
 	rwctxs := make([]*remoteWriteCtx, len(urls))
 	rwctxIdx := make([]int, len(urls))
 	if retryMaxTime.String() != "" {
@@ -292,7 +272,7 @@ func initRemoteWriteCtxs(urls []string) {
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, sanitizedURL)
 		rwctxIdx[i] = i
 	}
 
@@ -558,11 +538,9 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 	// Push metadata to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.tryPushMetadataInternal(mms) {
 				rwctx.pushFailures.Inc()
 				if forceDropSamplesOnFailure {
@@ -571,7 +549,7 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 				}
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -603,15 +581,13 @@ func tryPushTimeSeriesToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prom
 	// Push tssBlock to remote storage systems in parallel to reduce
 	// the time needed for sending the data to multiple remote storage systems.
 	var wg sync.WaitGroup
-	wg.Add(len(rwctxs))
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
-		go func(rwctx *remoteWriteCtx) {
-			defer wg.Done()
+		wg.Go(func() {
 			if !rwctx.TryPushTimeSeries(tssBlock, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -633,13 +609,11 @@ func tryShardingTimeSeriesAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock
 		if len(shard) == 0 {
 			continue
 		}
-		wg.Add(1)
-		go func(rwctx *remoteWriteCtx, tss []prompb.TimeSeries) {
-			defer wg.Done()
-			if !rwctx.TryPushTimeSeries(tss, forceDropSamplesOnFailure) {
+		wg.Go(func() {
+			if !rwctx.TryPushTimeSeries(shard, forceDropSamplesOnFailure) {
 				anyPushFailed.Store(true)
 			}
-		}(rwctx, shard)
+		})
 	}
 	wg.Wait()
 	return !anyPushFailed.Load()
@@ -848,7 +822,7 @@ type remoteWriteCtx struct {
 	rowsDroppedOnPushFailure     *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
@@ -863,6 +837,23 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	}
 
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
+	queuesSize := queues.GetOptionalArg(argIdx)
+	if queuesSize > maxQueues {
+		queuesSize = maxQueues
+	} else if queuesSize <= 0 {
+		queuesSize = 1
+	}
+
+	maxInmemoryBlocks := memory.Allowed() / len(*remoteWriteURLs) / *maxRowsPerBlock / 100
+	if maxInmemoryBlocks/queuesSize > 100 {
+		// There is no much sense in keeping higher number of blocks in memory,
+		// since this means that the producer outperforms consumer and the queue
+		// will continue growing. It is better storing the queue to file.
+		maxInmemoryBlocks = 100 * queuesSize
+	}
+	if maxInmemoryBlocks < 2 {
+		maxInmemoryBlocks = 2
+	}
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
 		return float64(fq.GetPendingBytes())
@@ -880,16 +871,16 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 	var c *client
 	switch remoteWriteURL.Scheme {
 	case "http", "https":
-		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, *queues)
+		c = newHTTPClient(argIdx, remoteWriteURL.String(), sanitizedURL, fq, queuesSize)
 	default:
 		logger.Fatalf("unsupported scheme: %s for remoteWriteURL: %s, want `http`, `https`", remoteWriteURL.Scheme, sanitizedURL)
 	}
-	c.init(argIdx, *queues, sanitizedURL)
+	c.init(argIdx, queuesSize, sanitizedURL)
 
 	// Initialize pss
 	sf := significantFigures.GetOptionalArg(argIdx)
 	rd := roundDigits.GetOptionalArg(argIdx)
-	pssLen := *queues
+	pssLen := queuesSize
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
 		// since every pendingSeries can saturate up to a single CPU.

app/vmalert/config/types.go

@@ -76,11 +76,14 @@ func (t *Type) ValidateExpr(expr string) error {
 		if err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
-		fields, _ := q.GetStatsByFields()
-		for i := range fields {
+		labels, err := q.GetStatsLabels()
+		if err != nil {
+			return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
+		}
+		for i := range labels {
 			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
 			// making the result meaningless and may lead to cardinality issues.
-			if fields[i] == "_time" {
+			if labels[i] == "_time" {
 				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
 			}
 		}

app/vmalert/main.go

@@ -81,9 +81,7 @@ absolute path to all .tpl files in root.
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
 
-var (
-	extURL *url.URL
-)
+var extURL *url.URL
 
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
@@ -161,7 +159,7 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	manager, err := newManager(ctx)
 	if err != nil {
-		logger.Fatalf("failed to init: %s", err)
+		logger.Fatalf("failed to create manager: %s", err)
 	}
 	logger.Infof("reading rules configuration file from %q", strings.Join(*rulePath, ";"))
 	groupsCfg, err := config.Parse(*rulePath, validateTplFn, *validateExpressions)

app/vmalert/manager_test.go

@@ -65,11 +65,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
@@ -79,7 +77,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 				}
 				_ = m.update(context.Background(), cfg, false)
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }

app/vmalert/notifier/alertmanager.go

@@ -14,7 +14,6 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
@@ -172,11 +171,6 @@ const alertManagerPath = "/api/v2/alerts"
 func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg promauth.HTTPClientConfig,
 	relabelCfg *promrelabel.ParsedConfigs, timeout time.Duration,
 ) (*AlertManager, error) {
-
-	if err := httputil.CheckURL(alertManagerURL); err != nil {
-		return nil, fmt.Errorf("invalid alertmanager URL: %w", err)
-	}
-
 	tls := &promauth.TLSConfig{}
 	if authCfg.TLSConfig != nil {
 		tls = authCfg.TLSConfig

app/vmalert/notifier/config_watcher_test.go

@@ -212,18 +212,16 @@ consul_sd_configs:
 
 	const workers = 500
 	const iterations = 10
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(n int) {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for n := range workers {
+		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
 				_ = cw.reload(paths[rnd]) // update can fail and this is expected
 				_ = cw.notifiers()
 			}
-		}(i)
+		})
 	}
 	wg.Wait()
 }

app/vmalert/notifier/init.go

@@ -11,8 +11,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
@@ -229,6 +229,9 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
+		if err := httputil.CheckURL(addr); err != nil {
+			return nil, fmt.Errorf("invalid notifier.url %q: %w", addr, err)
+		}
 		addr = strings.TrimSuffix(addr, "/")
 		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, sendTimeout.GetOptionalArg(i))
 		if err != nil {
@@ -266,7 +269,7 @@ func GetTargets() map[TargetType][]Target {
 	if getActiveNotifiers == nil {
 		return nil
 	}
-	var targets = make(map[TargetType][]Target)
+	targets := make(map[TargetType][]Target)
 	// use cached targets from configWatcher instead of getActiveNotifiers for the extra target labels
 	if cw != nil {
 		cw.targetsMu.RLock()
@@ -287,7 +290,7 @@ func GetTargets() map[TargetType][]Target {
 }
 
 // Send sends alerts to all active notifiers
-func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) *vmalertutil.ErrGroup {
+func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) chan error {
 	alertsToSend := make([]Alert, 0, len(alerts))
 	lblss := make([][]prompb.Label, 0, len(alerts))
 	// apply global relabel config first without modifying original alerts in alerts
@@ -300,17 +303,18 @@ func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string
 		lblss = append(lblss, lbls)
 	}
 
-	errGr := new(vmalertutil.ErrGroup)
 	wg := sync.WaitGroup{}
 	activeNotifiers := getActiveNotifiers()
+	errCh := make(chan error, len(activeNotifiers))
+	defer close(errCh)
 	for i := range activeNotifiers {
 		nt := activeNotifiers[i]
 		wg.Go(func() {
 			if err := nt.Send(ctx, alertsToSend, lblss, notifierHeaders); err != nil {
-				errGr.Add(fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err))
+				errCh <- fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err)
 			}
 		})
 	}
 	wg.Wait()
-	return errGr
+	return errCh
 }

app/vmalert/notifier/init_test.go

@@ -55,9 +55,9 @@ func TestInitNegative(t *testing.T) {
 		*blackHole = oldBlackHole
 	}()
 
-	f := func(path, addr string, bh bool) {
+	f := func(path string, addr []string, bh bool) {
 		*configPath = path
-		*addrs = flagutil.ArrayString{addr}
+		*addrs = flagutil.ArrayString(addr)
 		*blackHole = bh
 		if err := Init(nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
@@ -65,9 +65,12 @@ func TestInitNegative(t *testing.T) {
 	}
 
 	// *configPath, *addrs and *blackhole are mutually exclusive
-	f("/dummy/path", "127.0.0.1", false)
-	f("/dummy/path", "", true)
-	f("", "127.0.0.1", true)
+	f("/dummy/path", []string{"127.0.0.1"}, false)
+	f("/dummy/path", []string{}, true)
+	f("", []string{"127.0.0.1"}, true)
+	// addr cannot be ""
+	f("", []string{""}, false)
+	f("", []string{"127.0.0.1", ""}, false)
 }
 
 func TestBlackHole(t *testing.T) {
@@ -202,7 +205,9 @@ alert_relabel_configs:
 		},
 	}
 	errG := Send(context.Background(), firingAlerts, nil)
-	if errG.Err() != nil {
-		t.Fatalf("unexpected error when sending alerts: %s", err)
+	for err := range errG {
+		if err != nil {
+			t.Errorf("unexpected error when sending alerts: %s", err)
+		}
 	}
 }

app/vmalert/rule/alerting.go

@@ -818,7 +818,9 @@ func (ar *AlertingRule) restore(ctx context.Context, q datasource.Querier, ts ti
 	expr := fmt.Sprintf("default_rollup(%s{%s%s}[%ds])",
 		alertForStateMetricName, nameStr, labelsFilter, int(lookback.Seconds()))
 
-	res, _, err := q.Query(ctx, expr, ts)
+	// query ALERTS_FOR_STATE at `ts-1s` instead `ts` to avoid retrieving data written in the current run,
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10335
+	res, _, err := q.Query(ctx, expr, ts.Add(-1*time.Second))
 	if err != nil {
 		return fmt.Errorf("failed to execute restore query %q: %w ", expr, err)
 	}

app/vmalert/rule/group.go

@@ -18,6 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -374,7 +375,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 
 	g.infof("started")
 
-	eval := func(ctx context.Context, ts time.Time) {
+	eval := func(ctx context.Context, ts time.Time) time.Time {
 		g.metrics.iterationTotal.Inc()
 
 		start := time.Now()
@@ -382,7 +383,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 		if len(g.Rules) < 1 {
 			g.metrics.iterationDuration.UpdateDuration(start)
 			g.LastEvaluation = start
-			return
+			return ts
 		}
 
 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
@@ -396,6 +397,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 		}
 		g.metrics.iterationDuration.UpdateDuration(start)
 		g.LastEvaluation = start
+		return ts
 	}
 
 	evalCtx, cancel := context.WithCancel(ctx)
@@ -404,7 +406,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	g.mu.Unlock()
 	defer g.evalCancel()
 
-	eval(evalCtx, evalTS)
+	realEvalTS := eval(evalCtx, evalTS)
 
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
@@ -412,7 +414,7 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	// restore the rules state after the first evaluation
 	// so only active alerts can be restored.
 	if rr != nil {
-		err := g.restore(ctx, rr, evalTS, *remoteReadLookBack)
+		err := g.restore(ctx, rr, realEvalTS, *remoteReadLookBack)
 		if err != nil {
 			logger.Errorf("error while restoring ruleState for group %q: %s", g.Name, err)
 		}
@@ -755,6 +757,7 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		return fmt.Errorf("rule %q: failed to execute: %w", r, err)
 	}
 
+	var errG vmalertutil.ErrGroup
 	if e.Rw != nil {
 		pushToRW := func(tss []prompb.TimeSeries) error {
 			var lastErr error
@@ -766,20 +769,26 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 			return lastErr
 		}
 		if err := pushToRW(tss); err != nil {
-			return err
+			errG.Add(err)
 		}
 	}
 
 	ar, ok := r.(*AlertingRule)
 	if !ok {
-		return nil
+		return errG.Err()
 	}
 
 	alerts := ar.alertsToSend(resolveDuration, *resendDelay)
 	if len(alerts) < 1 {
-		return nil
+		return errG.Err()
 	}
 
-	errGr := notifier.Send(ctx, alerts, e.notifierHeaders)
-	return errGr.Err()
+	notifierErr := notifier.Send(ctx, alerts, e.notifierHeaders)
+	for err := range notifierErr {
+		if err != nil {
+			errG.Add(fmt.Errorf("rule %q: notifier failure: %w", r, err))
+		}
+	}
+
+	return errG.Err()
 }

app/vmalert/rule/rule_test.go

@@ -65,17 +65,15 @@ func TestRule_stateConcurrent(_ *testing.T) {
 	r := &AlertingRule{state: &ruleState{entries: make([]StateEntry, 20)}}
 	const workers = 50
 	const iterations = 100
-	wg := sync.WaitGroup{}
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func() {
-			defer wg.Done()
+	var wg sync.WaitGroup
+	for range workers {
+		wg.Go(func() {
 			for i := 0; i < iterations; i++ {
 				r.state.add(StateEntry{At: time.Now()})
 				r.state.getAll()
 				r.state.getLast()
 			}
-		}()
+		})
 	}
 	wg.Wait()
 }

app/vmalert/vmalertutil/err_group.go

@@ -45,7 +45,7 @@ func (eg *ErrGroup) Error() string {
 		return ""
 	}
 	var b strings.Builder
-	fmt.Fprintf(&b, "errors(%d): ", len(eg.errs))
+	fmt.Fprintf(&b, "errors(%d): \n", len(eg.errs))
 	for i, err := range eg.errs {
 		b.WriteString(err.Error())
 		if i != len(eg.errs)-1 {

app/vmalert/vmalertutil/err_group_test.go

@@ -30,8 +30,8 @@ func TestErrGroup(t *testing.T) {
 	}
 
 	f(nil, "")
-	f([]error{errors.New("timeout")}, "errors(1): timeout")
-	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): timeout\ndeadline")
+	f([]error{errors.New("timeout")}, "errors(1): \ntimeout")
+	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): \ntimeout\ndeadline")
 }
 
 // TestErrGroupConcurrent supposed to test concurrent

app/vmalert/web.qtpl

@@ -9,6 +9,7 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
     "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
+    "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 ) %}
 
 {% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
@@ -78,6 +79,8 @@
 {% func Welcome(r *http.Request) %}
     {%= tpl.Header(r, navItems, "vmalert", getLastConfigError()) %}
     <p>
+        Version {%s buildinfo.Version %} <br>
+
         API:<br>
         {% for _, p := range apiLinks  %}
             {%code p, doc := p[0], p[1] %}

app/vmauth/auth_config.go

@@ -113,10 +113,8 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
-		ui.concurrencyLimitReached.Inc()
-
-		// The per-user limit for the number of concurrent requests is reached.
-		// Wait until the currently executed requests are finished, so the current request could be executed.
+		// The number of concurrently executed requests for the given user equals the limt.
+		// Wait until some of the currently executed requests are finished, so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case ui.concurrencyLimitCh <- struct{}{}:
@@ -124,6 +122,8 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 		case <-ctx.Done():
 			err := ctx.Err()
 			if errors.Is(err, context.DeadlineExceeded) {
+				// The current request couldn't be executed until the request timeout.
+				ui.concurrencyLimitReached.Inc()
 				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because %d concurrent requests from the user %s are executed",
 					*maxQueueDuration, ui.getMaxConcurrentRequests(), ui.name())
 			}
@@ -150,12 +150,22 @@ func (ui *UserInfo) stopHealthChecks() {
 	if ui == nil {
 		return
 	}
-	if ui.URLPrefix == nil {
-		return
-	}
 
-	bus := ui.URLPrefix.bus.Load()
-	bus.stopHealthChecks()
+	if ui.URLPrefix != nil {
+		bus := ui.URLPrefix.bus.Load()
+		bus.stopHealthChecks()
+	}
+	if ui.DefaultURL != nil {
+		bus := ui.DefaultURL.bus.Load()
+		bus.stopHealthChecks()
+	}
+	for i := range ui.URLMaps {
+		um := &ui.URLMaps[i]
+		if um.URLPrefix != nil {
+			bus := um.URLPrefix.bus.Load()
+			bus.stopHealthChecks()
+		}
+	}
 }
 
 // Header is `Name: Value` http header, which must be added to the proxied request.
@@ -363,12 +373,10 @@ func (bu *backendURL) isBroken() bool {
 
 func (bu *backendURL) setBroken() {
 	if bu.broken.CompareAndSwap(false, true) {
-		bu.healthCheckWG.Add(1)
-		go func() {
-			defer bu.healthCheckWG.Done()
+		bu.healthCheckWG.Go(func() {
 			bu.runHealthCheck()
 			bu.broken.Store(false)
-		}()
+		})
 	}
 }
 
@@ -733,11 +741,9 @@ func initAuthConfig() {
 	configTimestamp.Set(fasttime.UnixTimestamp())
 
 	stopCh = make(chan struct{})
-	authConfigWG.Add(1)
-	go func() {
-		defer authConfigWG.Done()
+	authConfigWG.Go(func() {
 		authConfigReloader(sighupCh)
-	}()
+	})
 }
 
 func stopAuthConfig() {

app/vmauth/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ioutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -40,27 +41,38 @@ var (
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
-	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
-		"See also -maxConcurrentRequests")
-	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
+	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
+	idleConnTimeout        = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
 		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
 	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 
-	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
-		"'429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
-	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
-		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options "+
-		"and max_concurrent_requests option in per-user config")
-	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration the request waits for execution when the number of concurrently executed "+
-		"requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. "+
-		"This allows graceful handling of short spikes in the number of concurrent requests")
+	requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
+		"This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. "+
+		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
+	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
+		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
+		"See also -requestBufferSize")
+
+	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process simultaneously. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This protects vmauth itself from overloading and out-of-memory (OOM) failures. See also -maxConcurrentPerUserRequests "+
+		"and https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 100, "The maximum number of concurrent requests vmauth can process per each configured user. "+
+		"Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. "+
+		"This provides fairness and isolation between users, preventing a single user from consuming all the available resources. "+
+		"It works in conjunction with -maxConcurrentRequests, which sets the global limit across all users. "+
+		"This default can be overridden for individual users via max_concurrent_requests option in per-user config. "+
+		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
+	maxQueueDuration = flag.Duration("maxQueueDuration", 10*time.Second, "The maximum duration to wait before rejecting incoming requests if concurrency limit "+
+		"specified via -maxConcurrentRequests or -maxConcurrentPerUserRequests command-line flags is reached. "+
+		"Requests are rejected with '429 Too Many Requests' http status code if the limit is still reached after the -maxQueueDuration duration. "+
+		"This allows graceful handling of short spikes in concurrent requests. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting")
 
 	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
-	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
-	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
-		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
+	failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
+
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -215,48 +227,121 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
 	defer cancel()
 
-	// Limit the concurrency of requests to backends
+	// Acquire global concurrency limit.
+	if err := beginConcurrencyLimit(ctx); err != nil {
+		handleConcurrencyLimitError(w, r, err)
+		return
+	}
+	defer endConcurrencyLimit()
+
+	// Set read deadline for reading the initial chunk for the request body.
+	rc := http.NewResponseController(w)
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		logger.Panicf("BUG: expecting valid deadline for the context")
+	}
+	if err := rc.SetReadDeadline(deadline); err != nil {
+		logger.Panicf("BUG: cannot set read deadline: %s", err)
+	}
+
+	// Read the initial chunk for the request body.
+	userName := ui.name()
+	if userName == "" {
+		userName = "unauthorized"
+	}
+	bb, err := bufferRequestBody(ctx, r.Body, userName)
+	if err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	r.Body = bb
+
+	// Disable the read deadline for the rest of the request body.
+	if err := rc.SetReadDeadline(time.Time{}); err != nil {
+		logger.Panicf("BUG: cannot reset read deadline: %s", err)
+	}
+
+	// Acquire concurrency limit for the given user.
+	if err := ui.beginConcurrencyLimit(ctx); err != nil {
+		handleConcurrencyLimitError(w, r, err)
+		return
+	}
+	defer ui.endConcurrencyLimit()
+
+	// Process the request.
+	processRequest(w, r, ui)
+}
+
+func beginConcurrencyLimit(ctx context.Context) error {
 	concurrencyLimitOnce.Do(concurrencyLimitInit)
 	select {
 	case concurrencyLimitCh <- struct{}{}:
-		if err := ui.beginConcurrencyLimit(ctx); err != nil {
-			handleConcurrencyLimitError(w, r, err)
-			<-concurrencyLimitCh
-			return
-		}
+		return nil
 	default:
 		// The -maxConcurrentRequests are executed. Wait until some of the requests are finished,
 		// so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
 		case concurrencyLimitCh <- struct{}{}:
-			if err := ui.beginConcurrencyLimit(ctx); err != nil {
-				handleConcurrencyLimitError(w, r, err)
-				<-concurrencyLimitCh
-				return
-			}
+			return nil
 		case <-ctx.Done():
 			err := ctx.Err()
-
-			concurrentRequestsLimitReached.Inc()
-
 			if errors.Is(err, context.DeadlineExceeded) {
-				err = fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
+				// The current request couldn't be executed until the request timeout.
+				concurrentRequestsLimitReached.Inc()
+				return fmt.Errorf("cannot start executing the request during -maxQueueDuration=%s because -maxConcurrentRequests=%d concurrent requests are executed",
 					*maxQueueDuration, cap(concurrencyLimitCh))
-				handleConcurrencyLimitError(w, r, err)
-				return
 			}
-
-			err = fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
-			handleConcurrencyLimitError(w, r, err)
-			return
+			return fmt.Errorf("cannot start executing the request because -maxConcurrentRequests=%d concurrent requests are executed: %w", cap(concurrencyLimitCh), err)
 		}
 	}
-	processRequest(w, r, ui)
-	ui.endConcurrencyLimit()
+}
+
+func endConcurrencyLimit() {
 	<-concurrencyLimitCh
 }
 
+func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (io.ReadCloser, error) {
+	if r == nil {
+		// This is a GET request with nil reader.
+		return nil, nil
+	}
+
+	maxBufSize := max(requestBufferSize.IntN(), maxRequestBodySizeToRetry.IntN())
+	if maxBufSize <= 0 {
+		return r, nil
+	}
+
+	lr := ioutil.GetLimitedReader(r, int64(maxBufSize))
+	defer ioutil.PutLimitedReader(lr)
+
+	start := time.Now()
+	buf, err := io.ReadAll(lr)
+	bufferRequestBodyDuration.UpdateDuration(start)
+
+	if err != nil {
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			rejectSlowClientRequests.Inc()
+
+			d := time.Since(start)
+
+			return nil, &httpserver.ErrorWithStatusCode{
+				Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
+					userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
+				StatusCode: http.StatusBadRequest,
+			}
+		}
+
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot read request body: %w", err),
+			StatusCode: http.StatusBadRequest,
+		}
+	}
+
+	bb := newBufferedBody(r, buf, maxBufSize)
+	return bb, nil
+}
+
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
@@ -282,9 +367,6 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		isDefault = true
 	}
 
-	rtb := newReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
-	r.Body = rtb
-
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getBackendURL()
@@ -292,18 +374,19 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			break
 		}
 		targetURL := bu.url
-		// Don't change path and add request_path query param for default route.
 		if isDefault {
+			// Don't change path and add request_path query param for default route.
 			query := targetURL.Query()
 			query.Set("request_path", u.String())
 			targetURL.RawQuery = query.Encode()
-		} else { // Update path for regular routes.
+		} else {
+			// Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		}
 
 		wasLocalRetry := false
 	again:
-		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui, bu)
 		if needLocalRetry && !wasLocalRetry {
 			wasLocalRetry = true
 			goto again
@@ -313,6 +396,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if ok {
 			return
 		}
+
 		bu.setBroken()
 		ui.backendErrors.Inc()
 	}
@@ -324,7 +408,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ui.requestErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo, bu *backendURL) (bool, bool) {
 	ui.backendRequests.Inc()
 	req := sanitizeRequestHeaders(r)
 
@@ -339,30 +423,19 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		}
 	}
 
-	rtb, rtbOK := req.Body.(*readTrackingBody)
+	bb, bbOK := req.Body.(*bufferedBody)
+	canRetry := !bbOK || bb.canRetry()
+
 	res, err := ui.rt.RoundTrip(req)
 
-	if ctxErr := r.Context().Err(); ctxErr != nil {
-		// Override the error returned by the RoundTrip with the context error if it isn't non-nil
-		// This makes sure the proper logging for canceled and timed out requests - log the real cause of the error
-		// instead of the random error, which could be returned from RoundTrip because of canceled or timed out request.
-		err = ctxErr
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
+		clientCanceledRequests.Inc()
+		return true, false
 	}
+
 	if err != nil {
-		// Do not retry canceled
-		if errors.Is(err, context.Canceled) {
-			clientCanceledRequests.Inc()
-			return true, false
-		}
-		// Do not retry timed out requests
-		if errors.Is(err, context.DeadlineExceeded) {
-			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-			requestURI := httpserver.GetRequestURI(r)
-			// Timed out request must be counted as errors, since this usually means that the backend is slow.
-			logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
-			return false, false
-		}
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("cannot proxy the request to %s: %w", targetURL, err),
@@ -371,27 +444,32 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
 			ui.requestErrors.Inc()
+			bu.setBroken()
 			return true, false
 		}
 		if netutil.IsTrivialNetworkError(err) {
 			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
+			if bbOK {
+				bb.resetReader()
+			}
 			return false, true
 		}
 
-		// Request body wasn't read yet, this usually means that the backend isn't reachable; retry the request at another backend
+		// Retry the request at another backend
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, req.URL, targetURL, err)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed: %s, retrying the request at another backend", remoteAddr, requestURI, targetURL, err)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
-		_ = res.Body.Close()
-		if !rtbOK || !rtb.canRetry() {
+		if !canRetry {
 			// If we get an error from the retry_status_codes list, but cannot execute retry,
 			// we consider such a request an error as well.
 			err := &httpserver.ErrorWithStatusCode{
-				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request has been already consumed",
+				Err: fmt.Errorf("got response status code=%d from %s, but cannot retry the request at another backend, because the request body has been already consumed",
 					res.StatusCode, targetURL),
 				StatusCode: http.StatusServiceUnavailable,
 			}
@@ -400,13 +478,16 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			ui.requestErrors.Inc()
 			return true, false
 		}
+
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
-		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body, which may fail retries.
+		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; request to %s failed, retrying the request at another backend because response status code=%d belongs to retry_status_codes=%d",
-			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
+			remoteAddr, requestURI, targetURL, res.StatusCode, retryStatusCodes)
+		if bbOK {
+			bb.resetReader()
+		}
 		return false, false
 	}
 	removeHopHeaders(res.Header)
@@ -416,13 +497,16 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	err = copyStreamToClient(w, res.Body)
 	_ = res.Body.Close()
-	if errors.Is(err, context.Canceled) {
+
+	if errors.Is(r.Context().Err(), context.Canceled) {
+		// Do not retry canceled requests.
 		clientCanceledRequests.Inc()
 		return true, false
-	} else if err != nil && !netutil.IsTrivialNetworkError(err) {
+	}
+
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
-
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
 		ui.requestErrors.Inc()
 		return true, false
@@ -553,6 +637,9 @@ var (
 	invalidAuthTokenRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="invalid_auth_token"}`)
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
 	clientCanceledRequests   = metrics.NewCounter(`vmauth_http_request_errors_total{reason="client_canceled"}`)
+	rejectSlowClientRequests = metrics.NewCounter(`vmauth_http_request_errors_total{reason="reject_slow_client"}`)
+
+	bufferRequestBodyDuration = metrics.NewSummary(`vmauth_buffer_request_body_duration_seconds`)
 )
 
 func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, insecureSkipVerifyP *bool) (http.RoundTripper, error) {
@@ -636,8 +723,7 @@ func handleMissingAuthorizationError(w http.ResponseWriter) {
 }
 
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
-	ctx := r.Context()
-	if errors.Is(ctx.Err(), context.Canceled) {
+	if errors.Is(r.Context().Err(), context.Canceled) {
 		// Do not return any response for the request canceled by the client,
 		// since the connection to the client is already closed.
 		clientCanceledRequests.Inc()
@@ -652,123 +738,78 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-// readTrackingBody must be obtained via getReadTrackingBody()
-type readTrackingBody struct {
-	// maxBodySize is the maximum body size to cache in buf.
+// bufferedBody serves two purposes:
+//  1. Enables request retries when the body size does not exceed maxBodySize
+//     by fully buffering the body in memory.
+//  2. Prevents slow clients from reducing effective server capacity by
+//     buffering the request body before acquiring a per-user concurrency slot.
+//
+// See bufferRequestBody for details on how bufferedBody is used.
+type bufferedBody struct {
+	// r contains reader for reading the data after buf is read.
 	//
-	// Bigger bodies cannot be retried.
-	maxBodySize int
-
-	// r contains reader for initial data reading
+	// r is nil if buf contains all the data.
 	r io.ReadCloser
 
-	// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
-	// If more than maxBodySize is read from r, then cannotRetry is set to true.
+	// buf contains the initial buffer read from r.
 	buf []byte
 
-	// readBuf points to the cached data at buf, which must be read in the next call to Read().
-	readBuf []byte
+	// bufOffset is the offset at buf for already read bytes.
+	bufOffset int
 
-	// cannotRetry is set to true when more than maxBodySize bytes are read from r.
-	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
+	// cannotRetry is set to true after Close() call on non-nil r.
 	cannotRetry bool
-
-	// bufComplete is set to true when buf contains complete request body read from r.
-	bufComplete bool
 }
 
-func newReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
-	// do not use sync.Pool there
-	// since http.RoundTrip may still use request body after return
-	// See this issue for details https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
-	rtb := &readTrackingBody{}
-	if maxBodySize < 0 {
-		maxBodySize = 0
+func newBufferedBody(r io.ReadCloser, buf []byte, maxBufSize int) *bufferedBody {
+	// Do not use sync.Pool here, since http.RoundTrip may still use request body after return.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
+
+	if len(buf) < maxBufSize {
+		// Read the full request body into buf.
+		r = nil
 	}
-	rtb.maxBodySize = maxBodySize
 
-	if r == nil {
-		// This is GET request without request body
-		r = (*zeroReader)(nil)
+	return &bufferedBody{
+		r:   r,
+		buf: buf,
 	}
-	rtb.r = r
-	return rtb
-}
-
-type zeroReader struct{}
-
-func (r *zeroReader) Read(_ []byte) (int, error) {
-	return 0, io.EOF
-}
-
-func (r *zeroReader) Close() error {
-	return nil
 }
 
 // Read implements io.Reader interface.
-func (rtb *readTrackingBody) Read(p []byte) (int, error) {
-	if len(rtb.readBuf) > 0 {
-		n := copy(p, rtb.readBuf)
-		rtb.readBuf = rtb.readBuf[n:]
+func (bb *bufferedBody) Read(p []byte) (int, error) {
+	if bb.cannotRetry {
+		return 0, fmt.Errorf("cannot read already closed body")
+	}
+	if bb.bufOffset < len(bb.buf) {
+		n := copy(p, bb.buf[bb.bufOffset:])
+		bb.bufOffset += n
 		return n, nil
 	}
-
-	if rtb.r == nil {
-		if rtb.bufComplete {
-			return 0, io.EOF
-		}
-		return 0, fmt.Errorf("cannot read client request body after closing client reader")
+	if bb.r == nil {
+		return 0, io.EOF
 	}
-
-	n, err := rtb.r.Read(p)
-	if rtb.cannotRetry {
-		return n, err
-	}
-
-	if len(rtb.buf)+n > rtb.maxBodySize {
-		rtb.cannotRetry = true
-		return n, err
-	}
-	rtb.buf = append(rtb.buf, p[:n]...)
-	if err == io.EOF {
-		rtb.bufComplete = true
-	}
-	return n, err
+	return bb.r.Read(p)
 }
 
-func (rtb *readTrackingBody) canRetry() bool {
-	if rtb.cannotRetry {
-		return false
-	}
-	if rtb.bufComplete {
-		return true
-	}
-	return rtb.r != nil
+func (bb *bufferedBody) canRetry() bool {
+	return bb.r == nil
 }
 
 // Close implements io.Closer interface.
-func (rtb *readTrackingBody) Close() error {
-	if !rtb.cannotRetry {
-		rtb.readBuf = rtb.buf
-	} else {
-		rtb.readBuf = nil
+func (bb *bufferedBody) Close() error {
+	bb.resetReader()
+	if bb.r != nil {
+		bb.cannotRetry = true
+		return bb.r.Close()
 	}
-
-	// Close rtb.r only if the request body is completely read or if it is too big.
-	// http.Roundtrip performs body.Close call even without any Read calls,
-	// so this hack allows us to reuse request body.
-	if rtb.bufComplete || rtb.cannotRetry {
-		if rtb.r == nil {
-			return nil
-		}
-		err := rtb.r.Close()
-		rtb.r = nil
-		return err
-	}
-
 	return nil
 }
 
+func (bb *bufferedBody) resetReader() {
+	bb.bufOffset = 0
+}
+
 func debugInfo(u *url.URL, r *http.Request) string {
 	s := &strings.Builder{}
 	fmt.Fprintf(s, " (host: %q; ", r.Host)

app/vmauth/main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net"
@@ -10,6 +11,7 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
@@ -546,28 +548,300 @@ func (w *fakeResponseWriter) WriteHeader(statusCode int) {
 	}
 }
 
-func TestReadTrackingBody_RetrySuccess(t *testing.T) {
+// This is needed for net/http.ResponseController
+func (w *fakeResponseWriter) SetReadDeadline(deadline time.Time) error {
+	return nil
+}
+
+func TestBufferRequestBody_Success(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	f := func(body *bytes.Buffer, requestBufferSizeFlag, maxRequestBodySizeToRetryFlag string) {
+		t.Helper()
+
+		expectedResponse := "statusCode=200"
+		if body.Len() > 0 {
+			expectedResponse += "\n" + body.String()
+		}
+
+		if err := requestBufferSize.Set(requestBufferSizeFlag); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		if err := maxRequestBodySizeToRetry.Set(maxRequestBodySizeToRetryFlag); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if !backendCalled {
+			t.Fatalf("backend is not called")
+		}
+	}
+
+	// no body, no buffering, no retry
+	f(bytes.NewBuffer(nil), "0", "0")
+
+	// no body, buffering on, no retry
+	f(bytes.NewBuffer(nil), "100", "0")
+
+	// no body, no buffering, retry on
+	f(bytes.NewBuffer(nil), "0", "100")
+
+	// no body, buffering on, retry on
+	f(bytes.NewBuffer(nil), "100", "100")
+
+	// body smaller than buffer, retry max on
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "101", "101")
+
+	// body smaller than buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "501", "0")
+
+	// body same size as buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "500", "0")
+
+	// body bigger than a buffer
+	f(bytes.NewBufferString(strings.Repeat("abcdf", 100)), "499", "0")
+
+	// body bigger than tmpBuf 8KiB used in buffering
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16384", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16385", "")
+
+	f(bytes.NewBufferString(strings.Repeat("a", 32*1024)), "16383", "")
+}
+
+func TestBufferRequestBody_Failure(t *testing.T) {
+	defaultRequestBufferSize := requestBufferSize.String()
+	defer func() {
+		if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+			t.Fatalf("cannot reset requestBufferSize: %s", err)
+		}
+	}()
+
+	defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+	defer func() {
+		if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+			t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+		}
+	}()
+
+	defaultMaxQueueDuration := *maxQueueDuration
+	defer func() {
+		*maxQueueDuration = defaultMaxQueueDuration
+	}()
+
+	f := func(body *mockBody, expectedResponse string) {
+		t.Helper()
+
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+		if err := requestBufferSize.Set("2048"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
+		*maxQueueDuration = 100 * time.Millisecond
+
+		var backendCalled bool
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			backendCalled = true
+
+			b, err := io.ReadAll(r.Body)
+			if err != nil {
+				http.Error(w, fmt.Sprintf("cannot read body: %s", err), http.StatusBadRequest)
+				return
+			}
+			if _, err := w.Write(b); err != nil {
+				http.Error(w, fmt.Sprintf("cannot write body: %s", err), http.StatusInternalServerError)
+				return
+			}
+		}))
+		defer ts.Close()
+
+		// regular url_prefix
+		cfgStr := strings.ReplaceAll(`
+unauthorized_user:
+  url_prefix: {BACKEND}/foo`, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodPost, `http://some-host.com`, body)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		w := &fakeResponseWriter{}
+		if !requestHandlerWithInternalRoutes(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+
+		if response != expectedResponse {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, expectedResponse)
+		}
+		if backendCalled {
+			t.Fatalf("backend is called")
+		}
+	}
+
+	// an error at the beginning of reading
+	f(&mockBody{err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+
+	// an error after reading 1024 bytes, buffer size is 2048 bytes
+	f(&mockBody{head: make([]byte, 1024), err: fmt.Errorf("an error")}, `statusCode=400
+cannot read request body: an error`)
+}
+
+type mockBody struct {
+	head []byte
+	err  error
+	tail []byte
+}
+
+func (r *mockBody) Read(p []byte) (n int, err error) {
+	if len(r.head) > 0 {
+		n = copy(p, r.head)
+		r.head = r.head[n:]
+		return n, nil
+	}
+
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	if len(r.tail) > 0 {
+		n = copy(p, r.tail)
+		r.tail = r.tail[n:]
+		return n, nil
+	}
+
+	return 0, io.EOF
+}
+
+func TestBufferedBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		for i := 0; i < 5; i++ {
-			data, err := io.ReadAll(rtb)
+			data, err := io.ReadAll(rb)
 			if err != nil {
 				t.Fatalf("unexpected error when reading all the data at iteration %d: %s", i, err)
 			}
 			if string(data) != s {
 				t.Fatalf("unexpected data read at iteration %d\ngot\n%s\nwant\n%s", i, data, s)
 			}
-			if err := rtb.Close(); err != nil {
-				t.Fatalf("unexpected error when closing readTrackingBody at iteration %d: %s", i, err)
-			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true at iteration %d", i)
+			if err := rb.Close(); err != nil {
+				t.Fatalf("unexpected error when closing bufferedBody at iteration %d: %s", i, err)
 			}
 		}
 	}
@@ -577,19 +851,48 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
+func TestBufferedBody_RetrySuccessPartialRead(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
 		// Check the case with partial read
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set("0"); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
+			t.Fatalf("canRetry must return true")
+		}
 		for i := 0; i < len(s); i++ {
 			buf := make([]byte, i)
-			n, err := io.ReadFull(rtb, buf)
+			n, err := io.ReadFull(rb, buf)
 			if err != nil {
 				t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
 			}
@@ -599,26 +902,20 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 			if string(buf) != s[:i] {
 				t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
 			}
-			if err := rtb.Close(); err != nil {
+			if err := rb.Close(); err != nil {
 				t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
 			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
-			}
 		}
 
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 	}
 
@@ -627,30 +924,53 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 	f("", 100)
 	f("foo", 100)
 	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f(newTestString(1000), 1001)
 }
 
-func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
+func TestBufferedBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set("0"); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true before reading anything")
+		defaultMaxRequestBodySizeToRetry := maxRequestBodySizeToRetry.String()
+		defer func() {
+			if err := maxRequestBodySizeToRetry.Set(defaultMaxRequestBodySizeToRetry); err != nil {
+				t.Fatalf("cannot reset maxRequestBodySizeToRetry: %s", err)
+			}
+		}()
+		if err := maxRequestBodySizeToRetry.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set maxRequestBodySizeToRetry: %s", err)
+		}
+
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if canRetry {
+			t.Fatalf("canRetry() must return false because of too big request body")
 		}
 		buf := make([]byte, 1)
-		n, err := io.ReadFull(rtb, buf)
+		n, err := io.ReadFull(rb, buf)
 		if err != nil {
 			t.Fatalf("unexpected error when reading a single byte: %s", err)
 		}
 		if n != 1 {
 			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
 		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after reading one byte")
-		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
@@ -658,14 +978,11 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 		if dataRead != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		data, err = io.ReadAll(rtb)
+		data, err = io.ReadAll(rb)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -679,35 +996,48 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 	f(newTestString(2*maxBodySize), maxBodySize)
 }
 
-func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
+func TestBufferedBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defaultRequestBufferSize := requestBufferSize.String()
+		defer func() {
+			if err := requestBufferSize.Set(defaultRequestBufferSize); err != nil {
+				t.Fatalf("cannot reset requestBufferSize: %s", err)
+			}
+		}()
+		if err := requestBufferSize.Set(fmt.Sprintf("%d", maxBodySize)); err != nil {
+			t.Fatalf("cannot set requestBufferSize: %s", err)
+		}
 
-		if !rtb.canRetry() {
+		ctx := context.Background()
+		rb, err := bufferRequestBody(ctx, io.NopCloser(bytes.NewBufferString(s)), "foo")
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		bb, ok := rb.(*bufferedBody)
+		canRetry := !ok || bb.canRetry()
+
+		if !canRetry {
 			t.Fatalf("canRetry() must return true before reading anything")
 		}
-		data, err := io.ReadAll(rtb)
+		data, err := io.ReadAll(rb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
 		if string(data) != s {
 			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		if err := rb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing bufferedBody: %s", err)
 		}
 
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		data, err = io.ReadAll(rb)
+		if err != nil {
+			t.Fatalf("unexpected error in io.ReadAll: %s", err)
 		}
-		data, err = io.ReadAll(rtb)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if len(data) != 0 {
-			t.Fatalf("unexpected non-empty data read: %q", data)
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
 	}
 

app/vmctl/backoff/backoff.go

@@ -7,6 +7,8 @@ import (
 	"math"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
@@ -55,6 +57,7 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
 			return attempt, err // fail fast if not recoverable
 		}
 		attempt++
+		retriesTotal.Inc()
 		backoff := float64(b.minDuration) * math.Pow(b.factor, float64(i))
 		dur := time.Duration(backoff)
 		logger.Errorf("got error: %s on attempt: %d; will retry in %v", err, attempt, dur)
@@ -74,3 +77,7 @@ func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
 	}
 	return attempt, fmt.Errorf("execution failed after %d retry attempts", b.retries)
 }
+
+var (
+	retriesTotal = metrics.NewCounter(`vmctl_backoff_retries_total`)
+)

app/vmctl/flags.go

@@ -14,6 +14,12 @@ const (
 	globalSilent             = "s"
 	globalVerbose            = "verbose"
 	globalDisableProgressBar = "disable-progress-bar"
+
+	globalPushMetricsURL         = "pushmetrics.url"
+	globalPushMetricsInterval    = "pushmetrics.interval"
+	globalPushExtraLabels        = "pushmetrics.extraLabel"
+	globalPushHeaders            = "pushmetrics.header"
+	globalPushDisableCompression = "pushmetrics.disableCompression"
 )
 
 var (
@@ -33,6 +39,29 @@ var (
 			Value: false,
 			Usage: "Whether to disable progress bar during the import.",
 		},
+		&cli.StringSliceFlag{
+			Name:  globalPushMetricsURL,
+			Usage: "Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics",
+		},
+		&cli.DurationFlag{
+			Name:  globalPushMetricsInterval,
+			Value: 10 * time.Second,
+			Usage: "Interval for pushing metrics to every -pushmetrics.url",
+		},
+		&cli.StringSliceFlag{
+			Name: globalPushExtraLabels,
+			Usage: "Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. " +
+				"Flag can be set multiple times, to add few additional labels. " +
+				"For example, -pushmetrics.extraLabel='instance=\"foo\"' adds instance=\"foo\" label to all the metrics pushed to every -pushmetrics.url",
+		},
+		&cli.StringSliceFlag{
+			Name:  globalPushHeaders,
+			Usage: "Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.",
+		},
+		&cli.BoolFlag{
+			Name:  globalPushDisableCompression,
+			Usage: "Whether to disable compression when pushing metrics.",
+		},
 	}
 )
 
@@ -123,32 +152,32 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
-				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination.",
+				"By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination.",
 		},
 		&cli.StringFlag{
 			Name:  vmCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to '--vmAddr'",
+			Usage: "Optional path to client-side TLS key to use when connecting to '--vm-addr'",
 		},
 		&cli.StringFlag{
 			Name:  vmCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used",
+			Usage: "Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used",
 		},
 		&cli.StringFlag{
 			Name:  vmServerName,
-			Usage: "Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used",
+			Usage: "Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used",
 		},
 		&cli.BoolFlag{
 			Name:  vmInsecureSkipVerify,
-			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
+			Usage: "Whether to skip tls verification when connecting to '--vm-addr'",
 			Value: false,
 		},
 		&cli.IntFlag{
@@ -598,7 +627,7 @@ var (
 			Name:  vmExtraLabel,
 			Value: nil,
 			Usage: "Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag" +
-				"will have priority. Flag can be set multiple times, to add few additional labels.",
+				" will have priority. Flag can be set multiple times, to add few additional labels.",
 		},
 		&cli.Int64Flag{
 			Name: vmRateLimit,
@@ -625,8 +654,8 @@ var (
 		&cli.BoolFlag{
 			Name: vmNativeDisableBinaryProtocol,
 			Usage: "Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format " +
-				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API." +
-				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." +
+				"instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API. " +
+				"Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks. " +
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},

app/vmctl/influx.go

@@ -7,6 +7,8 @@ import (
 	"log"
 	"sync"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
@@ -52,6 +54,7 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
+	influxSeriesTotal.Add(len(series))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing series"), len(series))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -63,18 +66,18 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 	ip.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(ip.cc)
-	for i := 0; i < ip.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range ip.cc {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if err := ip.do(s); err != nil {
+					influxErrorsTotal.Inc()
 					errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
 					return
 				}
+				influxSeriesProcessed.Inc()
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import
@@ -83,6 +86,7 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("influx error: %s", infErr)
 		case vmErr := <-ip.im.Errors():
+			influxErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
 		case seriesCh <- s:
 		}
@@ -95,6 +99,7 @@ func (ip *influxProcessor) run(ctx context.Context) error {
 	// drain import errors channel
 	for vmErr := range ip.im.Errors() {
 		if vmErr.Err != nil {
+			influxErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, ip.isVerbose))
 		}
 	}
@@ -169,3 +174,9 @@ func (ip *influxProcessor) do(s *influx.Series) error {
 		}
 	}
 }
+
+var (
+	influxSeriesTotal     = metrics.NewCounter(`vmctl_influx_migration_series_total`)
+	influxSeriesProcessed = metrics.NewCounter(`vmctl_influx_migration_series_processed`)
+	influxErrorsTotal     = metrics.NewCounter(`vmctl_influx_migration_errors_total`)
+)

app/vmctl/limiter/limiter.go

@@ -4,6 +4,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 )
 
@@ -45,9 +47,16 @@ func (l *Limiter) Register(dataLen int) {
 			t := timerpool.Get(d)
 			<-t.C
 			timerpool.Put(t)
+			limiterThrottleEventsTotal.Inc()
 		}
 		l.budget += limit
 		l.deadline = time.Now().Add(time.Second)
 	}
 	l.budget -= int64(dataLen)
+	limiterBytesProcessed.Add(dataLen)
 }
+
+var (
+	limiterBytesProcessed      = metrics.NewCounter(`vmctl_limiter_bytes_processed_total`)
+	limiterThrottleEventsTotal = metrics.NewCounter(`vmctl_limiter_throttle_events_total`)
+)

app/vmctl/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"log"
 	"net/http"
@@ -19,7 +20,9 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
@@ -41,11 +44,20 @@ func main() {
 	ctx, cancelCtx := context.WithCancel(context.Background())
 	start := time.Now()
 	beforeFn := func(c *cli.Context) error {
+		flag.Parse()
+		logger.Init()
 		isSilent = c.Bool(globalSilent)
 		if c.Bool(globalDisableProgressBar) {
 			barpool.Disable(true)
 		}
 		netutil.EnableIPv6()
+		pushmetrics.InitWith(&pushmetrics.Config{
+			URLs:               c.StringSlice(globalPushMetricsURL),
+			Interval:           c.Duration(globalPushMetricsInterval),
+			ExtraLabels:        c.StringSlice(globalPushExtraLabels),
+			DisableCompression: c.Bool(globalPushDisableCompression),
+			Headers:            c.StringSlice(globalPushHeaders),
+		})
 		return nil
 	}
 	app := &cli.App{
@@ -451,6 +463,7 @@ func main() {
 		log.Fatalln(err)
 	}
 	log.Printf("Total time: %v", time.Since(start))
+	pushmetrics.StopAndPush()
 }
 
 func initConfigVM(c *cli.Context) (vm.Config, error) {

app/vmctl/native/client.go

@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
 )
 
@@ -36,12 +38,15 @@ type Response struct {
 
 // Explore finds metric names by provided filter from api/v1/label/__name__/values
 func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start, end time.Time) ([]string, error) {
+	startTime := time.Now()
+	exploreRequestsTotal.Inc()
 	url := fmt.Sprintf("%s/%s", c.Addr, nativeMetricNamesAddr)
 	if tenantID != "" {
 		url = fmt.Sprintf("%s/select/%s/prometheus/%s", c.Addr, tenantID, nativeMetricNamesAddr)
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
+		exploreRequestsErrorsTotal.Inc()
 		return nil, fmt.Errorf("cannot create request to %q: %s", url, err)
 	}
 
@@ -53,37 +58,53 @@ func (c *Client) Explore(ctx context.Context, f Filter, tenantID string, start,
 
 	resp, err := c.do(req, http.StatusOK)
 	if err != nil {
+		exploreRequestsErrorsTotal.Inc()
+		exploreDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("series request failed: %s", err)
 	}
 
 	var response Response
 	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		exploreRequestsErrorsTotal.Inc()
+		exploreDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("cannot decode series response: %s", err)
 	}
+	exploreDuration.UpdateDuration(startTime)
 	return response.MetricNames, resp.Body.Close()
 }
 
 // ImportPipe uses pipe reader in request to process data
 func (c *Client) ImportPipe(ctx context.Context, dstURL string, pr *io.PipeReader) error {
+	startTime := time.Now()
+	importRequestsTotal.Inc()
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, dstURL, pr)
 	if err != nil {
+		importRequestsErrorsTotal.Inc()
 		return fmt.Errorf("cannot create import request to %q: %s", c.Addr, err)
 	}
 
 	importResp, err := c.do(req, http.StatusNoContent)
 	if err != nil {
+		importRequestsErrorsTotal.Inc()
+		importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("import request failed: %s", err)
 	}
 	if err := importResp.Body.Close(); err != nil {
+		importRequestsErrorsTotal.Inc()
+		importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("cannot close import response body: %s", err)
 	}
+	importDuration.UpdateDuration(startTime)
 	return nil
 }
 
 // ExportPipe makes request by provided filter and return io.ReadCloser which can be used to get data
 func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadCloser, error) {
+	startTime := time.Now()
+	exportRequestsTotal.Inc()
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
+		exportRequestsErrorsTotal.Inc()
 		return nil, fmt.Errorf("cannot create request to %q: %s", c.Addr, err)
 	}
 
@@ -102,8 +123,11 @@ func (c *Client) ExportPipe(ctx context.Context, url string, f Filter) (io.ReadC
 
 	resp, err := c.do(req, http.StatusOK)
 	if err != nil {
+		exportRequestsErrorsTotal.Inc()
+		exportDuration.UpdateDuration(startTime)
 		return nil, fmt.Errorf("export request failed: %w", err)
 	}
+	exportDuration.UpdateDuration(startTime)
 	return resp.Body, nil
 }
 
@@ -162,3 +186,16 @@ func (c *Client) do(req *http.Request, expSC int) (*http.Response, error) {
 	}
 	return resp, err
 }
+
+var (
+	importRequestsTotal        = metrics.NewCounter(`vmctl_vm_native_requests_total{type="import"}`)
+	exportRequestsTotal        = metrics.NewCounter(`vmctl_vm_native_requests_total{type="export"}`)
+	exploreRequestsTotal       = metrics.NewCounter(`vmctl_vm_native_requests_total{type="explore"}`)
+	importRequestsErrorsTotal  = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="import"}`)
+	exportRequestsErrorsTotal  = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="export"}`)
+	exploreRequestsErrorsTotal = metrics.NewCounter(`vmctl_vm_native_request_errors_total{type="explore"}`)
+
+	importDuration  = metrics.NewHistogram(`vmctl_vm_native_import_duration_seconds`)
+	exportDuration  = metrics.NewHistogram(`vmctl_vm_native_export_duration_seconds`)
+	exploreDuration = metrics.NewHistogram(`vmctl_vm_native_explore_duration_seconds`)
+)

app/vmctl/opentsdb.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	vmetrics "github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/cheggaaa/pb/v3"
@@ -57,6 +59,7 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 	if !prompt(ctx, question) {
 		return nil
 	}
+
 	op.im.ResetStats()
 	var startTime int64
 	if op.oc.HardTS != 0 {
@@ -84,23 +87,24 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 		seriesCh := make(chan queryObj, op.otsdbcc)
 		errCh := make(chan error)
 		// we're going to make serieslist * queryRanges queries, so we should represent that in the progress bar
+		otsdbSeriesTotal.Add(len(serieslist) * queryRanges)
 		bar := pb.StartNew(len(serieslist) * queryRanges)
 		defer func(bar *pb.ProgressBar) {
 			bar.Finish()
 		}(bar)
 		var wg sync.WaitGroup
-		wg.Add(op.otsdbcc)
-		for i := 0; i < op.otsdbcc; i++ {
-			go func() {
-				defer wg.Done()
+		for range op.otsdbcc {
+			wg.Go(func() {
 				for s := range seriesCh {
 					if err := op.do(s); err != nil {
+						otsdbErrorsTotal.Inc()
 						errCh <- fmt.Errorf("couldn't retrieve series for %s : %s", metric, err)
 						return
 					}
+					otsdbSeriesProcessed.Inc()
 					bar.Increment()
 				}
-			}()
+			})
 		}
 		/*
 			Loop through all series for this metric, processing all retentions and time ranges
@@ -117,6 +121,7 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 					case otsdbErr := <-errCh:
 						return fmt.Errorf("opentsdb error: %s", otsdbErr)
 					case vmErr := <-op.im.Errors():
+						otsdbErrorsTotal.Inc()
 						return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
 					case seriesCh <- queryObj{
 						Tr: tr, StartTime: startTime,
@@ -141,6 +146,7 @@ func (op *otsdbProcessor) run(ctx context.Context) error {
 	op.im.Close()
 	for vmErr := range op.im.Errors() {
 		if vmErr.Err != nil {
+			otsdbErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, op.isVerbose))
 		}
 	}
@@ -171,3 +177,9 @@ func (op *otsdbProcessor) do(s queryObj) error {
 	}
 	return op.im.Input(&ts)
 }
+
+var (
+	otsdbSeriesTotal     = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_total`)
+	otsdbSeriesProcessed = vmetrics.NewCounter(`vmctl_opentsdb_migration_series_processed`)
+	otsdbErrorsTotal     = vmetrics.NewCounter(`vmctl_opentsdb_migration_errors_total`)
+)

app/vmctl/prometheus.go

@@ -11,6 +11,8 @@ import (
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
@@ -113,6 +115,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 }
 
 func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
+	promBlocksTotal.Add(len(blocks))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -124,18 +127,18 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	pp.im.ResetStats()
 
 	var wg sync.WaitGroup
-	wg.Add(pp.cc)
-	for i := 0; i < pp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range pp.cc {
+		wg.Go(func() {
 			for br := range blockReadersCh {
 				if err := pp.do(br); err != nil {
+					promErrorsTotal.Inc()
 					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
 					return
 				}
+				promBlocksProcessed.Inc()
 				bar.Increment()
 			}
-		}()
+		})
 	}
 	// any error breaks the import
 	for _, br := range blocks {
@@ -145,6 +148,7 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 			return fmt.Errorf("prometheus error: %s", promErr)
 		case vmErr := <-pp.im.Errors():
 			close(blockReadersCh)
+			promErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
 		case blockReadersCh <- br:
 		}
@@ -158,6 +162,7 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 	// drain import errors channel
 	for vmErr := range pp.im.Errors() {
 		if vmErr.Err != nil {
+			promErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, pp.isVerbose))
 		}
 	}
@@ -167,3 +172,9 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
 
 	return nil
 }
+
+var (
+	promBlocksTotal     = metrics.NewCounter(`vmctl_prometheus_migration_blocks_total`)
+	promBlocksProcessed = metrics.NewCounter(`vmctl_prometheus_migration_blocks_processed`)
+	promErrorsTotal     = metrics.NewCounter(`vmctl_prometheus_migration_errors_total`)
+)

app/vmctl/remoteread.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
@@ -51,6 +53,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 		return nil
 	}
 
+	remoteReadRangesTotal.Add(len(ranges))
 	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing ranges"), len(ranges))
 	if err := barpool.Start(); err != nil {
 		return err
@@ -66,18 +69,18 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	errCh := make(chan error)
 
 	var wg sync.WaitGroup
-	wg.Add(rrp.cc)
-	for i := 0; i < rrp.cc; i++ {
-		go func() {
-			defer wg.Done()
+	for range rrp.cc {
+		wg.Go(func() {
 			for r := range rangeC {
 				if err := rrp.do(ctx, r); err != nil {
+					remoteReadErrorsTotal.Inc()
 					errCh <- fmt.Errorf("request failed for: %s", err)
 					return
 				}
+				remoteReadRangesProcessed.Inc()
 				bar.Increment()
 			}
-		}()
+		})
 	}
 
 	for _, r := range ranges {
@@ -85,6 +88,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 		case infErr := <-errCh:
 			return fmt.Errorf("remote read error: %s", infErr)
 		case vmErr := <-rrp.dst.Errors():
+			remoteReadErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
 		case rangeC <- &remoteread.Filter{
 			StartTimestampMs: r[0].UnixMilli(),
@@ -100,6 +104,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 	// drain import errors channel
 	for vmErr := range rrp.dst.Errors() {
 		if vmErr.Err != nil {
+			remoteReadErrorsTotal.Inc()
 			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, rrp.isVerbose))
 		}
 	}
@@ -120,3 +125,9 @@ func (rrp *remoteReadProcessor) do(ctx context.Context, filter *remoteread.Filte
 		return nil
 	})
 }
+
+var (
+	remoteReadRangesTotal     = metrics.NewCounter(`vmctl_remote_read_migration_ranges_total`)
+	remoteReadRangesProcessed = metrics.NewCounter(`vmctl_remote_read_migration_ranges_processed`)
+	remoteReadErrorsTotal     = metrics.NewCounter(`vmctl_remote_read_migration_errors_total`)
+)

app/vmctl/vm/vm.go

@@ -12,6 +12,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
@@ -80,6 +82,12 @@ type Importer struct {
 
 	s       *stats
 	backoff *backoff.Backoff
+
+	importRequestsTotal       *metrics.Counter
+	importRequestsErrorsTotal *metrics.Counter
+	importSamplesTotal        *metrics.Counter
+	importBytesTotal          *metrics.Counter
+	importDuration            *metrics.Histogram
 }
 
 // ResetStats resets im stats.
@@ -147,6 +155,12 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
 		backoff:    cfg.Backoff,
+
+		importRequestsTotal:       metrics.GetOrCreateCounter(`vmctl_importer_requests_total`),
+		importRequestsErrorsTotal: metrics.GetOrCreateCounter(`vmctl_importer_request_errors_total`),
+		importSamplesTotal:        metrics.GetOrCreateCounter(`vmctl_importer_samples_total`),
+		importBytesTotal:          metrics.GetOrCreateCounter(`vmctl_importer_bytes_total`),
+		importDuration:            metrics.GetOrCreateHistogram(`vmctl_importer_request_duration_seconds`),
 	}
 	if err := im.Ping(); err != nil {
 		return nil, fmt.Errorf("ping to %q failed: %s", addr, err)
@@ -156,15 +170,13 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		cfg.BatchSize = 1e5
 	}
 
-	im.wg.Add(int(cfg.Concurrency))
-	for i := 0; i < int(cfg.Concurrency); i++ {
+	for i := range int(cfg.Concurrency) {
 		pbPrefix := fmt.Sprintf(`{{ green "VM worker %d:" }}`, i)
 		bar := barpool.AddWithTemplate(pbPrefix+pbTpl, 0)
 
-		go func(bar barpool.Bar) {
-			defer im.wg.Done()
+		im.wg.Go(func() {
 			im.startWorker(ctx, bar, cfg.BatchSize, cfg.SignificantFigures, cfg.RoundDigits)
-		}(bar)
+		})
 	}
 	im.ResetStats()
 	return im, nil
@@ -313,9 +325,13 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 		return nil
 	}
 
+	startTime := time.Now()
+	im.importRequestsTotal.Inc()
+
 	pr, pw := io.Pipe()
 	req, err := http.NewRequest(http.MethodPost, im.importPath, pr)
 	if err != nil {
+		im.importRequestsErrorsTotal.Inc()
 		return fmt.Errorf("cannot create request to %q: %s", im.addr, err)
 	}
 	if im.user != "" {
@@ -335,6 +351,7 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	if im.compress {
 		zw, err := gzip.NewWriterLevel(w, 1)
 		if err != nil {
+			im.importRequestsErrorsTotal.Inc()
 			return fmt.Errorf("unexpected error when creating gzip writer: %s", err)
 		}
 		w = zw
@@ -346,29 +363,39 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 	for _, ts := range tsBatch {
 		n, err := ts.write(bw)
 		if err != nil {
+			im.importRequestsErrorsTotal.Inc()
 			return fmt.Errorf("write err: %w", err)
 		}
 		totalBytes += n
 		totalSamples += len(ts.Values)
 	}
 	if err := bw.Flush(); err != nil {
+		im.importRequestsErrorsTotal.Inc()
 		return err
 	}
 	if closer, ok := w.(io.Closer); ok {
 		err := closer.Close()
 		if err != nil {
+			im.importRequestsErrorsTotal.Inc()
 			return err
 		}
 	}
 	if err := pw.Close(); err != nil {
+		im.importRequestsErrorsTotal.Inc()
 		return err
 	}
 
 	requestErr := <-errCh
 	if requestErr != nil {
+		im.importRequestsErrorsTotal.Inc()
+		im.importDuration.UpdateDuration(startTime)
 		return fmt.Errorf("import request error for %q: %w", im.addr, requestErr)
 	}
 
+	im.importSamplesTotal.Add(totalSamples)
+	im.importBytesTotal.Add(totalBytes)
+	im.importDuration.UpdateDuration(startTime)
+
 	im.s.Lock()
 	im.s.bytes += uint64(totalBytes)
 	im.s.samples += uint64(totalSamples)

app/vmctl/vm_native.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
@@ -82,13 +84,19 @@ func (p *vmNativeProcessor) run(ctx context.Context) error {
 		if !prompt(ctx, question) {
 			return nil
 		}
+		migrationTenantsTotal.Set(uint64(len(tenants)))
 	}
 
 	for _, tenantID := range tenants {
 		err := p.runBackfilling(ctx, tenantID, ranges)
 		if err != nil {
+			migrationErrorsTotal.Inc()
 			return fmt.Errorf("migration failed: %s", err)
 		}
+
+		if p.interCluster {
+			migrationTenantsProcessed.Inc()
+		}
 	}
 
 	log.Println("Import finished!")
@@ -156,6 +164,7 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
 	p.s.bytes += uint64(written)
 	p.s.requests++
 	p.s.Unlock()
+	migrationBytesTransferredTotal.AddInt64(written)
 
 	if err := pw.Close(); err != nil {
 		return err
@@ -199,7 +208,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var foundSeriesMsg string
 	var requestsToMake int
-	var metrics = map[string][][]time.Time{
+	var metricsMap = map[string][][]time.Time{
 		"": ranges,
 	}
 
@@ -211,11 +220,11 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	if !p.disablePerMetricRequests {
 		format = fmt.Sprintf(nativeWithBackoffTpl, barPrefix)
-		metrics, err = p.explore(ctx, p.src, tenantID, ranges)
+		metricsMap, err = p.explore(ctx, p.src, tenantID, ranges)
 		if err != nil {
 			return fmt.Errorf("failed to explore metric names: %s", err)
 		}
-		if len(metrics) == 0 {
+		if len(metricsMap) == 0 {
 			errMsg := "no metrics found"
 			if tenantID != "" {
 				errMsg = fmt.Sprintf("%s for tenant id: %s", errMsg, tenantID)
@@ -223,10 +232,14 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			log.Println(errMsg)
 			return nil
 		}
-		for _, m := range metrics {
+		for _, m := range metricsMap {
 			requestsToMake += len(m)
 		}
-		foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metrics), requestsToMake)
+		foundSeriesMsg = fmt.Sprintf("Found %d unique metric names to import. Total import/export requests to make %d", len(metricsMap), requestsToMake)
+
+		migrationMetricsTotal.Add(len(metricsMap))
+	} else {
+		requestsToMake = len(ranges)
 	}
 
 	if !p.interCluster {
@@ -240,6 +253,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		log.Print(foundSeriesMsg)
 	}
 
+	migrationRequestsPlanned.Add(requestsToMake)
 	bar := barpool.NewSingleProgress(format, requestsToMake)
 	bar.Start()
 	defer bar.Finish()
@@ -249,9 +263,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 
 	var wg sync.WaitGroup
 	for i := 0; i < p.cc; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			for f := range filterCh {
 				if !p.disablePerMetricRequests {
 					if err := p.do(ctx, f, srcURL, dstURL, nil); err != nil {
@@ -265,12 +277,13 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 						return
 					}
 				}
+				migrationRequestsCompleted.Inc()
 			}
-		}()
+		})
 	}
 
 	// any error breaks the import
-	for mName, mRanges := range metrics {
+	for mName, mRanges := range metricsMap {
 		match, err := buildMatchWithFilter(p.filter.Match, mName)
 		if err != nil {
 			logger.Errorf("failed to build filter %q for metric name %q: %s", p.filter.Match, mName, err)
@@ -290,6 +303,9 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 			}:
 			}
 		}
+		if !p.disablePerMetricRequests {
+			migrationMetricsProcessed.Inc()
+		}
 	}
 
 	close(filterCh)
@@ -398,3 +414,18 @@ func buildMatchWithFilter(filter string, metricName string) (string, error) {
 	match := "{" + strings.Join(filters, " or ") + "}"
 	return match, nil
 }
+
+var (
+	migrationMetricsTotal     = metrics.NewCounter(`vmctl_vm_native_migration_metrics_total`)
+	migrationMetricsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_metrics_processed`)
+
+	migrationRequestsPlanned   = metrics.NewCounter(`vmctl_vm_native_migration_requests_planned`)
+	migrationRequestsCompleted = metrics.NewCounter(`vmctl_vm_native_migration_requests_completed`)
+
+	migrationErrorsTotal = metrics.NewCounter(`vmctl_vm_native_migration_errors_total`)
+
+	migrationTenantsTotal     = metrics.NewCounter(`vmctl_vm_native_migration_tenants_total`)
+	migrationTenantsProcessed = metrics.NewCounter(`vmctl_vm_native_migration_tenants_processed`)
+
+	migrationBytesTransferredTotal = metrics.NewCounter(`vmctl_vm_native_migration_bytes_transferred_total`)
+)

app/vminsert/netstorage/netstorage.go

@@ -150,11 +150,7 @@ func (sn *storageNode) run(snb *storageNodesBucket, snIdx int) {
 		replicas = len(sns)
 	}
 
-	sn.readOnlyCheckerWG.Add(1)
-	go func() {
-		defer sn.readOnlyCheckerWG.Done()
-		sn.readOnlyChecker()
-	}()
+	sn.readOnlyCheckerWG.Go(sn.readOnlyChecker)
 	defer sn.readOnlyCheckerWG.Wait()
 
 	d := timeutil.AddJitterToDuration(time.Millisecond * 200)
@@ -622,11 +618,9 @@ func initStorageNodes(unsortedAddrs []string, rpcCall vminsertapi.RPCCall, hashS
 	}
 
 	for idx, sn := range sns {
-		wg.Add(1)
-		go func(sn *storageNode, idx int) {
+		wg.Go(func() {
 			sn.run(snb, idx)
-			wg.Done()
-		}(sn, idx)
+		})
 	}
 
 	return snb

app/vmselect/clusternative/vmselect.go

@@ -146,14 +146,13 @@ type workItem struct {
 
 func newBlockIterator(qt *querytracer.Tracer, denyPartialResponse bool, sq *storage.SearchQuery, deadline searchutil.Deadline) *blockIterator {
 	bi := getBlockIterator()
-	bi.wg.Add(1)
 	workers, processBlocks := netstorage.PrepareProcessRawBlocks(qt, denyPartialResponse, sq, deadline)
 	bi.workCh = make(chan workItem, workers)
 	bi.wis = slicesutil.SetLength(bi.wis, workers)
 	for i := range bi.wis {
 		bi.wis[i].doneCh = make(chan struct{})
 	}
-	go func() {
+	bi.wg.Go(func() {
 		_, err := processBlocks(func(mb []byte, workerID uint) error {
 			wi := bi.wis[workerID]
 			wi.rawMetricBlock = mb
@@ -163,8 +162,7 @@ func newBlockIterator(qt *querytracer.Tracer, denyPartialResponse bool, sq *stor
 		})
 		close(bi.workCh)
 		bi.err = err
-		bi.wg.Done()
-	}()
+	})
 	return bi
 }
 

app/vmselect/graphite/transform.go

@@ -3896,27 +3896,9 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 	seriesCh := make(chan *series, goroutines)
 	errCh := make(chan error, 1)
 	var wg sync.WaitGroup
-	wg.Add(goroutines)
-	go func() {
-		var err error
-		for {
-			s, e := nextSeries()
-			if e != nil || s == nil {
-				err = e
-				break
-			}
-			seriesCh <- s
-		}
-		close(seriesCh)
-		wg.Wait()
-		close(resultCh)
-		errCh <- err
-		close(errCh)
-	}()
 	var skipProcessing atomic.Bool
-	for i := 0; i < goroutines; i++ {
-		go func() {
-			defer wg.Done()
+	for range goroutines {
+		wg.Go(func() {
 			for s := range seriesCh {
 				if skipProcessing.Load() {
 					continue
@@ -3934,8 +3916,24 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 					}
 				}
 			}
-		}()
+		})
 	}
+	go func() {
+		var err error
+		for {
+			s, e := nextSeries()
+			if e != nil || s == nil {
+				err = e
+				break
+			}
+			seriesCh <- s
+		}
+		close(seriesCh)
+		wg.Wait()
+		close(resultCh)
+		errCh <- err
+		close(errCh)
+	}()
 	wrapper := func() (*series, error) {
 		r := <-resultCh
 		if r == nil {

app/vmselect/main.go

@@ -597,6 +597,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		}
 		w.Header().Add("Content-Type", "text/html; charset=utf-8")
 		fmt.Fprintf(w, "<h2>VictoriaMetrics cluster - vmselect</h2></br>")
+		fmt.Fprintf(w, "Version %s<br>", buildinfo.Version)
 		fmt.Fprintf(w, "See <a href='https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format'>docs</a></br>")
 		fmt.Fprintf(w, "Useful endpoints:</br>")
 		fmt.Fprintf(w, `<a href="vmui">Web UI</a><br>`)

app/vmselect/netstorage/netstorage.go

@@ -335,14 +335,12 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := range workChs {
-		wg.Add(1)
-		qtChild := qt.NewChild("worker #%d", i)
-		go func(workerID uint) {
-			timeseriesWorker(qtChild, workChs, workerID)
+	for workerID := range workChs {
+		qtChild := qt.NewChild("worker #%d", workerID)
+		wg.Go(func() {
+			timeseriesWorker(qtChild, workChs, uint(workerID))
 			qtChild.Done()
-			wg.Done()
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 
@@ -555,12 +553,10 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbfs []*tmpBlocksFile, t
 
 	// Start workers and wait until they finish the work.
 	var wg sync.WaitGroup
-	for i := 0; i < workers; i++ {
-		wg.Add(1)
-		go func(workerID uint) {
-			unpackWorker(workChs, workerID)
-			wg.Done()
-		}(uint(i))
+	for workerID := range workers {
+		wg.Go(func() {
+			unpackWorker(workChs, uint(workerID))
+		})
 	}
 	wg.Wait()
 
@@ -3232,14 +3228,12 @@ func initStorageNodes(addrs []string) *storageNodesBucket {
 		groupName, addr = netutil.ParseGroupAddr(addr)
 		group := groupsMap[groupName]
 
-		wg.Add(1)
-		go func(addr string) {
-			defer wg.Done()
+		wg.Go(func() {
 			sn := newStorageNode(ms, group, addr)
 			snsLock.Lock()
 			sns = append(sns, sn)
 			snsLock.Unlock()
-		}(addr)
+		})
 	}
 	wg.Wait()
 	metrics.RegisterSet(ms)

app/vmselect/promql/aggr_incremental_test.go

@@ -103,15 +103,13 @@ func testIncrementalParallelAggr(iafc *incrementalAggrFuncContext, tssSrc, tssEx
 	workersCount := netstorage.MaxWorkers()
 	tsCh := make(chan *timeseries)
 	var wg sync.WaitGroup
-	wg.Add(workersCount)
-	for i := 0; i < workersCount; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workersCount {
+		wg.Go(func() {
 			for ts := range tsCh {
 				runtime.Gosched() // allow other goroutines performing the work
-				iafc.updateTimeseries(ts, workerID)
+				iafc.updateTimeseries(ts, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	for _, ts := range tssSrc {
 		tsCh <- ts

app/vmselect/promql/eval.go

@@ -496,22 +496,18 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		var tssFirst []*timeseries
 		var errFirst error
 		qtFirst := qt.NewChild("expr1")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
 			qtFirst.Done()
-		}()
+		})
 
 		var tssSecond []*timeseries
 		var errSecond error
 		qtSecond := qt.NewChild("expr2")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
+		wg.Go(func() {
 			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
 			qtSecond.Done()
-		}()
+		})
 
 		wg.Wait()
 		if errFirst != nil {
@@ -729,17 +725,13 @@ func evalExprsInParallel(qt *querytracer.Tracer, ec *EvalConfig, es []metricsql.
 	qt.Printf("eval function args in parallel")
 	var wg sync.WaitGroup
 	for i, e := range es {
-		wg.Add(1)
 		qtChild := qt.NewChild("eval arg %d", i)
-		go func(e metricsql.Expr, i int) {
-			defer func() {
-				qtChild.Done()
-				wg.Done()
-			}()
+		wg.Go(func() {
+			defer qtChild.Done()
 			rv, err := evalExpr(qtChild, ec, e)
 			rvs[i] = rv
 			errs[i] = err
-		}(e, i)
+		})
 	}
 	wg.Wait()
 	for _, err := range errs {
@@ -1040,16 +1032,14 @@ func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, time
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(workers)
-	for i := 0; i < workers; i++ {
-		go func(workerID uint) {
-			defer wg.Done()
+	for workerID := range workers {
+		wg.Go(func() {
 			var tmpValues []float64
 			var tmpTimestamps []int64
 			for ts := range workChs[workerID] {
-				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, workerID)
+				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, uint(workerID))
 			}
-		}(uint(i))
+		})
 	}
 	wg.Wait()
 }
@@ -1771,6 +1761,7 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 		return nil, err
 	}
 	defer rml.Put(uint64(rollupMemorySize))
+	qs.addMemoryUsage(rollupMemorySize)
 	qt.Printf("the rollup evaluation needs an estimated %d bytes of RAM for %d series and %d points per series (summary %d points)",
 		rollupMemorySize, timeseriesLen, pointsPerSeries, rollupPoints)
 

app/vmselect/promql/exec.go

@@ -44,7 +44,7 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
 				return
 			}
 			at := ec.AuthTokens[0]
-			querystats.RegisterQuery(at.AccountID, at.ProjectID, q, ec.End-ec.Start, startTime)
+			querystats.RegisterQuery(at.AccountID, at.ProjectID, q, ec.End-ec.Start, startTime, ec.QueryStats.memoryUsage())
 		}()
 	}
 

app/vmselect/promql/query_stats.go

@@ -13,6 +13,8 @@ type QueryStats struct {
 	ExecutionDuration atomic.Pointer[time.Duration]
 	// SeriesFetched contains the number of series fetched from storage or cache.
 	SeriesFetched atomic.Int64
+	// MemoryUsage contains the estimated memory consumption of the query
+	MemoryUsage atomic.Int64
 
 	at *auth.Token
 
@@ -53,3 +55,17 @@ func (qs *QueryStats) addExecutionTimeMsec(startTime time.Time) {
 	d := time.Since(startTime)
 	qs.ExecutionDuration.Store(&d)
 }
+
+func (qs *QueryStats) addMemoryUsage(memoryUsage int64) {
+	if qs == nil {
+		return
+	}
+	qs.MemoryUsage.Store(memoryUsage)
+}
+
+func (qs *QueryStats) memoryUsage() int64 {
+	if qs == nil {
+		return 0
+	}
+	return qs.MemoryUsage.Load()
+}

app/vmselect/promql/rollup.go

@@ -535,7 +535,10 @@ type rollupFuncArg struct {
 	timestamps []int64
 
 	// Real value preceding values.
-	// Is populated if preceding value is within the rc.LookbackDelta.
+	// Is populated if the preceding sample falls within the rc.LookbackDelta range, or if rc.LookbackDelta is not set.
+	//
+	// It provides an additional check and value for rollup functions such as increase(), changes(),
+	// when the prevValue is NaN due to a gap or a small lookback window.
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -718,7 +721,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	// Extend dstValues in order to remove mallocs below.
 	dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
 
-	// Use step as the scrape interval for instant queries (when start == end).
+	// Set maxPrevInterval for subsequent rfa.prevValue calculations in rollupFunc:
+	// For instant queries, use rc.Step directly as maxPrevInterval.
+	// For range queries, rc.Step is typically too small to serve as the lookback window between two rollup points.
+	// Instead, estimate the scrape interval from raw sample timestamps (using the 0.6 quantile of the last 20 intervals)
+	// and slightly inflate the scrape interval to set maxPrevInterval, allowing for some tolerance to jitter.
 	maxPrevInterval := rc.Step
 	if rc.Start < rc.End {
 		scrapeInterval := getScrapeInterval(timestamps, rc.Step)
@@ -734,22 +741,21 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		}
 	}
 	window := rc.Window
+	// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
+	// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
+	// when the window doesn't cover at least two raw samples (this is what most users expect).
+	//
+	// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
+	// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 	if window <= 0 {
 		window = rc.Step
 		if rc.MayAdjustWindow && window < maxPrevInterval {
-			// Adjust lookbehind window only if it isn't set explicitly, e.g. rate(foo).
-			// In the case of missing lookbehind window it should be adjusted in order to return non-empty graph
-			// when the window doesn't cover at least two raw samples (this is what most users expect).
-			//
-			// If the user explicitly sets the lookbehind window to some fixed value, e.g. rate(foo[1s]),
-			// then it is expected he knows what he is doing. Do not adjust the lookbehind window then.
-			//
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3483
 			window = maxPrevInterval
 		}
+		// Artificial window cannot exceed explicit rc.LookbackDelta, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 		if rc.isDefaultRollup && rc.LookbackDelta > 0 && window > rc.LookbackDelta {
-			// Implicit window exceeds -search.maxStalenessInterval, so limit it to -search.maxStalenessInterval
-			// according to https://github.com/VictoriaMetrics/VictoriaMetrics/issues/784
 			window = rc.LookbackDelta
 		}
 	}
@@ -2112,9 +2118,15 @@ func rollupChanges(rfa *rollupFuncArg) float64 {
 		if len(values) == 0 {
 			return nan
 		}
-		prevValue = values[0]
-		values = values[1:]
-		n++
+		// Assume that the value didn't change during the current gap
+		// if realPrevValue exists.
+		if !math.IsNaN(rfa.realPrevValue) {
+			prevValue = rfa.realPrevValue
+		} else {
+			n++
+			prevValue = values[0]
+			values = values[1:]
+		}
 	}
 	for _, v := range values {
 		if v != prevValue {

app/vmselect/promql/rollup_test.go

@@ -232,6 +232,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
 	}
 	var rfa rollupFuncArg
 	rfa.prevValue = nan
+	rfa.realPrevValue = nan
 	rfa.prevTimestamp = 0
 	rfa.values = append(rfa.values, testValues...)
 	rfa.timestamps = append(rfa.timestamps, testTimestamps...)
@@ -1654,7 +1655,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1674,7 +1675,7 @@ func TestRollupDeltaWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1794,7 +1795,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1814,7 +1815,7 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
 		if samplesScanned != 7 {
-			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
 		valuesExpected := []float64{1, 0}
 		timestampsExpected := []int64{0, 45e3}
@@ -1888,3 +1889,126 @@ func TestRollupIncreasePureWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
+
+func TestRollupChangesWithStaleness(t *testing.T) {
+	// there is a gap between samples in the dataset below
+	timestamps := []int64{0, 15000, 30000, 70000}
+	values := []float64{1, 1, 1, 1}
+
+	// if step > gap, then changes will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 7 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta>0 then changes will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// there is a staleness marker between samples in the dataset below
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{1, 1, 1, decimal.StaleNaN, 1}
+
+	t.Run("staleness marker", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                40000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 1, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280
+	//
+	// When there are gaps between samples that exceed maxPrevInterval,
+	// either due to changes in the scrape interval or missing scrapes.
+	// For example, if the scrape interval was initially 30s and later changed to 10s,
+	// the auto-calculated scrape interval is 10s, with maxPrevInterval inflated to 15s.
+	//
+	// At t=30s:
+	// prevValue is NaN, as the last sample at t=0s is considered stale for t=30s given the maxPrevInterval.
+	// realPrevValue is 1, taken from t=0s, since LookbackDelta=0 ignores staleness.
+	// the result should be `changes(1, 1) -> 0` instead of `changes(1, NaN)`.
+	// At t=100s:
+	// preValue is also NaN, as the last sample at t=70s is considered stale for t=100s.
+	// realPrevValue is 1, taken from t=70s,
+	// result should be `changes(2, 1) -> 1`.
+	timestamps = []int64{0, 30000, 40000, 50000, 60000, 70000, 100000}
+	values = []float64{1, 1, 1, 1, 1, 1, 2}
+	t.Run("issue-10280", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupChanges,
+			Start:              0,
+			End:                100e3,
+			Step:               10e3,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, _ := rc.Do(nil, values, timestamps)
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3, 80e3, 90e3, 100e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}

app/vmselect/querystats/querystats.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 )
@@ -15,7 +16,8 @@ import (
 var (
 	lastQueriesCount = flag.Int("search.queryStats.lastQueriesCount", 20000, "Query stats for /api/v1/status/top_queries is tracked on this number of last queries. "+
 		"Zero value disables query stats tracking")
-	minQueryDuration = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
+	minQueryDuration    = flag.Duration("search.queryStats.minQueryDuration", time.Millisecond, "The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats")
+	minQueryMemoryUsage = flagutil.NewBytes("search.queryStats.minQueryMemoryUsage", 1024, "The minimum memory bytes consumption for queries to track in query stats at /api/v1/status/top_queries. Queries with lower memory bytes consumption are ignored in query stats")
 )
 
 var (
@@ -31,9 +33,9 @@ func Enabled() bool {
 // RegisterQuery registers the query on the given timeRangeMsecs, which has been started at startTime.
 //
 // RegisterQuery must be called when the query is finished.
-func RegisterQuery(accountID, projectID uint32, query string, timeRangeMsecs int64, startTime time.Time) {
+func RegisterQuery(accountID, projectID uint32, query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
 	initOnce.Do(initQueryStats)
-	qsTracker.registerQuery(accountID, projectID, query, timeRangeMsecs, startTime)
+	qsTracker.registerQuery(accountID, projectID, query, timeRangeMsecs, startTime, memoryUsage)
 }
 
 // RegisterQueryMultiTenant registers the query on the given timeRangeMsecs, which has been started at startTime.
@@ -75,6 +77,7 @@ type queryStatRecord struct {
 	registerTime  time.Time
 	duration      time.Duration
 	multiTenant   bool
+	memoryUsage   int64
 }
 
 type queryStatKey struct {
@@ -95,20 +98,21 @@ func initQueryStats() {
 	if recordsCount <= 0 {
 		recordsCount = 1
 	} else {
-		logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s",
-			*lastQueriesCount, *minQueryDuration)
+		logger.Infof("enabled query stats tracking at `/api/v1/status/top_queries` with -search.queryStats.lastQueriesCount=%d, -search.queryStats.minQueryDuration=%s, -search.queryStats.minQueryMemoryUsage=%s",
+			*lastQueriesCount, *minQueryDuration, minQueryMemoryUsage)
 	}
 	qsTracker = &queryStatsTracker{
 		a: make([]queryStatRecord, recordsCount),
 	}
 }
 
-func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, apFilter *accountProjectFilter, maxLifetime time.Duration) {
+func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, apiFilter *accountProjectFilter, maxLifetime time.Duration) {
 	fmt.Fprintf(w, `{"topN":"%d","maxLifetime":"%s",`, topN, maxLifetime)
 	fmt.Fprintf(w, `"search.queryStats.lastQueriesCount":%d,`, *lastQueriesCount)
 	fmt.Fprintf(w, `"search.queryStats.minQueryDuration":"%s",`, *minQueryDuration)
+	fmt.Fprintf(w, `"search.queryStats.minQueryMemoryUsage":"%s",`, minQueryMemoryUsage)
 	fmt.Fprintf(w, `"topByCount":[`)
-	topByCount := qst.getTopByCount(topN, apFilter, maxLifetime)
+	topByCount := qst.getTopByCount(topN, apiFilter, maxLifetime)
 	for i, r := range topByCount {
 		fmt.Fprintf(w, `{"accountID":%d,"projectID":%d,"query":%s,"timeRangeSeconds":%d,"count":%d,"multiTenant":%v}`, r.accountID, r.projectID, stringsutil.JSONString(r.query), r.timeRangeSecs, r.count, r.multiTenant)
 		if i+1 < len(topByCount) {
@@ -116,7 +120,7 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, apFilte
 		}
 	}
 	fmt.Fprintf(w, `],"topByAvgDuration":[`)
-	topByAvgDuration := qst.getTopByAvgDuration(topN, apFilter, maxLifetime)
+	topByAvgDuration := qst.getTopByAvgDuration(topN, apiFilter, maxLifetime)
 	for i, r := range topByAvgDuration {
 		fmt.Fprintf(w, `{"accountID":%d,"projectID":%d,"query":%s,"timeRangeSeconds":%d,"avgDurationSeconds":%.3f,"count":%d,"multiTenant": %v}`,
 			r.accountID, r.projectID, stringsutil.JSONString(r.query), r.timeRangeSecs, r.duration.Seconds(), r.count, r.multiTenant)
@@ -125,7 +129,7 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, apFilte
 		}
 	}
 	fmt.Fprintf(w, `],"topBySumDuration":[`)
-	topBySumDuration := qst.getTopBySumDuration(topN, apFilter, maxLifetime)
+	topBySumDuration := qst.getTopBySumDuration(topN, apiFilter, maxLifetime)
 	for i, r := range topBySumDuration {
 		fmt.Fprintf(w, `{"accountID":%d,"projectID":%d,"query":%s,"timeRangeSeconds":%d,"sumDurationSeconds":%.3f,"count":%d,"multiTenant":%v}`,
 			r.accountID, r.projectID, stringsutil.JSONString(r.query), r.timeRangeSecs, r.duration.Seconds(), r.count, r.multiTenant)
@@ -133,15 +137,28 @@ func (qst *queryStatsTracker) writeJSONQueryStats(w io.Writer, topN int, apFilte
 			fmt.Fprintf(w, `,`)
 		}
 	}
+
+	fmt.Fprintf(w, `],"topByAvgMemoryUsage":[`)
+	topByAvgMemoryConsumption := qst.getTopByAvgMemoryUsage(topN, apiFilter, maxLifetime)
+	for i, r := range topByAvgMemoryConsumption {
+		fmt.Fprintf(w, `{"query":%s,"timeRangeSeconds":%d,"avgMemoryBytes":%d,"count":%d}`, stringsutil.JSONString(r.query), r.timeRangeSecs, r.memoryUsage, r.count)
+		if i+1 < len(topByAvgMemoryConsumption) {
+			fmt.Fprintf(w, `,`)
+		}
+	}
+
 	fmt.Fprintf(w, `]}`)
 }
 
-func (qst *queryStatsTracker) registerQuery(accountID, projectID uint32, query string, timeRangeMsecs int64, startTime time.Time) {
+func (qst *queryStatsTracker) registerQuery(accountID, projectID uint32, query string, timeRangeMsecs int64, startTime time.Time, memoryUsage int64) {
 	registerTime := time.Now()
 	duration := registerTime.Sub(startTime)
 	if duration < *minQueryDuration {
 		return
 	}
+	if memoryUsage < int64(minQueryMemoryUsage.IntN()) {
+		return
+	}
 
 	qst.mu.Lock()
 	defer qst.mu.Unlock()
@@ -159,6 +176,7 @@ func (qst *queryStatsTracker) registerQuery(accountID, projectID uint32, query s
 	r.timeRangeSecs = timeRangeMsecs / 1000
 	r.registerTime = registerTime
 	r.duration = duration
+	r.memoryUsage = memoryUsage
 }
 
 func (qst *queryStatsTracker) registerQueryMultiTenant(query string, timeRangeMsecs int64, startTime time.Time) {
@@ -335,3 +353,47 @@ func (qst *queryStatsTracker) getTopBySumDuration(topN int, apFilter *accountPro
 	}
 	return a
 }
+
+type queryStatByMemory struct {
+	query         string
+	timeRangeSecs int64
+	memoryUsage   int64
+	count         int
+}
+
+func (qst *queryStatsTracker) getTopByAvgMemoryUsage(topN int, apFilter *accountProjectFilter, maxLifetime time.Duration) []queryStatByMemory {
+	currentTime := time.Now()
+	qst.mu.Lock()
+	type countSum struct {
+		count int
+		sum   int64
+	}
+	m := make(map[queryStatKey]countSum)
+	for _, r := range qst.a {
+		if r.matches(apFilter, currentTime, maxLifetime) {
+			k := r.key()
+			ks := m[k]
+			ks.count++
+			ks.sum += r.memoryUsage
+			m[k] = ks
+		}
+	}
+	qst.mu.Unlock()
+
+	var a []queryStatByMemory
+	for k, ks := range m {
+		a = append(a, queryStatByMemory{
+			query:         k.query,
+			timeRangeSecs: k.timeRangeSecs,
+			memoryUsage:   ks.sum / int64(ks.count),
+			count:         ks.count,
+		})
+	}
+	sort.Slice(a, func(i, j int) bool {
+		return a[i].memoryUsage > a[j].memoryUsage
+	})
+	if len(a) > topN {
+		a = a[:topN]
+	}
+	return a
+}

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-B6lol36n.js"></script>
+  <script type="module" crossorigin src="./assets/index-C4RD5Sxk.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-VQRcNK83.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-D7CzMv1O.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -387,9 +387,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 		return
 	}
 	snapshotsMaxAgeDur := snapshotsMaxAge.Duration()
-	staleSnapshotsRemoverWG.Add(1)
-	go func() {
-		defer staleSnapshotsRemoverWG.Done()
+	staleSnapshotsRemoverWG.Go(func() {
 		d := timeutil.AddJitterToDuration(time.Second * 11)
 		t := time.NewTicker(d)
 		defer t.Stop()
@@ -401,7 +399,7 @@ func initStaleSnapshotsRemover(strg *storage.Storage) {
 			}
 			strg.MustDeleteStaleSnapshots(snapshotsMaxAgeDur)
 		}
-	}()
+	})
 }
 
 func stopStaleSnapshotsRemover() {

app/vmui/Makefile

@@ -1,18 +1,23 @@
 # All these commands must run from repository root.
-
 copy-metricsql-docs:
 	cp docs/victoriametrics/MetricsQL.md app/vmui/packages/vmui/src/assets/MetricsQL.md
 
-vmui-package-base-image:
-	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
-
-vmui-build: copy-metricsql-docs vmui-package-base-image
+vmui-run-npm-command: vmui-package-base-image
 	docker run --rm \
 		--user $(shell id -u):$(shell id -g) \
 		--mount type=bind,src="$(shell pwd)/app/vmui",dst=/build \
 		-w /build/packages/vmui \
 		--entrypoint=/bin/bash \
-		vmui-builder-image -c "npm install && npm run build"
+		vmui-builder-image -c "[ \"$$VMUI_SKIP_INSTALL\" = \"true\" ] || npm ci; $(NPM_COMMAND)"
+
+vmui-install:
+	NPM_COMMAND="true" $(MAKE) vmui-run-npm-command
+
+vmui-package-base-image:
+	docker build -t vmui-builder-image -f app/vmui/Dockerfile-build ./app/vmui
+
+vmui-build: copy-metricsql-docs
+	NPM_COMMAND="npm run build" $(MAKE) vmui-run-npm-command
 
 vmui-release: vmui-build
 	docker build -t ${DOCKER_NAMESPACE}/vmui:latest -f app/vmui/Dockerfile-web ./app/vmui/packages/vmui
@@ -30,11 +35,11 @@ vmui-update: vmui-build
 vmui-install-dependencies:
 	cd app/vmui/packages/vmui && npm ci
 
-vmui-lint: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run lint
+vmui-lint:
+	NPM_COMMAND="npm run lint" $(MAKE) vmui-run-npm-command
 
-vmui-typecheck: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run typecheck
+vmui-typecheck:
+	NPM_COMMAND="npm run typecheck" $(MAKE) vmui-run-npm-command
 
-vmui-test: vmui-install-dependencies
-	cd app/vmui/packages/vmui && npm run test
+vmui-test:
+	NPM_COMMAND="npm run test" $(MAKE) vmui-run-npm-command

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -23,7 +23,6 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   totalSeries = 0,
   totalSeriesPrev = 0,
   totalSeriesAll = 0,
-  seriesCountByMetricName = [],
   metricNameStats,
   isPrometheus,
 }) => {
@@ -34,7 +33,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   const focusLabel = searchParams.get("focusLabel");
   const isMetric = /__name__/.test(match || "");
 
-  const progress = seriesCountByMetricName[0]?.value / totalSeriesAll * 100;
+  const progress = totalSeries / totalSeriesAll * 100;
   const diff = totalSeries - totalSeriesPrev;
   const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
 

app/vmui/packages/vmui/vite.config.ts

@@ -4,7 +4,7 @@ import { defineConfig, ProxyOptions } from "vite";
 import preact from "@preact/preset-vite";
 
 const getProxy = (): Record<string, ProxyOptions> | undefined => {
-  const playground = process.env.PLAYGROUND.toLowerCase();
+  const playground = process.env.PLAYGROUND?.toLowerCase();
 
   if (playground !== "true") {
     return undefined;

dashboards/vm/vmagent.json

@@ -627,7 +627,7 @@
           "refId": "A"
         }
       ],
-      "title": "Log errors (30m)",
+      "title": "Log messages (30m)",
       "type": "stat"
     },
     {
@@ -8413,4 +8413,4 @@
   "title": "VictoriaMetrics - vmagent (VM)",
   "uid": "G7Z9GzMGz_vm",
   "version": 1
-}
+}

dashboards/vm/vmauth.json

@@ -2573,13 +2573,13 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
               "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
           ],
-          "title": "Log errors",
+          "title": "Logging rate",
           "type": "timeseries"
         },
         {

dashboards/vmagent.json

@@ -626,7 +626,7 @@
           "refId": "A"
         }
       ],
-      "title": "Log errors (30m)",
+      "title": "Log messages (30m)",
       "type": "stat"
     },
     {
@@ -8412,4 +8412,4 @@
   "title": "VictoriaMetrics - vmagent",
   "uid": "G7Z9GzMGz",
   "version": 1
-}
+}

dashboards/vmauth.json

@@ -2572,13 +2572,13 @@
                 "uid": "$ds"
               },
               "editorMode": "code",
-              "expr": "sum(increase(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\", instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (level)",
               "legendFormat": "__auto",
               "range": true,
               "refId": "A"
             }
           ],
-          "title": "Log errors",
+          "title": "Logging rate",
           "type": "timeseries"
         },
         {

deployment/docker/rules/alerts-health.yml

@@ -71,15 +71,16 @@ groups:
             the service could work unreliably with delays in processing.
 
       - alert: TooManyLogs
-        expr: sum(increase(vm_log_messages_total{level="error"}[5m])) without (app_version, location) > 0
+        expr: sum(increase(vm_log_messages_total{level!="info"}[5m])) without (app_version, location, is_printed) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
+          summary: "Too many logs are generated for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
           description: >
-            Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.
-            Worth to check logs for specific error messages.
+            The job \"{{ $labels.job }}\" ({{ $labels.instance }}) generated {{ $value }} log messages with the level higher than info for the last 5 minutes.
+            Check the logs for the given target. Check also the \"location\" label at the vm_log_messages_total metric if -loggerLevel command-line flag is set to value other than INFO.
+            This label contains code locations responsible for generating log messages suppressed by -loggerLevel.
 
       - alert: TooManyTSIDMisses
         expr: increase(vm_missing_tsids_for_metric_id_total[5m]) > 0

deployment/docker/rules/alerts-vmagent.yml

@@ -169,4 +169,36 @@ groups:
         annotations:
           summary: "Deduplication \"{{ $labels.job }}\" (instance {{ $labels.instance }}) can't be finished within configured deduplication interval."
           description: "Deduplication process can't keep up with the load and might produce incorrect results. Check docs https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication and logs for more details.
-            Possible solutions: increase deduplication interval; deduplicate smaller number of series; reduce samples' ingestion rate."
+            Possible solutions: increase deduplication interval; deduplicate smaller number of series; reduce samples' ingestion rate."
+
+      - alert: PersistentQueueRunsOutOfSpaceIn12Hours
+        expr: |
+          (
+            (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) /
+            (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0)
+          ) * 
+          on(job, instance, path) group_left(url)
+          (vmagent_remotewrite_pending_data_bytes * 0 + 1) < 12 * 3600 >0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 12 hours."
+          description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
+            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
+
+      - alert: PersistentQueueRunsOutOfSpaceIn4Hours
+        expr: |
+          (
+            (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) /
+            (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0)
+          ) * 
+          on(job, instance, path) group_left(url)
+          (vmagent_remotewrite_pending_data_bytes * 0 + 1) < 4 * 3600 >0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours."
+          description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
+            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."

deployment/docker/rules/alerts-vmauth.yml

@@ -13,18 +13,28 @@ groups:
         annotations:
           dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
           summary: "vmauth ({{ $labels.instance }}) reached concurrent requests limit"
-          description: "Possible solutions: increase the limit with flag: -maxConcurrentRequests, 
+          description: "Possible solutions: increase -maxQueueDuration flag value, increase -maxConcurrentRequests flag value,
           deploy additional vmauth replicas, check requests latency at backend service. 
           See more details at https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting"
       - alert: UserConcurrentRequestsLimitReached
-        expr: sum(increase(vmauth_user_concurrent_requests_limit_reached_total[1m])) by (username) > 0
+        expr: sum(increase(vmauth_user_concurrent_requests_limit_reached_total[1m])) by (instance, username) > 0
         for: 3m
         labels:
           severity: warning
         annotations:
           dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
-          summary: "vmauth has reached concurrent requests limit for username {{ $labels.username }}"
-          description: "Possible solutions: increase limit with flag: -maxConcurrentPerUserRequests, 
+          summary: "vmauth ({{ $labels.instance }}) has reached concurrent requests limit for username {{ $labels.username }}"
+          description: "Possible solutions: increase -maxQueueDuration flag value, increase -maxConcurrentPerUserRequests flag value,
+          deploy additional vmauth replicas, check requests latency at backend service."
+      - alert: UnauthorizedUserConcurrentRequestsLimitReached
+        expr: sum(increase(vmauth_unauthorized_user_concurrent_requests_limit_reached_total[1m])) by (instance) > 0
+        for: 3m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
+          summary: "vmauth ({{ $labels.instance }}) has reached concurrent requests limit for unauthorized user"
+          description: "Possible solutions: increase -maxQueueDuration flag value, increase -maxConcurrentPerUserRequests flag value,
           deploy additional vmauth replicas, check requests latency at backend service."
       - alert: UnauthorizedUserRequestErrors
         expr: increase(vmauth_unauthorized_user_request_errors_total[5m]) > 0

docs/Makefile

@@ -11,7 +11,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
 endef
 export FLAGS_HEADER
@@ -241,6 +241,49 @@ docs-update-vmstorage-flags:
 	sed -i '/The maximum number of concurrent vmselect requests the vmstorage can process at./ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmstorage_common_flags.md
 	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmstorage_common_flags.md
 
+docs-update-vmctl-flags:
+	(cd /tmp/vm-opensource-single-node && make vmctl)
+	(cd /tmp/vm-opensource-single-node && ./bin/vmctl -help 2>&1) > /tmp/vm-opensource-single-node/vmctl_flags_tmp.md
+	(cd /tmp/vm-opensource-single-node && ./bin/vmctl opentsdb -help 2>&1) > /tmp/vm-opensource-single-node/vmctl_opentsdb_flags_tmp.md
+	(cd /tmp/vm-opensource-single-node && ./bin/vmctl influx -help 2>&1) > /tmp/vm-opensource-single-node/vmctl_influx_flags_tmp.md
+	(cd /tmp/vm-opensource-single-node && ./bin/vmctl remote-read -help 2>&1) > /tmp/vm-opensource-single-node/vmctl_remote-read_flags_tmp.md
+	(cd /tmp/vm-opensource-single-node && ./bin/vmctl prometheus -help 2>&1) > /tmp/vm-opensource-single-node/vmctl_prometheus_flags_tmp.md
+	(cd /tmp/vm-opensource-single-node && ./bin/vmctl vm-native -help 2>&1) > /tmp/vm-opensource-single-node/vmctl_vm-native_flags_tmp.md
+
+	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_flags.md
+	cat /tmp/vm-opensource-single-node/vmctl_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_flags.md
+	printf -- '```\n' >> docs/victoriametrics/vmctl/vmctl_flags.md
+
+	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_opentsdb_flags.md
+	cat /tmp/vm-opensource-single-node/vmctl_opentsdb_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_opentsdb_flags.md
+	printf -- '```\n' >> docs/victoriametrics/vmctl/vmctl_opentsdb_flags.md
+
+	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_influx_flags.md
+	cat /tmp/vm-opensource-single-node/vmctl_influx_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_influx_flags.md
+	printf -- '```\n' >> docs/victoriametrics/vmctl/vmctl_influx_flags.md
+
+	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_remote-read_flags.md
+	cat /tmp/vm-opensource-single-node/vmctl_remote-read_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_remote-read_flags.md
+	printf -- '```\n' >> docs/victoriametrics/vmctl/vmctl_remote-read_flags.md
+
+	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_prometheus_flags.md
+	cat /tmp/vm-opensource-single-node/vmctl_prometheus_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_prometheus_flags.md
+	printf -- '```\n' >> docs/victoriametrics/vmctl/vmctl_prometheus_flags.md
+
+	echo "$$FLAGS_HEADER" > docs/victoriametrics/vmctl/vmctl_vm-native_flags.md
+	cat /tmp/vm-opensource-single-node/vmctl_vm-native_flags_tmp.md >> docs/victoriametrics/vmctl/vmctl_vm-native_flags.md
+	printf -- '```\n' >> docs/victoriametrics/vmctl/vmctl_vm-native_flags.md
+
+	# remove Total time line from all vmctl flag files to reduce diffs noise
+	sed -i '/Total time:/d' docs/victoriametrics/vmctl/vmctl_flags.md
+	sed -i '/Total time:/d' docs/victoriametrics/vmctl/vmctl_opentsdb_flags.md
+	sed -i '/Total time:/d' docs/victoriametrics/vmctl/vmctl_influx_flags.md
+	sed -i '/Total time:/d' docs/victoriametrics/vmctl/vmctl_remote-read_flags.md
+	sed -i '/Total time:/d' docs/victoriametrics/vmctl/vmctl_prometheus_flags.md
+	sed -i '/Total time:/d' docs/victoriametrics/vmctl/vmctl_vm-native_flags.md
+
+	# remove Version line and the actual version line from vmctl_flags.md to reduce diffs noise
+	sed -i '/^VERSION:/,+1d' docs/victoriametrics/vmctl/vmctl_flags.md
 
 # docs-update-flags updates flags in the documentation using the actual binaries compiled
 # from the latest enterprise-single-node and enterprise-cluster branches (hardcoded for now).
@@ -280,6 +323,7 @@ docs-update-flags:
 	git worktree remove /tmp/vm-opensource-single-node || true
 	git worktree add /tmp/vm-opensource-single-node opensource/master
 
+	make docs-update-vmctl-flags
 	make docs-update-vmsingle-flags
 	make docs-update-vmalert-flags
 	make docs-update-vmauth-flags

docs/guides/examples/guide-vmcluster-vmagent-values.yaml

@@ -37,12 +37,6 @@ config:
       relabel_configs:
         - action: labelmap
           regex: __meta_kubernetes_node_label_(.+)
-        - target_label: __address__
-          replacement: kubernetes.default.svc:443
-        - source_labels: [__meta_kubernetes_node_name]
-          regex: (.+)
-          target_label: __metrics_path__
-          replacement: /api/v1/nodes/$1/proxy/metrics
     - job_name: "kubernetes-nodes-cadvisor"
       scheme: https
       tls_config:
@@ -51,15 +45,12 @@ config:
       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       kubernetes_sd_configs:
         - role: node
+      metrics_path: /metrics/cadvisor
       relabel_configs:
         - action: labelmap
           regex: __meta_kubernetes_node_label_(.+)
-        - target_label: __address__
-          replacement: kubernetes.default.svc:443
-        - source_labels: [__meta_kubernetes_node_name]
-          regex: (.+)
-          target_label: __metrics_path__
-          replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
+        - source_labels: [__metrics_path__]
+          target_label: metrics_path
       metric_relabel_configs:
         - action: replace
           source_labels: [pod]

docs/guides/examples/guide-vmsingle-values.yaml

@@ -37,12 +37,6 @@ server:
           relabel_configs:
             - action: labelmap
               regex: __meta_kubernetes_node_label_(.+)
-            - target_label: __address__
-              replacement: kubernetes.default.svc:443
-            - source_labels: [ __meta_kubernetes_node_name ]
-              regex: (.+)
-              target_label: __metrics_path__
-              replacement: /api/v1/nodes/$1/proxy/metrics
         - job_name: "kubernetes-nodes-cadvisor"
           scheme: https
           tls_config:
@@ -51,15 +45,12 @@ server:
           bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
           kubernetes_sd_configs:
             - role: node
+          metrics_path: /metrics/cadvisor
           relabel_configs:
             - action: labelmap
               regex: __meta_kubernetes_node_label_(.+)
-            - target_label: __address__
-              replacement: kubernetes.default.svc:443
-            - source_labels: [ __meta_kubernetes_node_name ]
-              regex: (.+)
-              target_label: __metrics_path__
-              replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
+            - source_labels: [__metrics_path__]
+              target_label: metrics_path
           metric_relabel_configs:
             - action: replace
               source_labels: [pod]
@@ -78,4 +69,4 @@ server:
               source_labels: [id]
               regex: '^/system\.slice/(.+)\.service$'
               target_label: systemd_service_name
-              replacement: '${1}'
+              replacement: '${1}'

docs/guides/k8s-ha-monitoring-via-vm-cluster/README.md

@@ -187,12 +187,6 @@ scrape_configs:
       relabel_configs:
         - action: labelmap
           regex: __meta_kubernetes_node_label_(.+)
-        - target_label: __address__
-          replacement: kubernetes.default.svc:443
-        - source_labels: [__meta_kubernetes_node_name]
-          regex: (.+)
-          target_label: __metrics_path__
-          replacement: /api/v1/nodes/$1/proxy/metrics
     - job_name: "kubernetes-nodes-cadvisor"
       scheme: https
       tls_config:
@@ -201,15 +195,12 @@ scrape_configs:
       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       kubernetes_sd_configs:
         - role: node
+      metrics_path: /metrics/cadvisor
       relabel_configs:
         - action: labelmap
           regex: __meta_kubernetes_node_label_(.+)
-        - target_label: __address__
-          replacement: kubernetes.default.svc:443
-        - source_labels: [__meta_kubernetes_node_name]
-          regex: (.+)
-          target_label: __metrics_path__
-          replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
+        - source_labels: [__metrics_path__]
+          target_label: metrics_path
       metric_relabel_configs:
         - action: replace
           source_labels: [pod]

docs/guides/k8s-monitoring-via-vm-cluster/README.md

@@ -213,12 +213,6 @@ config:
       relabel_configs:
         - action: labelmap
           regex: __meta_kubernetes_node_label_(.+)
-        - target_label: __address__
-          replacement: kubernetes.default.svc:443
-        - source_labels: [__meta_kubernetes_node_name]
-          regex: (.+)
-          target_label: __metrics_path__
-          replacement: /api/v1/nodes/$1/proxy/metrics
     - job_name: "kubernetes-nodes-cadvisor"
       scheme: https
       tls_config:
@@ -227,15 +221,12 @@ config:
       bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       kubernetes_sd_configs:
         - role: node
+      metrics_path: /metrics/cadvisor
       relabel_configs:
         - action: labelmap
           regex: __meta_kubernetes_node_label_(.+)
-        - target_label: __address__
-          replacement: kubernetes.default.svc:443
-        - source_labels: [__meta_kubernetes_node_name]
-          regex: (.+)
-          target_label: __metrics_path__
-          replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
+        - source_labels: [__metrics_path__]
+          target_label: metrics_path
       metric_relabel_configs:
         - action: replace
           source_labels: [pod]

docs/guides/k8s-monitoring-via-vm-single/README.md

@@ -114,12 +114,6 @@ server:
           relabel_configs:
             - action: labelmap
               regex: __meta_kubernetes_node_label_(.+)
-            - target_label: __address__
-              replacement: kubernetes.default.svc:443
-            - source_labels: [ __meta_kubernetes_node_name ]
-              regex: (.+)
-              target_label: __metrics_path__
-              replacement: /api/v1/nodes/$1/proxy/metrics
         - job_name: "kubernetes-nodes-cadvisor"
           scheme: https
           tls_config:
@@ -128,15 +122,12 @@ server:
           bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
           kubernetes_sd_configs:
             - role: node
+          metrics_path: /metrics/cadvisor
           relabel_configs:
             - action: labelmap
               regex: __meta_kubernetes_node_label_(.+)
-            - target_label: __address__
-              replacement: kubernetes.default.svc:443
-            - source_labels: [ __meta_kubernetes_node_name ]
-              regex: (.+)
-              target_label: __metrics_path__
-              replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
+            - source_labels: [__metrics_path__]
+              target_label: metrics_path
           metric_relabel_configs:
             - action: replace
               source_labels: [pod]

docs/victoriametrics/FAQ.md

@@ -487,7 +487,7 @@ The query engine may behave differently for some functions. Please see [this art
 
 ## If downsampling and deduplication are enabled how will this work?
 
-[Deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is a special case of zero-offset [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling). So, if both downsampling and deduplication are enabled, then deduplication is replaced by zero-offset downsampling
+[Deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is a special case of zero-offset [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling). So, if both downsampling and deduplication are enabled, then deduplication is replaced by zero-offset downsampling.
 
 ## How to upgrade or downgrade VictoriaMetrics without downtime?
 

docs/victoriametrics/README.md

@@ -1347,10 +1347,6 @@ If multiple raw samples have **the same timestamp** on the given `-dedup.minScra
 then the sample with **the biggest value** is kept.
 Numerical values are preferred over [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers).
 
-[Prometheus staleness markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers) are processed as any other value during de-duplication.
-If raw sample with the biggest timestamp on `-dedup.minScrapeInterval` contains a stale marker, then it is kept after the deduplication.
-This allows properly preserving staleness markers during the de-duplication.
-
 Please note, [labels](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels) of raw samples should be identical
 in order to be deduplicated. For example, this is why [HA pair of vmagents](https://docs.victoriametrics.com/victoriametrics/vmagent/#high-availability)
 needs to be identically configured.

docs/victoriametrics/Troubleshooting.md

@@ -493,3 +493,19 @@ for VictoriaMetrics components will notify about issues and provide recommendati
 
 Internally, we heavily rely both on dashboards and alerts, and constantly improve them.
 It is important to stay up to date with such changes.
+
+
+## Filesystem read corruption on ZFS
+
+On some ZFS filesystems, mixing reads from memory-mapped files (`mmap`) with usage of the `mincore()` syscall can trigger a bug in the ZFS in-memory cache (ARC), potentially resulting in **data read corruption** in VictoriaMetrics processes. This scenario has been observed when VictoriaMetrics instances access data directories on ZFS.
+
+Symptoms:
+- Unexpected read errors when accessing data on ZFS.
+- Corrupted or inconsistent query results.
+- Crashes or panics in storage/query components when reading from ZFS.
+
+It could be mitigated with `--fs.disableMincore` flag:
+
+```text
+./bin/victoria-metrics --storageDataPath /path/to/zfs/data --fs.disableMincore
+```

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,7 +26,32 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+## [v1.135.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.135.0)
+
+Released at 2026-01-30
+
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): improved scrape size display. Sizes below 1024 bytes are now shown in `B`, and larger sizes are shown as whole `KiB` (rounded up). This prevents confusion where values like 123.456 KiB were interpreted as 123456 KiB, while the actual size was only 123 KiB. See [#10307](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10307).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow buffering request bodies before proxying them to backends. This reduces load on backends when processing requests from slow clients such as IoT devices connected to `vmauth` via slow networks. See [#10309](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10309) and [request body buffering docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering).
+* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): allow completely disabling scheduled backups by using `-disableScheduledBackups` command-line flag. This is useful to run `vmbackupmanager` only for on-demand backups and restores triggered via API. See [#10364](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10364).
+* FEATURE: [monitoring](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring): take into account all the generated log messages at `vm_log_messages_total` metric, including suppressed logs if the `-loggerLevel` command-line flag is set to values other than `INFO`. Add `is_printed` label to the `vm_log_messages_total` metric in order to understand whether the log at the given code `location` was suppressed or not. This simplifies troubleshooting of VictoriaMetrics components when logs aren't available. See [#10304](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10304).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): support configuring different `-remoteWrite.queues` per remoteWrite url. This allows setting `-remoteWrite.queues=1` for backends that do not support out-of-order ingestion (e.g. Mimir), while keeping higher queue counts for other backends such as VictoriaMetrics. Previously, this required running multiple vmagent instances with different queue settings. See [#10270](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10270).
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add new alerting rules `PersistentQueueRunsOutOfSpaceIn12Hours` and `PersistentQueueRunsOutOfSpaceIn4Hours` for `vmagent` persistent queue capacity. These alerts help users to take proactive actions before `vmagent` starts dropping metrics due to insufficient persistent queue space. See [#10193](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10193)
+* FEATURE: All VictoriaMetrics components: add build version information to the home page for consistency with other projects. See [#10249](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10249).
+* FEATURE: all VictoriaMetrics components: add flag `fs.disableMincore`, which allows to disable `mincore` syscall. See [#10327](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose topN average memory bytes consumption queries in `/api/v1/status/top_queries`. It can help users to find queries that consume a lot of memory and potentially cause OOM. See [#9330](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9330).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add metrics for tracking the migration progress. See [vmctl - monitoring the migration process](https://docs.victoriametrics.com/victoriametrics/vmctl/#monitoring-the-migration-process) and [#10276](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10276).
+
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): stop backend health checks for URL prefixes defined in `url_map` during configuration reloads. Previously, stale backends kept being health-checked and produced repeated warning logs after reloads. See [#10334](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10334).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly return [/api/v1/status/tsdb](https://docs.victoriametrics.com/victoriametrics/#tsdb-stats) response for time range outside [partition index](https://docs.victoriametrics.com/victoriametrics/#indexdb). See [#10315](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10315).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly report `*_requests_total`, `*_misses_total`, `*_resets_total`, `*_syncs_total`, `*_rotations_total` metrics for `indexdb/tagFiltersToMetricIDs`, `indexdb/metricID` and `indexdb/date_metricID` caches. See [#10275](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10275).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly return tenants results for `/admin/tenants` when `start` or `end` are specified. See [#10312](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10312). Thanks to @Defined2014 for the contribution.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix "Percentage from total" calculation on the Cardinality Explorer page when multiple metrics match the filter. See [#10323](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10323). Thanks to @PleasingFungus for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): apply `-promscrape.maxScrapeSize` check to decompressed data instead of compressed data. See [#9481](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9481).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): disallow setting the `-notifier.url` command-line flag to a null value. See [#10355](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10355).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix [alert restore](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerts-state-on-restarts) when a group contains many rules and is slow to complete evaluation. Previously, the restore process might not retrieve the correct previous alert state. See [#10335](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10335).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): do not skip sending alert notifications to `-notifier.url` if remote write requests to `-remoteWrite.url` fail. See [#10376](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10376).
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): fix `changes()` function when gaps between samples exceed the lookbehind window. Previously, it could yield a non-zero value even when the sample value remained unchanged. See [#10280](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): previously ingestion could hit lock contention that triggered frequent context switches and storage connection saturation spikes; now the contention is removed to keep ingestion steady. See [#10367](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10367).
 
 ## [v1.134.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.134.0)
 
@@ -59,6 +84,10 @@ Released at 2026-01-16
 Released at 2026-01-02
 
 **Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Upgrading to per-partition index requires registering all active time series. Expect slow down of data ingestion and queries during upgrade roll-out. This is a one-time operation. Additionally, for users with retention periods shorter than 1 month the disk usage may increase.
+**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Running this version in deployments with big datasets may cause high CPU utilization. See [10297](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10297).
+**Update Note 3:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): lock contention in the ingestion path may cause frequent context switches and storage connection saturation spikes. See [10367](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10367). Addressed in `v1.135.0`.
+**Update Note 4:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): TSDB status may be empty if the partition index does not have records for the requested date. See [10315](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10315). Addressed in `v1.135.0`.
+**Update Note 5:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `indexdb/tagFiltersToMetricIDs`, `indexdb/metricID` and `indexdb/date_metricID` cache metrics are not reported properly. See [10275](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10275). Addressed in `v1.135.0`.
 
 * SECURITY: upgrade base docker image (Alpine) from 3.22.2 to 3.23.2. See [Alpine 3.23.2 release notes](https://www.alpinelinux.org/posts/Alpine-3.23.2-released.html).
 

docs/victoriametrics/scrape_config_examples.md

@@ -322,16 +322,11 @@ scrape_configs:
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   tls_config:
     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-
+  metrics_path: /metrics/cadvisor
   relabel_configs:
-    # Cadvisor metrics are better to scrape from Kubernetes API server proxy.
     # There is no need to add container, pod and node labels to the scraped metrics,
     # since cadvisor adds these labels on itself.
     #
-  - source_labels: [__meta_kubernetes_node_name]
-    target_label: __address__
-    regex: '(.+)'
-    replacement: https://kubernetes.default.svc/api/v1/nodes/$1/proxy/metrics/cadvisor
   - source_labels: [__meta_kubernetes_node_name]
     target_label: instance
 ```

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
 
 victoria-metrics is a time series database and monitoring solution.
@@ -68,6 +68,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*
      Flag value can be read from the given file when using -forceMergeAuthKey=file:///abs/path/to/file or -forceMergeAuthKey=file://./relative/path/to/file.
      Flag value can be read from the given http/https url when using -forceMergeAuthKey=http://host/path or -forceMergeAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int
@@ -318,7 +320,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      The maximum size of http response headers from Prometheus scrape targets
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 4096)
   -promscrape.maxScrapeSize size
-     The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected. See also max_scrape_size option at https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs
+     The maximum size of uncompressed scrape response in bytes to process from Prometheus targets. Bigger uncompressed responses are rejected. See also max_scrape_size option at https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16777216)
   -promscrape.minResponseSizeForStreamParse size
      The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/victoriametrics/vmagent/#stream-parsing-mode
@@ -373,7 +375,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path
   -retentionPeriod value
      Data with timestamps outside the retentionPeriod is automatically deleted. The minimum retentionPeriod is 24h or 1d. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter
-     The following optional suffixes are supported: s (second), h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1M)
+     The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 1M)
   -retentionTimezoneOffset duration
      The offset for performing indexdb rotation. If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)
   -search.cacheTimestampOffset duration
@@ -484,6 +486,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Query stats for /api/v1/status/top_queries is tracked on this number of last queries. Zero value disables query stats tracking (default 20000)
   -search.queryStats.minQueryDuration duration
      The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats (default 1ms)
+  -search.queryStats.minQueryMemoryUsage size
+     The minimum memory bytes consumption for queries to track in query stats at /api/v1/status/top_queries. Queries with lower memory bytes consumption are ignored in query stats
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1024)
   -search.resetCacheAuthKey value
      Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It could be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -search.resetCacheAuthKey=file:///abs/path/to/file or -search.resetCacheAuthKey=file://./relative/path/to/file.
@@ -514,7 +519,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Deprecated: this flag does nothing
   -snapshotsMaxAge value
      Automatically delete snapshots older than -snapshotsMaxAge if it is set to non-zero duration. Make sure that backup process has enough time to finish the backup before the corresponding snapshot is automatically deleted
-     The following optional suffixes are supported: s (second), h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 3d)
+     The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 3d)
   -sortLabels
      Whether to sort labels for incoming samples before writing them to storage. This may be needed for reducing memory usage at storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. Enabled sorting for labels can slow down ingestion performance a bit
   -storage.cacheSizeIndexDBDataBlocks size

docs/victoriametrics/victoria_metrics_enterprise_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
   -downsampling.period array
      Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs to leave a single sample per 10 minutes for samples older than 30 days. The 'offset' must be a multiple of 'interval', and when setting multiple downsampling periods for a single filter, those periods must also be multiples of each other. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/

docs/victoriametrics/vmagent_common_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
 
 vmagent collects metrics data via popular data ingestion protocols and routes it to VictoriaMetrics.
@@ -48,6 +48,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file.
      Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int
@@ -284,7 +286,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      The maximum size of http response headers from Prometheus scrape targets
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 4096)
   -promscrape.maxScrapeSize size
-     The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected. See also max_scrape_size option at https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs
+     The maximum size of uncompressed scrape response in bytes to process from Prometheus targets. Bigger uncompressed responses are rejected. See also max_scrape_size option at https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16777216)
   -promscrape.minResponseSizeForStreamParse size
      The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/victoriametrics/vmagent/#stream-parsing-mode
@@ -457,8 +459,10 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Optional proxy URL for writing data to the corresponding -remoteWrite.url. Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-  -remoteWrite.queues int
+  -remoteWrite.queues array
      The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues isn't enough for sending high volume of collected data to remote storage. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage (default 2*cgroup.AvailableCPUs())
+     Supports array of values separated by comma or specified via multiple flags.
+     Empty values are set to default value.
   -remoteWrite.rateLimit array
      Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data is sent after temporary unavailability of the remote storage. See also -maxIngestionRate (default 0)
      Supports array of values separated by comma or specified via multiple flags.

docs/victoriametrics/vmagent_enterprise_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
   -eula
      Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/

docs/victoriametrics/vmalert_common_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
 
 vmalert processes alerts and recording rules.
@@ -95,6 +95,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file.
      Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int

docs/victoriametrics/vmalert_enterprise_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
   -clusterMode
      If clusterMode is enabled, then vmalert automatically adds the tenant specified in config groups to -datasource.url, -remoteWrite.url and -remoteRead.url. See https://docs.victoriametrics.com/victoriametrics/vmalert/#multitenancy . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/

docs/victoriametrics/vmauth.md

@@ -718,7 +718,7 @@ users:
 `vmauth` responds with `429 Too Many Requests` HTTP error when the number of concurrent requests exceeds the configured limits for the duration
 exceeding the `-maxQueueDuration` command-line flag value.
 
-The following [metrics](#monitoring) related to concurrency limits are exposed by `vmauth`:
+The following [metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/#monitoring) related to concurrency limits are exposed by `vmauth`:
 
 * `vmauth_concurrent_requests_capacity` - the global limit on the number of concurrent requests `vmauth` can serve.
   It is set via `-maxConcurrentRequests` command-line flag.
@@ -734,6 +734,27 @@ The following [metrics](#monitoring) related to concurrency limits are exposed b
 * `vmauth_unauthorized_user_concurrent_requests_limit_reached_total` - the number of requests rejected with `429 Too Many Requests` error
   because of the concurrency limit has been reached for unauthorized users (if `unauthorized_user` section is used).
 
+See also [request body buffering](https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering).
+
+## Request body buffering
+
+`vmauth` can buffer request bodies {{% available_from "v1.135.0" %}} before proxying the requests to backends. This prevent slow-writing clients from occupying connections to backends.
+This is especially important when clients send requests over unreliable or low-bandwidth networks (for example, [IoT](https://en.wikipedia.org/wiki/Internet_of_things) devices over EDGE networks),
+where slow uploads can exhaust concurrency limits, increase latency, reduce ingestion rate, and trigger `429 Too Many Requests` responses even when backend resources are not saturated.
+
+Request body buffering can be configured with the `-requestBufferSize` command-line flag, which defines the maximum number of bytes to buffer from the request body
+before proxying the request to the backend. If buffering takes longer than the duration specified via `-maxQueueDuration` command-line flag, then the request is rejected.
+
+Request bodies are buffered before applying per-user [concurrency limits](https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting).
+
+The following [metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/#monitoring) related to request buffering are exposed by `vmauth`:
+
+* `vmauth_http_request_errors_total{reason="reject_slow_client"}` - the number of rejected requests because the request body couldn't be read during `-maxQueueDuration`.
+* `vmauth_buffer_request_body_duration_seconds` - the [summary](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#summary) for the request body buffering duration.
+  Use this metric to understand buffering performance and identify slow clients.
+
+See also [concurrency limits](https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limits).
+
 ## Backend TLS setup
 
 By default, `vmauth` uses system settings when performing requests to HTTPS backends specified via `url_prefix` option in the [`-auth.config`](#auth-config). These settings can be overridden with the following command-line flags:
@@ -1076,7 +1097,7 @@ It is recommended to protect the following endpoints with authKeys:
 
 * `/-/reload` with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
 * `/flags` with `-flagsAuthKey` command-line flag, so unauthorized users couldn't get command-line flag values.
-* `/metrics` with `-metricsAuthKey` command-line flag, so unauthorized users couldn't access [vmauth metrics](#monitoring).
+* `/metrics` with `-metricsAuthKey` command-line flag, so unauthorized users couldn't access [vmauth metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/#monitoring).
 * `/debug/pprof` with `-pprofAuthKey` command-line flag, so unauthorized users couldn't access [profiling information](#profiling).
 
 As an alternative, it's possible to serve internal API routes at the different listen address with command-line flag `-httpInternalListenAddr=127.0.0.1:8426`. {{% available_from "v1.111.0" %}}

docs/victoriametrics/vmauth_common_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
 
 vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
@@ -124,15 +124,15 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
   -maxConcurrentPerUserRequests int
-     The maximum number of concurrent requests vmauth can process per each configured user. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration and -maxConcurrentRequests command-line options and max_concurrent_requests option in per-user config (default 300)
+     The maximum number of concurrent requests vmauth can process per each configured user. Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. This provides fairness and isolation between users, preventing a single user from consuming all the available resources. It works in conjunction with -maxConcurrentRequests, which sets the global limit across all users. This default can be overridden for individual users via max_concurrent_requests option in per-user config. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting (default 100)
   -maxConcurrentRequests int
-     The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxQueueDuration, -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options (default 1000)
+     The maximum number of concurrent requests vmauth can process simultaneously. Requests exceeding this limit are queued for up to -maxQueueDuration and then rejected with '429 Too Many Requests' http status code if the limit is still reached. This protects vmauth itself from overloading and out-of-memory (OOM) failures. See also -maxConcurrentPerUserRequests and https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting (default 1000)
   -maxIdleConnsPerBackend int
-     The maximum number of idle connections vmauth can open per each backend host. See also -maxConcurrentRequests (default 100)
+     The maximum number of idle connections vmauth can open per each backend host (default 100)
   -maxQueueDuration duration
-     The maximum duration the request waits for execution when the number of concurrently executed requests reach -maxConcurrentRequests or -maxConcurrentPerUserRequests before returning '429 Too Many Requests' error. This allows graceful handling of short spikes in the number of concurrent requests (default 10s)
+     The maximum duration to wait before rejecting incoming requests if concurrency limit specified via -maxConcurrentRequests or -maxConcurrentPerUserRequests command-line flags is reached. Requests are rejected with '429 Too Many Requests' http status code if the limit is still reached after the -maxQueueDuration duration. This allows graceful handling of short spikes in concurrent requests. See https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting (default 10s)
   -maxRequestBodySizeToRetry size
-     The maximum request body size, which can be cached and re-tried at other backends. Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests
+     The maximum request body size to buffer in memory for potential retries at other backends. Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. See also -requestBufferSize
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16384)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
@@ -175,6 +175,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
      Flag value can be read from the given http/https url when using -reloadAuthKey=http://host/path or -reloadAuthKey=https://host/path
   -removeXFFHTTPHeaderValue
      Whether to remove the X-Forwarded-For HTTP header value from client requests before forwarding them to the backend. Recommended when vmauth is exposed to the internet.
+  -requestBufferSize size
+     The size of the buffer for reading the request body before proxying the request to backends. This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 32768)
   -responseTimeout duration
      The timeout for receiving a response from backend (default 5m0s)
   -retryStatusCodes array

docs/victoriametrics/vmauth_enterprise_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
   -eula
      Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/

docs/victoriametrics/vmbackup.md

@@ -380,6 +380,8 @@ Run `vmbackup -help` in order to see all the available options:
   -flagsAuthKey value
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int

docs/victoriametrics/vmbackupmanager.md

@@ -43,6 +43,8 @@ Backup schedule is controlled by the following flags:
 * `-disableMonthly` - disable monthly run. Default false
 
 By default, all flags are turned on and Backup Manager backups data every hour for every interval (hourly, daily, weekly and monthly).
+Note that if all intervals are disabled, the `vmbackupmanager` will create latest backups every 24 hours.
+Using `-disableScheduledBackups`{{% available_from "v1.135.0" %}} allows completely disabling scheduled backups. In this mode API can be used to trigger backups and restores on demand.
 
 The backup manager creates the following directory hierarchy at `-dst`:
 
@@ -537,6 +539,8 @@ command-line flags:
      Disable hourly run. Default false
   -disableMonthly
      Disable monthly run. Default false
+  -disableScheduledBackups
+     Disable all scheduled backups. This is useful in order to run vmbackupmanager in API only mode to allow requesting restore and performing manual backups. Default false
   -disableWeekly
      Disable weekly run. Default false
   -dst string
@@ -555,6 +559,8 @@ command-line flags:
   -flagsAuthKey value
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int

docs/victoriametrics/vmctl/influxdb.md

@@ -110,24 +110,5 @@ billions of datapoints.
 See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
 
 See `./vmctl influx --help` for details and full list of flags:
-```shellhelp
-   --influx-addr value              InfluxDB server addr (default: "http://localhost:8086")
-   --influx-user value              InfluxDB user [$INFLUX_USERNAME]
-   --influx-password value          InfluxDB user password [$INFLUX_PASSWORD]
-   --influx-database value          InfluxDB database
-   --influx-retention-policy value  InfluxDB retention policy (default: "autogen")
-   --influx-chunk-size value        The chunkSize defines max amount of series to be returned in one chunk (default: 10000)
-   --influx-concurrency value       Number of concurrently running fetch queries to InfluxDB (default: 1)
-   --influx-filter-series value     InfluxDB filter expression to select series. E.g. "from cpu where arch='x86' AND hostname='host_2753'".
-      See for details https://docs.influxdata.com/influxdb/v1.7/query_language/schema_exploration#show-series
-   --influx-filter-time-start value            The time filter to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
-   --influx-filter-time-end value              The time filter to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
-   --influx-measurement-field-separator value  The {separator} symbol used to concatenate {measurement} and {field} names into series name {measurement}{separator}{field}. (default: "_")
-   --influx-skip-database-label                Whether to skip adding the label 'db' to timeseries. (default: false)
-   --influx-prometheus-mode                    Whether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write. (default: false)
-   --influx-cert-file value                    Optional path to client-side TLS certificate file to use when connecting to -influx-addr
-   --influx-key-file value                     Optional path to client-side TLS key to use when connecting to -influx-addr
-   --influx-CA-file value                      Optional path to TLS CA file to use for verifying connections to -influx-addr. By default, system CA is used
-   --influx-server-name value                  Optional TLS server name to use for connections to -influx-addr. By default, the server name from -influx-addr is used
-   --influx-insecure-skip-verify               Whether to skip tls verification when connecting to -influx-addr (default: false)
-```
+
+{{% content "vmctl_influx_flags.md" %}}

docs/victoriametrics/vmctl/opentsdb.md

@@ -145,19 +145,5 @@ during migration.
 See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
 
 See `./vmctl opentsdb --help` for details and full list of flags:
-```shellhelp
-   --otsdb-addr value                                     OpenTSDB server addr (default: "http://localhost:4242")
-   --otsdb-concurrency value                              Number of concurrently running fetch queries to OpenTSDB per metric (default: 1)
-   --otsdb-retentions value [ --otsdb-retentions value ]  Retentions patterns to collect on. Each pattern should describe the aggregation performed for the query, the row size (in HBase) that will define how long each individual query is, and the time range to query for. e.g. sum-1m-avg:1h:3d. The first time range defined should be a multiple of the row size in HBase. e.g. if the row size is 2 hours, 4h is good, 5h less so. We want each query to land on unique rows.
-   --otsdb-filters value [ --otsdb-filters value ]        Filters to process for discovering metrics in OpenTSDB (default: "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z")
-   --otsdb-offset-days value                              Days to offset our 'starting' point for collecting data from OpenTSDB (default: 0)
-   --otsdb-hard-ts-start value                            A specific timestamp to start from, will override using an offset (default: 0)
-   --otsdb-query-limit value                              Result limit on meta queries to OpenTSDB (affects both metric name and tag value queries, recommended to use a value exceeding your largest series) (default: 100000000)
-   --otsdb-msecstime                                      Whether OpenTSDB is writing values in milliseconds or seconds (default: false)
-   --otsdb-normalize                                      Whether to normalize all data received to lower case before forwarding to VictoriaMetrics (default: false)
-   --otsdb-cert-file value                                Optional path to client-side TLS certificate file to use when connecting to -otsdb-addr
-   --otsdb-key-file value                                 Optional path to client-side TLS key to use when connecting to -otsdb-addr
-   --otsdb-CA-file value                                  Optional path to TLS CA file to use for verifying connections to -otsdb-addr. By default, system CA is used
-   --otsdb-server-name value                              Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from -otsdb-addr is used
-   --otsdb-insecure-skip-verify                           Whether to skip tls verification when connecting to -otsdb-addr (default: false)
-```
+
+{{% content "vmctl_opentsdb_flags.md" %}}

docs/victoriametrics/vmctl/prometheus.md

@@ -110,11 +110,5 @@ setting `--prom-concurrency` to the number of available to vmctl CPU cores.
 See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
 
 See `./vmctl prometheus --help` for details and full list of flags:
-```shellhelp
-   --prom-snapshot value            Path to Prometheus snapshot. Pls see for details https://www.robustperception.io/taking-snapshots-of-prometheus-data
-   --prom-concurrency value         Number of concurrently running snapshot readers (default: 1)
-   --prom-filter-time-start value   The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
-   --prom-filter-time-end value     The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
-   --prom-filter-label value        Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
-   --prom-filter-label-value value  Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
-```
+
+{{% content "vmctl_prometheus_flags.md" %}}

docs/victoriametrics/vmctl/remoteread.md

@@ -71,27 +71,5 @@ By default, is uses `SAMPLES` mode.
 See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
 
 See `./vmctl remote-read --help` for details and full list of flags:
-```shellhelp
-   --remote-read-concurrency value         Number of concurrently running remote read readers (default: 1)
-   --remote-read-filter-time-start value   The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
-   --remote-read-filter-time-end value     The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
-   --remote-read-filter-label value        Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name. (default: "__name__")
-   --remote-read-filter-label-value value  Prometheus regular expression to filter label from "remote-read-filter-label-value" flag. (default: ".*")
-   --remote-read                           Use Prometheus remote read protocol (default: false)
-   --remote-read-use-stream                Defines whether to use SAMPLES or STREAMED_XOR_CHUNKS mode. By default, is uses SAMPLES mode. See https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/#streamed-chunks (default: false)
-   --remote-read-step-interval value       The time interval to split the migration into steps. For example, to migrate 1y of data with '--remote-read-step-interval=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--remote-read-filter-time-reverse'. Requires setting '--remote-read-filter-time-start'. Valid values are 'month','week','day','hour','minute'.
-   --remote-read-filter-time-reverse       Whether to reverse the order of time intervals split by '--remote-read-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest data. (default: false)
-   --remote-read-src-addr value            Remote read address to perform read from.
-   --remote-read-user value                Remote read username for basic auth [$REMOTE_READ_USERNAME]
-   --remote-read-password value            Remote read password for basic auth [$REMOTE_READ_PASSWORD]
-   --remote-read-http-timeout value        Timeout defines timeout for HTTP requests made by remote read client (default: 0s)
-   --remote-read-headers value             Optional HTTP headers to send with each request to the corresponding remote source storage 
-      For example, --remote-read-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding remote source storage. 
-      Multiple headers must be delimited by '^^': --remote-read-headers='header1:value1^^header2:value2'
-   --remote-read-cert-file value       Optional path to client-side TLS certificate file to use when connecting to -remote-read-src-addr
-   --remote-read-key-file value        Optional path to client-side TLS key to use when connecting to -remote-read-src-addr
-   --remote-read-CA-file value         Optional path to TLS CA file to use for verifying connections to -remote-read-src-addr. By default, system CA is used
-   --remote-read-server-name value     Optional TLS server name to use for connections to remoteReadSrcAddr. By default, the server name from -remote-read-src-addr is used
-   --remote-read-insecure-skip-verify  Whether to skip TLS certificate verification when connecting to the remote read address (default: false)
-   --remote-read-disable-path-append   Whether to disable automatic appending of the /api/v1/read suffix to --remote-read-src-addr (default: false)
-```
+
+{{% content "vmctl_remote-read_flags.md" %}}

docs/victoriametrics/vmctl/victoriametrics.md

@@ -189,42 +189,5 @@ error `use of closed network connection` when running a heavy export requests.
 See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
 
 See `./vmctl vm-native --help` for details and full list of flags:
-```shellhelp
-   --vm-native-filter-match value  Time series selector to match series for export. For example, select {instance!="localhost"} will match all series with "instance" label different to "localhost".
-   --vm-native-filter-time-start value  The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats
-   --vm-native-filter-time-end value    The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats
-   --vm-native-step-interval value      The time interval to split the migration into steps. For example, to migrate 1y of data with '--vm-native-step-interval=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--vm-native-filter-time-reverse'. Requires setting '--vm-native-filter-time-start'. Valid values are 'month','week','day','hour','minute'. (default: "month")
-   --vm-native-filter-time-reverse      Whether to reverse the order of time intervals split by '--vm-native-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest data. (default: false)
-   --vm-native-disable-http-keep-alive  Disable HTTP persistent connections for requests made to VictoriaMetrics components during export (default: false)
-   --vm-native-src-addr value           VictoriaMetrics address to perform export from. 
-   --vm-native-src-user value      VictoriaMetrics username for basic auth [$VM_NATIVE_SRC_USERNAME]
-   --vm-native-src-password value  VictoriaMetrics password for basic auth [$VM_NATIVE_SRC_PASSWORD]
-   --vm-native-src-headers value   Optional HTTP headers to send with each request to the corresponding source address. 
-2025/06/24 11:50:41 Total time: 1.551792ms
-      For example, --vm-native-src-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding source address. 
-      Multiple headers must be delimited by '^^': --vm-native-src-headers='header1:value1^^header2:value2'
-   --vm-native-src-bearer-token --vm-native-src-addr          Optional bearer auth token to use for the corresponding --vm-native-src-addr
-   --vm-native-src-cert-file --vm-native-src-addr             Optional path to client-side TLS certificate file to use when connecting to --vm-native-src-addr
-   --vm-native-src-key-file --vm-native-src-addr              Optional path to client-side TLS key to use when connecting to --vm-native-src-addr
-   --vm-native-src-ca-file --vm-native-src-addr               Optional path to TLS CA file to use for verifying connections to --vm-native-src-addr. By default, system CA is used
-   --vm-native-src-server-name --vm-native-src-addr           Optional TLS server name to use for connections to --vm-native-src-addr. By default, the server name from `--vm-native-src-addr` is used
-   --vm-native-src-insecure-skip-verify --vm-native-src-addr  Whether to skip TLS certificate verification when connecting to --vm-native-src-addr (default: false)
-   --vm-native-dst-addr value                                 VictoriaMetrics address to perform import to. 
-   --vm-native-dst-user value      VictoriaMetrics username for basic auth [$VM_NATIVE_DST_USERNAME]
-   --vm-native-dst-password value  VictoriaMetrics password for basic auth [$VM_NATIVE_DST_PASSWORD]
-   --vm-native-dst-headers value   Optional HTTP headers to send with each request to the corresponding destination address. 
-      For example, --vm-native-dst-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding destination address. 
-      Multiple headers must be delimited by '^^': --vm-native-dst-headers='header1:value1^^header2:value2'
-   --vm-native-dst-bearer-token --vm-native-dst-addr          Optional bearer auth token to use for the corresponding --vm-native-dst-addr
-   --vm-native-dst-cert-file --vm-native-dst-addr             Optional path to client-side TLS certificate file to use when connecting to --vm-native-dst-addr
-   --vm-native-dst-key-file --vm-native-dst-addr              Optional path to client-side TLS key to use when connecting to --vm-native-dst-addr
-   --vm-native-dst-ca-file --vm-native-dst-addr               Optional path to TLS CA file to use for verifying connections to --vm-native-dst-addr. By default, system CA is used
-   --vm-native-dst-server-name --vm-native-dst-addr           Optional TLS server name to use for connections to --vm-native-dst-addr. By default, the server name from `--vm-native-dst-addr` is used
-   --vm-native-dst-insecure-skip-verify --vm-native-dst-addr  Whether to skip TLS certificate verification when connecting to --vm-native-dst-addr (default: false)
-       In this mode --vm-native-src-addr flag format is: 'http://vmselect:8481/'. --vm-native-dst-addr flag format is: http://vminsert:8480/. 
-   --vm-native-disable-per-metric-migration  Defines whether to disable per-metric migration and migrate all data via one connection. In this mode, vmctl makes less export/import requests, but can't provide a progress bar or retry failed requests. (default: false)
-   --vm-native-disable-binary-protocol       Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API.Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks.Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side. (default: false)
-   --vm-native-backoff-retries value         How many export/import retries to perform before giving up. (default: 10)
-   --vm-native-backoff-factor value          Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0 (default: 1.8)
-   --vm-native-backoff-min-duration value    Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'. (default: 2s)
-```
+
+{{% content "vmctl_vm-native_flags.md" %}}

docs/victoriametrics/vmctl/vmctl.md

@@ -169,6 +169,71 @@ see `--vm-concurrency` flag.
 Please note, you can also use [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/)
 as a proxy between `vmctl` and destination with `-remoteWrite.rateLimit` flag enabled.
 
+### Monitoring the migration process
+
+`vmctl` can push internal metrics {{% available_from "v1.135.0" %}} to a remote storage for monitoring migration progress and performance.
+This is especially useful for long-running migrations where you want to track progress, detect issues,
+or build dashboards to visualize the migration status.
+
+Example usage with VictoriaMetrics as the metrics destination:
+
+```sh
+./vmctl influx \
+  --influx-addr=http://localhost:8086 \
+  --influx-database=mydb \
+  --vm-addr=http://localhost:8428 \
+  --pushmetrics.url=http://localhost:8428/api/v1/import/prometheus \
+  --pushmetrics.extraLabel='job="vmctl"' \
+  --pushmetrics.extraLabel='instance="migration-1"'
+```
+
+#### Available metrics
+
+The following metrics are exposed by `vmctl`:
+
+General metrics (available for all migration modes):
+
+| Metric | Description |
+|--------|-------------|
+| `vmctl_backoff_retries_total` | Total number of retry attempts across all operations |
+| `vmctl_limiter_bytes_processed_total` | Total bytes processed through rate limiter (when `--vm-rate-limit` is set) |
+| `vmctl_limiter_throttle_events_total` | Number of times rate limiting caused a pause |
+
+Mode-specific metrics:
+
+Each migration mode exposes its own set of metrics with the mode name embedded in the metric name:
+
+| Mode | Metrics |
+|------|---------|
+| `influx` | `vmctl_influx_migration_series_total`, `vmctl_influx_migration_series_processed`, `vmctl_influx_migration_errors_total` |
+| `prometheus` | `vmctl_prometheus_migration_blocks_total`, `vmctl_prometheus_migration_blocks_processed`, `vmctl_prometheus_migration_errors_total` |
+| `opentsdb` | `vmctl_opentsdb_migration_series_total`, `vmctl_opentsdb_migration_series_processed`, `vmctl_opentsdb_migration_errors_total` |
+| `remote-read` | `vmctl_remote_read_migration_ranges_total`, `vmctl_remote_read_migration_ranges_processed`, `vmctl_remote_read_migration_errors_total` |
+| `vm-native` | `vmctl_vm_native_migration_metrics_total`, `vmctl_vm_native_migration_metrics_processed`, `vmctl_vm_native_migration_requests_planned`, `vmctl_vm_native_migration_requests_completed`, `vmctl_vm_native_migration_tenants_total`, `vmctl_vm_native_migration_tenants_processed`, `vmctl_vm_native_migration_bytes_transferred_total`, `vmctl_vm_native_migration_errors_total` |
+
+#### Example PromQL queries
+
+Monitor migration progress:
+```promql
+# Migration completion percentage for influx mode
+vmctl_influx_migration_series_processed / vmctl_influx_migration_series_total * 100
+
+# Migration completion percentage for vm-native mode
+vmctl_vm_native_migration_metrics_processed / vmctl_vm_native_migration_metrics_total * 100
+
+# Retry rate
+rate(vmctl_backoff_retries_total[5m])
+
+# Rate limiter throttling events per second
+rate(vmctl_limiter_throttle_events_total[5m])
+
+# Data transfer speed in bytes per second (when rate limiting is enabled)
+rate(vmctl_limiter_bytes_processed_total[5m])
+
+# Data transfer speed in MB per second for vm-native mode
+rate(vmctl_vm_native_migration_bytes_transferred_total[5m]) / 1Mb
+```
+
 ## Verifying exported blocks from VictoriaMetrics
 
 In this mode, `vmctl` allows verifying correctness and integrity of data exported via
@@ -235,82 +300,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 Run `vmctl -help` in order to see all the available options.
 
-Commands:
-```shellhelp
-  influx
-     Migrate time series from InfluxDB
-  opentsdb
-     Migrate time series from OpenTSDB.
-  prometheus
-     Migrate time series from Prometheus.
-  remote-read
-     Migrate time series via Prometheus remote-read protocol.
-  verify-block
-     Verifies exported block with VictoriaMetrics Native format.
-  vm-native
-     Migrate time series between VictoriaMetrics installations.
-```
-
-Flags available for all commands:
-
-```shellhelp
-  -s
-     Whether to run in silent mode. If set to true no confirmation prompts will appear. (default false)
-  -verbose
-     Whether to enable verbosity in logs output. (default false)
-  -disable-progress-bar
-     Whether to disable progress bar during the import. (default false)
-     
-   --vm-addr vmctl
-     VictoriaMetrics address to perform import requests. 
-     Should be the same as --httpListenAddr value for single-node version or vminsert component. 
-     When importing into the clustered version do not forget to set additionally --vm-account-id flag. 
-     Please note, that vmctl performs initial readiness check for the given address by checking `/health` endpoint. (default: "http://localhost:8428")
-   --vm-user value
-     VictoriaMetrics username for basic auth [$VM_USERNAME]
-   --vm-password value
-     VictoriaMetrics password for basic auth [$VM_PASSWORD]
-   --vm-account-id value
-     AccountID is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). 
-     AccountID is required when importing into the clustered version of VictoriaMetrics. 
-     It is possible to set it as accountID:projectID, where projectID is also arbitrary 32-bit integer. 
-     If projectID isn't set, then it equals to 0
-   --vm-concurrency value
-     Number of workers concurrently performing import requests to VM (default: 2)
-   --vm-compress
-     Whether to apply gzip compression to import requests (default: true)
-   --vm-batch-size value
-     How many samples importer collects before sending the import request to VM (default: 200000)
-   --vm-significant-figures value
-     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures.
-     Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics.
-     See also --vm-round-digits option (default: 0)
-   --vm-round-digits value
-     Round metric values to the given number of decimal digits after the point. This option may be used for increasing 
-     on-disk compression level for the stored metrics (default: 100)
-   --vm-extra-label value [ --vm-extra-label value ]
-     Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flagwill have priority.
-     Flag can be set multiple times, to add few additional labels.
-   --vm-rate-limit value
-     Optional data transfer rate limit in bytes per second.
-     By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination. (default: 0)
-   --vm-cert-file value
-     Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'
-   --vm-key-file value
-     Optional path to client-side TLS key to use when connecting to '--vmAddr'
-   --vm-CA-file value
-     Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used
-   --vm-server-name value
-     Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used
-   --vm-insecure-skip-verify
-     Whether to skip tls verification when connecting to '--vmAddr' (default: false)
-   --vm-backoff-retries value
-     How many import retries to perform before giving up. (default: 10)
-   --vm-backoff-factor value
-     Factor to multiply the base duration after each failed import retry. Must be greater than 1.0 (default: 1.8)
-   --vm-backoff-min-duration value
-     Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'. (default: 2s)
-```
+{{% content "vmctl_flags.md" %}}
 
 See list of available cmd-line flags for each command in the corresponding section.
 

docs/victoriametrics/vmctl/vmctl_flags.md

@@ -0,0 +1,30 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
+```shellhelp
+NAME:
+   vmctl - VictoriaMetrics command-line tool
+
+USAGE:
+   vmctl [global options] command [command options]
+
+
+COMMANDS:
+   opentsdb      Migrate time series from OpenTSDB
+   influx        Migrate time series from InfluxDB
+   remote-read   Migrate time series via Prometheus remote-read protocol
+   prometheus    Migrate time series from Prometheus
+   vm-native     Migrate time series between VictoriaMetrics installations
+   verify-block  Verifies exported block with VictoriaMetrics Native format
+   help, h       Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --help, -h     show help
+   --version, -v  print the version
+```

docs/victoriametrics/vmctl/vmctl_influx_flags.md

@@ -0,0 +1,72 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
+```shellhelp
+NAME:
+   vmctl influx - Migrate time series from InfluxDB
+
+USAGE:
+   vmctl influx [command options]
+
+OPTIONS:
+   -s                                                                 Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
+   --verbose                                                          Whether to enable verbosity in logs output. (default: false)
+   --disable-progress-bar                                             Whether to disable progress bar during the import. (default: false)
+   --pushmetrics.url value [ --pushmetrics.url value ]                Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics
+   --pushmetrics.interval value                                       Interval for pushing metrics to every -pushmetrics.url (default: 10s)
+   --pushmetrics.extraLabel value [ --pushmetrics.extraLabel value ]  Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels. For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url
+   --pushmetrics.header value [ --pushmetrics.header value ]          Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.
+   --pushmetrics.disableCompression                                   Whether to disable compression when pushing metrics. (default: false)
+   --influx-addr value                                                InfluxDB server addr (default: "http://localhost:8086")
+   --influx-user value                                                InfluxDB user [$INFLUX_USERNAME]
+   --influx-password value                                            InfluxDB user password [$INFLUX_PASSWORD]
+   --influx-database value                                            InfluxDB database
+   --influx-retention-policy value                                    InfluxDB retention policy (default: "autogen")
+   --influx-chunk-size value                                          The chunkSize defines max amount of series to be returned in one chunk (default: 10000)
+   --influx-concurrency value                                         Number of concurrently running fetch queries to InfluxDB (default: 1)
+   --influx-filter-series value                                       InfluxDB filter expression to select series. E.g. "from cpu where arch='x86' AND hostname='host_2753'".
+      See for details https://docs.influxdata.com/influxdb/v1.7/query_language/schema_exploration#show-series
+   --influx-filter-time-start value            The time filter to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+   --influx-filter-time-end value              The time filter to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+   --influx-measurement-field-separator value  The {separator} symbol used to concatenate {measurement} and {field} names into series name {measurement}{separator}{field}. (default: "_")
+   --influx-skip-database-label                Whether to skip adding the label 'db' to timeseries. (default: false)
+   --influx-prometheus-mode                    Whether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write. (default: false)
+   --influx-cert-file value                    Optional path to client-side TLS certificate file to use when connecting to -influx-addr
+   --influx-key-file value                     Optional path to client-side TLS key to use when connecting to -influx-addr
+   --influx-CA-file value                      Optional path to TLS CA file to use for verifying connections to -influx-addr. By default, system CA is used
+   --influx-server-name value                  Optional TLS server name to use for connections to -influx-addr. By default, the server name from -influx-addr is used
+   --influx-insecure-skip-verify               Whether to skip tls verification when connecting to -influx-addr (default: false)
+   --vm-addr value                             VictoriaMetrics address to perform import requests. 
+      Should be the same as --httpListenAddr value for single-node version or vminsert component. 
+      When importing into the clustered version do not forget to set additionally --vm-account-id flag. 
+      Please note, that vmctl performs initial readiness check for the given address by checking /health endpoint. (default: "http://localhost:8428")
+   --vm-user value        VictoriaMetrics username for basic auth [$VM_USERNAME]
+   --vm-password value    VictoriaMetrics password for basic auth [$VM_PASSWORD]
+   --vm-account-id value  AccountID is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). 
+      AccountID is required when importing into the clustered version of VictoriaMetrics. 
+      It is possible to set it as accountID:projectID, where projectID is also arbitrary 32-bit integer. 
+      If projectID isn't set, then it equals to 0
+   --vm-concurrency value                             Number of workers concurrently performing import requests to VM (default: 2)
+   --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
+   --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
+   --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
+   --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
+      By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)
+   --vm-cert-file value             Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'
+   --vm-key-file value              Optional path to client-side TLS key to use when connecting to '--vm-addr'
+   --vm-CA-file value               Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used
+   --vm-server-name value           Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used
+   --vm-insecure-skip-verify        Whether to skip tls verification when connecting to '--vm-addr' (default: false)
+   --vm-backoff-retries value       How many import retries to perform before giving up. (default: 10)
+   --vm-backoff-factor value        Factor to multiply the base duration after each failed import retry. Must be greater than 1.0 (default: 1.8)
+   --vm-backoff-min-duration value  Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'. (default: 2s)
+   --help, -h                       show help
+```

docs/victoriametrics/vmctl/vmctl_opentsdb_flags.md

@@ -0,0 +1,67 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
+```shellhelp
+NAME:
+   vmctl opentsdb - Migrate time series from OpenTSDB
+
+USAGE:
+   vmctl opentsdb [command options]
+
+OPTIONS:
+   -s                                                                 Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
+   --verbose                                                          Whether to enable verbosity in logs output. (default: false)
+   --disable-progress-bar                                             Whether to disable progress bar during the import. (default: false)
+   --pushmetrics.url value [ --pushmetrics.url value ]                Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics
+   --pushmetrics.interval value                                       Interval for pushing metrics to every -pushmetrics.url (default: 10s)
+   --pushmetrics.extraLabel value [ --pushmetrics.extraLabel value ]  Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels. For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url
+   --pushmetrics.header value [ --pushmetrics.header value ]          Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.
+   --pushmetrics.disableCompression                                   Whether to disable compression when pushing metrics. (default: false)
+   --otsdb-addr value                                                 OpenTSDB server addr (default: "http://localhost:4242")
+   --otsdb-concurrency value                                          Number of concurrently running fetch queries to OpenTSDB per metric (default: 1)
+   --otsdb-retentions value [ --otsdb-retentions value ]              Retentions patterns to collect on. Each pattern should describe the aggregation performed for the query, the row size (in HBase) that will define how long each individual query is, and the time range to query for. e.g. sum-1m-avg:1h:3d. The first time range defined should be a multiple of the row size in HBase. e.g. if the row size is 2 hours, 4h is good, 5h less so. We want each query to land on unique rows.
+   --otsdb-filters value [ --otsdb-filters value ]                    Filters to process for discovering metrics in OpenTSDB (default: "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z")
+   --otsdb-offset-days value                                          Days to offset our 'starting' point for collecting data from OpenTSDB (default: 0)
+   --otsdb-hard-ts-start value                                        A specific timestamp to start from, will override using an offset (default: 0)
+   --otsdb-query-limit value                                          Result limit on meta queries to OpenTSDB (affects both metric name and tag value queries, recommended to use a value exceeding your largest series) (default: 100000000)
+   --otsdb-msecstime                                                  Whether OpenTSDB is writing values in milliseconds or seconds (default: false)
+   --otsdb-normalize                                                  Whether to normalize all data received to lower case before forwarding to VictoriaMetrics (default: false)
+   --otsdb-cert-file value                                            Optional path to client-side TLS certificate file to use when connecting to -otsdb-addr
+   --otsdb-key-file value                                             Optional path to client-side TLS key to use when connecting to -otsdb-addr
+   --otsdb-CA-file value                                              Optional path to TLS CA file to use for verifying connections to -otsdb-addr. By default, system CA is used
+   --otsdb-server-name value                                          Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from -otsdb-addr is used
+   --otsdb-insecure-skip-verify                                       Whether to skip tls verification when connecting to -otsdb-addr (default: false)
+   --vm-addr value                                                    VictoriaMetrics address to perform import requests. 
+      Should be the same as --httpListenAddr value for single-node version or vminsert component. 
+      When importing into the clustered version do not forget to set additionally --vm-account-id flag. 
+      Please note, that vmctl performs initial readiness check for the given address by checking /health endpoint. (default: "http://localhost:8428")
+   --vm-user value        VictoriaMetrics username for basic auth [$VM_USERNAME]
+   --vm-password value    VictoriaMetrics password for basic auth [$VM_PASSWORD]
+   --vm-account-id value  AccountID is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). 
+      AccountID is required when importing into the clustered version of VictoriaMetrics. 
+      It is possible to set it as accountID:projectID, where projectID is also arbitrary 32-bit integer. 
+      If projectID isn't set, then it equals to 0
+   --vm-concurrency value                             Number of workers concurrently performing import requests to VM (default: 2)
+   --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
+   --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
+   --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
+   --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
+      By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)
+   --vm-cert-file value             Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'
+   --vm-key-file value              Optional path to client-side TLS key to use when connecting to '--vm-addr'
+   --vm-CA-file value               Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used
+   --vm-server-name value           Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used
+   --vm-insecure-skip-verify        Whether to skip tls verification when connecting to '--vm-addr' (default: false)
+   --vm-backoff-retries value       How many import retries to perform before giving up. (default: 10)
+   --vm-backoff-factor value        Factor to multiply the base duration after each failed import retry. Must be greater than 1.0 (default: 1.8)
+   --vm-backoff-min-duration value  Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'. (default: 2s)
+   --help, -h                       show help
+```

docs/victoriametrics/vmctl/vmctl_prometheus_flags.md

@@ -0,0 +1,60 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
+```shellhelp
+NAME:
+   vmctl prometheus - Migrate time series from Prometheus
+
+USAGE:
+   vmctl prometheus [command options]
+
+OPTIONS:
+   -s                                                                 Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
+   --verbose                                                          Whether to enable verbosity in logs output. (default: false)
+   --disable-progress-bar                                             Whether to disable progress bar during the import. (default: false)
+   --pushmetrics.url value [ --pushmetrics.url value ]                Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics
+   --pushmetrics.interval value                                       Interval for pushing metrics to every -pushmetrics.url (default: 10s)
+   --pushmetrics.extraLabel value [ --pushmetrics.extraLabel value ]  Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels. For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url
+   --pushmetrics.header value [ --pushmetrics.header value ]          Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.
+   --pushmetrics.disableCompression                                   Whether to disable compression when pushing metrics. (default: false)
+   --prom-snapshot value                                              Path to Prometheus snapshot. Pls see for details https://www.robustperception.io/taking-snapshots-of-prometheus-data
+   --prom-concurrency value                                           Number of concurrently running snapshot readers (default: 1)
+   --prom-filter-time-start value                                     The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+   --prom-filter-time-end value                                       The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+   --prom-filter-label value                                          Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
+   --prom-filter-label-value value                                    Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
+   --prom-tmp-dir-path value                                          Path to directory to be used for temporary files. (default: "/var/folders/ds/3kj5p3v17ll0hsyvq380ryvm0000gn/T/")
+   --vm-addr value                                                    VictoriaMetrics address to perform import requests. 
+      Should be the same as --httpListenAddr value for single-node version or vminsert component. 
+      When importing into the clustered version do not forget to set additionally --vm-account-id flag. 
+      Please note, that vmctl performs initial readiness check for the given address by checking /health endpoint. (default: "http://localhost:8428")
+   --vm-user value        VictoriaMetrics username for basic auth [$VM_USERNAME]
+   --vm-password value    VictoriaMetrics password for basic auth [$VM_PASSWORD]
+   --vm-account-id value  AccountID is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). 
+      AccountID is required when importing into the clustered version of VictoriaMetrics. 
+      It is possible to set it as accountID:projectID, where projectID is also arbitrary 32-bit integer. 
+      If projectID isn't set, then it equals to 0
+   --vm-concurrency value                             Number of workers concurrently performing import requests to VM (default: 2)
+   --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
+   --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
+   --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
+   --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
+      By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)
+   --vm-cert-file value             Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'
+   --vm-key-file value              Optional path to client-side TLS key to use when connecting to '--vm-addr'
+   --vm-CA-file value               Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used
+   --vm-server-name value           Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used
+   --vm-insecure-skip-verify        Whether to skip tls verification when connecting to '--vm-addr' (default: false)
+   --vm-backoff-retries value       How many import retries to perform before giving up. (default: 10)
+   --vm-backoff-factor value        Factor to multiply the base duration after each failed import retry. Must be greater than 1.0 (default: 1.8)
+   --vm-backoff-min-duration value  Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'. (default: 2s)
+   --help, -h                       show help
+```

docs/victoriametrics/vmctl/vmctl_remote-read_flags.md

@@ -0,0 +1,75 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
+```shellhelp
+NAME:
+   vmctl remote-read - Migrate time series via Prometheus remote-read protocol
+
+USAGE:
+   vmctl remote-read [command options]
+
+OPTIONS:
+   -s                                                                                 Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
+   --verbose                                                                          Whether to enable verbosity in logs output. (default: false)
+   --disable-progress-bar                                                             Whether to disable progress bar during the import. (default: false)
+   --pushmetrics.url value [ --pushmetrics.url value ]                                Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics
+   --pushmetrics.interval value                                                       Interval for pushing metrics to every -pushmetrics.url (default: 10s)
+   --pushmetrics.extraLabel value [ --pushmetrics.extraLabel value ]                  Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels. For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url
+   --pushmetrics.header value [ --pushmetrics.header value ]                          Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.
+   --pushmetrics.disableCompression                                                   Whether to disable compression when pushing metrics. (default: false)
+   --remote-read-concurrency value                                                    Number of concurrently running remote read readers (default: 1)
+   --remote-read-filter-time-start value                                              The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+   --remote-read-filter-time-end value                                                The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+   --remote-read-filter-label value [ --remote-read-filter-label value ]              Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name. (default: __name__)
+   --remote-read-filter-label-value value [ --remote-read-filter-label-value value ]  Prometheus regular expression to filter label from "remote-read-filter-label-value" flag. (default: .*)
+   --remote-read                                                                      Use Prometheus remote read protocol (default: false)
+   --remote-read-use-stream                                                           Defines whether to use SAMPLES or STREAMED_XOR_CHUNKS mode. By default, is uses SAMPLES mode. See https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/#streamed-chunks (default: false)
+   --remote-read-step-interval value                                                  The time interval to split the migration into steps. For example, to migrate 1y of data with '--remote-read-step-interval=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--remote-read-filter-time-reverse'. Requires setting '--remote-read-filter-time-start'. Valid values are 'month','week','day','hour','minute'.
+   --remote-read-filter-time-reverse                                                  Whether to reverse the order of time intervals split by '--remote-read-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest data. (default: false)
+   --remote-read-src-addr value                                                       Remote read address to perform read from.
+   --remote-read-user value                                                           Remote read username for basic auth [$REMOTE_READ_USERNAME]
+   --remote-read-password value                                                       Remote read password for basic auth [$REMOTE_READ_PASSWORD]
+   --remote-read-http-timeout value                                                   Timeout defines timeout for HTTP requests made by remote read client (default: 0s)
+   --remote-read-headers value                                                        Optional HTTP headers to send with each request to the corresponding remote source storage 
+      For example, --remote-read-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding remote source storage. 
+      Multiple headers must be delimited by '^^': --remote-read-headers='header1:value1^^header2:value2'
+   --remote-read-cert-file value       Optional path to client-side TLS certificate file to use when connecting to -remote-read-src-addr
+   --remote-read-key-file value        Optional path to client-side TLS key to use when connecting to -remote-read-src-addr
+   --remote-read-CA-file value         Optional path to TLS CA file to use for verifying connections to -remote-read-src-addr. By default, system CA is used
+   --remote-read-server-name value     Optional TLS server name to use for connections to remoteReadSrcAddr. By default, the server name from -remote-read-src-addr is used
+   --remote-read-insecure-skip-verify  Whether to skip TLS certificate verification when connecting to the remote read address (default: false)
+   --remote-read-disable-path-append   Whether to disable automatic appending of the /api/v1/read suffix to --remote-read-src-addr (default: false)
+   --vm-addr value                     VictoriaMetrics address to perform import requests. 
+      Should be the same as --httpListenAddr value for single-node version or vminsert component. 
+      When importing into the clustered version do not forget to set additionally --vm-account-id flag. 
+      Please note, that vmctl performs initial readiness check for the given address by checking /health endpoint. (default: "http://localhost:8428")
+   --vm-user value        VictoriaMetrics username for basic auth [$VM_USERNAME]
+   --vm-password value    VictoriaMetrics password for basic auth [$VM_PASSWORD]
+   --vm-account-id value  AccountID is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). 
+      AccountID is required when importing into the clustered version of VictoriaMetrics. 
+      It is possible to set it as accountID:projectID, where projectID is also arbitrary 32-bit integer. 
+      If projectID isn't set, then it equals to 0
+   --vm-concurrency value                             Number of workers concurrently performing import requests to VM (default: 2)
+   --vm-compress                                      Whether to apply gzip compression to import requests (default: true)
+   --vm-batch-size value                              How many samples importer collects before sending the import request to VM (default: 200000)
+   --vm-significant-figures value                     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
+   --vm-round-digits value                            Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
+   --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
+   --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
+      By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)
+   --vm-cert-file value             Optional path to client-side TLS certificate file to use when connecting to '--vm-addr'
+   --vm-key-file value              Optional path to client-side TLS key to use when connecting to '--vm-addr'
+   --vm-CA-file value               Optional path to TLS CA file to use for verifying connections to '--vm-addr'. By default, system CA is used
+   --vm-server-name value           Optional TLS server name to use for connections to '--vm-addr'. By default, the server name from '--vm-addr' is used
+   --vm-insecure-skip-verify        Whether to skip tls verification when connecting to '--vm-addr' (default: false)
+   --vm-backoff-retries value       How many import retries to perform before giving up. (default: 10)
+   --vm-backoff-factor value        Factor to multiply the base duration after each failed import retry. Must be greater than 1.0 (default: 1.8)
+   --vm-backoff-min-duration value  Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'. (default: 2s)
+   --help, -h                       show help
+```

docs/victoriametrics/vmctl/vmctl_vm-native_flags.md

@@ -0,0 +1,73 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
+```shellhelp
+NAME:
+   vmctl vm-native - Migrate time series between VictoriaMetrics installations
+
+USAGE:
+   vmctl vm-native [command options]
+
+OPTIONS:
+   -s                                                                 Whether to run in silent mode. If set to true no confirmation prompts will appear. (default: false)
+   --verbose                                                          Whether to enable verbosity in logs output. (default: false)
+   --disable-progress-bar                                             Whether to disable progress bar during the import. (default: false)
+   --pushmetrics.url value [ --pushmetrics.url value ]                Optional URL to push metrics. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#push-metrics
+   --pushmetrics.interval value                                       Interval for pushing metrics to every -pushmetrics.url (default: 10s)
+   --pushmetrics.extraLabel value [ --pushmetrics.extraLabel value ]  Extra labels to add to pushed metrics. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels. For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url
+   --pushmetrics.header value [ --pushmetrics.header value ]          Optional HTTP headers to add to pushed metrics. Flag can be set multiple times, to add few additional headers.
+   --pushmetrics.disableCompression                                   Whether to disable compression when pushing metrics. (default: false)
+   --vm-native-filter-match value                                     Time series selector to match series for export. For example, select {instance!="localhost"} will match all series with "instance" label different to "localhost".
+       See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format (default: "{__name__!=\"\"}")
+   --vm-native-filter-time-start value  The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats
+   --vm-native-filter-time-end value    The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats
+   --vm-native-step-interval value      The time interval to split the migration into steps. For example, to migrate 1y of data with '--vm-native-step-interval=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--vm-native-filter-time-reverse'. Requires setting '--vm-native-filter-time-start'. Valid values are 'month','week','day','hour','minute'. (default: "month")
+   --vm-native-filter-time-reverse      Whether to reverse the order of time intervals split by '--vm-native-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest data. (default: false)
+   --vm-native-disable-http-keep-alive  Disable HTTP persistent connections for requests made to VictoriaMetrics components during export (default: false)
+   --vm-native-src-addr value           VictoriaMetrics address to perform export from. 
+       Should be the same as --httpListenAddr value for single-node version or vmselect component. If exporting from cluster version see https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format
+   --vm-native-src-user value      VictoriaMetrics username for basic auth [$VM_NATIVE_SRC_USERNAME]
+   --vm-native-src-password value  VictoriaMetrics password for basic auth [$VM_NATIVE_SRC_PASSWORD]
+   --vm-native-src-headers value   Optional HTTP headers to send with each request to the corresponding source address. 
+      For example, --vm-native-src-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding source address. 
+      Multiple headers must be delimited by '^^': --vm-native-src-headers='header1:value1^^header2:value2'
+   --vm-native-src-bearer-token value    Optional bearer auth token to use for the corresponding --vm-native-src-addr
+   --vm-native-src-cert-file value       Optional path to client-side TLS certificate file to use when connecting to --vm-native-src-addr
+   --vm-native-src-key-file value        Optional path to client-side TLS key to use when connecting to --vm-native-src-addr
+   --vm-native-src-ca-file value         Optional path to TLS CA file to use for verifying connections to --vm-native-src-addr. By default, system CA is used
+   --vm-native-src-server-name value     Optional TLS server name to use for connections to --vm-native-src-addr. By default, the server name from --vm-native-src-addr is used
+   --vm-native-src-insecure-skip-verify  Whether to skip TLS certificate verification when connecting to --vm-native-src-addr (default: false)
+   --vm-native-dst-addr value            VictoriaMetrics address to perform import to. 
+       Should be the same as --httpListenAddr value for single-node version or vminsert component. If importing into cluster version see https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format
+   --vm-native-dst-user value      VictoriaMetrics username for basic auth [$VM_NATIVE_DST_USERNAME]
+   --vm-native-dst-password value  VictoriaMetrics password for basic auth [$VM_NATIVE_DST_PASSWORD]
+   --vm-native-dst-headers value   Optional HTTP headers to send with each request to the corresponding destination address. 
+      For example, --vm-native-dst-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding destination address. 
+      Multiple headers must be delimited by '^^': --vm-native-dst-headers='header1:value1^^header2:value2'
+   --vm-native-dst-bearer-token value                 Optional bearer auth token to use for the corresponding --vm-native-dst-addr
+   --vm-native-dst-cert-file value                    Optional path to client-side TLS certificate file to use when connecting to --vm-native-dst-addr
+   --vm-native-dst-key-file value                     Optional path to client-side TLS key to use when connecting to --vm-native-dst-addr
+   --vm-native-dst-ca-file value                      Optional path to TLS CA file to use for verifying connections to --vm-native-dst-addr. By default, system CA is used
+   --vm-native-dst-server-name value                  Optional TLS server name to use for connections to --vm-native-dst-addr. By default, the server name from --vm-native-dst-addr is used
+   --vm-native-dst-insecure-skip-verify               Whether to skip TLS certificate verification when connecting to --vm-native-dst-addr (default: false)
+   --vm-extra-label value [ --vm-extra-label value ]  Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
+   --vm-rate-limit value                              Optional data transfer rate limit in bytes per second.
+      By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases. 
+      Rate limit is applied per worker, see --vm-concurrency. (default: 0)
+   --vm-intercluster  Enables cluster-to-cluster migration mode with automatic tenants data migration.
+       In this mode --vm-native-src-addr flag format is: 'http://vmselect:8481/'. --vm-native-dst-addr flag format is: http://vminsert:8480/. 
+       TenantID will be appended automatically after discovering tenants from src. (default: false)
+   --vm-concurrency value                    Number of workers concurrently performing import requests to VM (default: 2)
+   --vm-native-disable-per-metric-migration  Defines whether to disable per-metric migration and migrate all data via one connection. In this mode, vmctl makes less export/import requests, but can't provide a progress bar or retry failed requests. (default: false)
+   --vm-native-disable-binary-protocol       Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API. Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks. Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side. (default: false)
+   --vm-native-backoff-retries value         How many export/import retries to perform before giving up. (default: 10)
+   --vm-native-backoff-factor value          Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0 (default: 1.8)
+   --vm-native-backoff-min-duration value    Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'. (default: 2s)
+   --help, -h                                show help
+```

docs/victoriametrics/vmgateway.md

@@ -380,6 +380,8 @@ Below is the list of configuration flags (it can be viewed by running `./vmgatew
   -flagsAuthKey value
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int

docs/victoriametrics/vminsert_common_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
 
 vminsert accepts data via popular data ingestion protocols and routes it to vmstorage nodes configured via -storageNode.
@@ -50,6 +50,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file.
      Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int

docs/victoriametrics/vminsert_enterprise_flags.md

@@ -6,7 +6,7 @@ build:
 sitemap:
   disable: true
 ---
-<!-- The file has to be manually updated during feature work in PR, make docs-update-flags command could be used peridically to ensure the flags in sync. -->
+<!-- The file should not be updated manually. Run make docs-update-flags while preparing a new release to sync flags in docs from actual binaries. -->
 ```shellhelp
   -cluster.tls
      Whether to use TLS for connections to -storageNode. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/victoriametrics/enterprise/

docs/victoriametrics/vmrestore.md

@@ -82,6 +82,8 @@ Run `vmrestore -help` in order to see all the available options:
   -flagsAuthKey value
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*
      Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path
+  -fs.disableMincore
+     Whether to disable the mincore() syscall for checking mmap()ed files. By default, mincore() is used to detect whether mmap()ed file pages are resident in memory. Disabling mincore() may be needed on older ZFS filesystems (below 2.1.5), since it may trigger ZFS bug. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10327 for details.
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -fs.maxConcurrency int