app/vmselect: fail the query request directly when there is not enough disk space

Previously, vmselect may crash if there is not enough free disk space to store tmp files from vmstorage. vmselect stores data blocks received from vmstorage if response size exceeds in-memory limit.
 It reduced vmselect availability, because it makes impossible to serve small queries. And huge query may crash vmselect for all users.

 So this commit adds a error check instead. If there is not enough disk space, vmselect will stop query execution and return error back to user.

fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4688

README.md

@@ -1,6 +1,6 @@
 # VictoriaMetrics
 
-[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?logo=github&labelColor=gray&color=gray&label=Release)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)

app/vmagent/remotewrite/remotewrite.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
@@ -62,9 +63,10 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
+	queues = flagutil.NewArrayIntWithDynamicDefault("remoteWrite.queues", cgroup.AvailableCPUs()*2, "2*cgroup.AvailableCPUs()",
+		"The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+			"isn't enough for sending high volume of collected data to remote storage. "+
+			"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	inmemoryQueues = flagutil.NewArrayInt("remoteWrite.inmemoryQueues", 0, "The number of additional workers per each -remoteWrite.url, which send only recently ingested data from the in-memory queue, "+
 		"while the file-based queue at -remoteWrite.tmpDataPath is drained by workers configured via -remoteWrite.queues. "+
 		"This reduces delivery lag for fresh samples when the file-based queue contains a backlog accumulated during remote storage outages.")
@@ -233,6 +235,7 @@ func Init() {
 	initStreamAggrConfigGlobal()
 
 	initRemoteWriteCtxs(*remoteWriteURLs)
+	appmetrics.MustCreateUncleanShutdownMarker(*tmpDataPath)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -391,6 +394,8 @@ func Stop() {
 	if sl := dailySeriesLimiter; sl != nil {
 		sl.MustStop()
 	}
+
+	appmetrics.MustRemoveUncleanShutdownMarker(*tmpDataPath)
 }
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url

app/vmalert/remotewrite/client.go

@@ -284,7 +284,15 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	bb := writeRequestBufPool.Get()
 	bb.B = wr.MarshalProtobuf(bb.B[:0])
 	zb := compressBufPool.Get()
-	defer compressBufPool.Put(zb)
+	// A failed send may leave the http transport still reading zb.B in a separate goroutine
+	// even after send returns, so zb is returned to the pool only if no send attempt has failed.
+	// See https://pkg.go.dev/net/http#RoundTripper
+	sendFailed := false
+	defer func() {
+		if !sendFailed {
+			compressBufPool.Put(zb)
+		}
+	}()
 	if c.isVMRemoteWrite.Load() {
 		zb.B = zstd.CompressLevel(zb.B[:0], bb.B, 0)
 	} else {
@@ -303,10 +311,13 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 L:
 	for {
 		err := c.send(ctx, zb.B)
-		if err != nil && (errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err)) {
-			// Something in the middle between client and destination might be closing
-			// the connection. So we do a one more attempt in hope request will succeed.
-			err = c.send(ctx, zb.B)
+		if err != nil {
+			sendFailed = true
+			if errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err) {
+				// Something in the middle between client and destination might be closing
+				// the connection. So we do a one more attempt in hope request will succeed.
+				err = c.send(ctx, zb.B)
+			}
 		}
 		if err == nil {
 			sentRows.Add(len(wr.Timeseries))

app/vmalert/remotewrite/debug_client.go

@@ -36,6 +36,13 @@ func NewDebugClient() (*DebugClient, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
+	tr.IdleConnTimeout = *idleConnectionTimeout
+	// DebugClient sends every series in a separate request, so it needs more idle
+	// connections than the two http.DefaultTransport keeps per host.
+	tr.MaxIdleConnsPerHost = *maxIdleConnections
+	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+	}
 	c := &DebugClient{
 		c: &http.Client{
 			Timeout:   *sendTimeout,

app/vmalert/remotewrite/debug_client_conns_test.go

@@ -0,0 +1,45 @@
+package remotewrite
+
+import (
+	"net/http"
+	"testing"
+)
+
+// TestDebugClient_IdleConns makes sure DebugClient keeps enough idle connections
+// to -remoteWrite.url. Every series is pushed in a separate request, so with the
+// two idle connections per host of http.DefaultTransport most of the concurrent
+// requests would open a new connection and leave a socket in TIME_WAIT state.
+func TestDebugClient_IdleConns(t *testing.T) {
+	f := func(maxIdle int) {
+		t.Helper()
+
+		oldAddr, oldMaxIdle := *addr, *maxIdleConnections
+		*addr, *maxIdleConnections = "http://localhost:8428", maxIdle
+		defer func() {
+			*addr, *maxIdleConnections = oldAddr, oldMaxIdle
+		}()
+
+		client, err := NewDebugClient()
+		if err != nil {
+			t.Fatalf("failed to create debug client: %s", err)
+		}
+		tr, ok := client.c.Transport.(*http.Transport)
+		if !ok {
+			t.Fatalf("unexpected transport type %T", client.c.Transport)
+		}
+		if tr.MaxIdleConnsPerHost != maxIdle {
+			t.Fatalf("unexpected MaxIdleConnsPerHost; got %d; want %d", tr.MaxIdleConnsPerHost, maxIdle)
+		}
+		if tr.MaxIdleConns != 0 && tr.MaxIdleConns < maxIdle {
+			t.Fatalf("MaxIdleConns=%d is lower than MaxIdleConnsPerHost=%d", tr.MaxIdleConns, maxIdle)
+		}
+		if tr.IdleConnTimeout != *idleConnectionTimeout {
+			t.Fatalf("unexpected IdleConnTimeout; got %s; want %s", tr.IdleConnTimeout, *idleConnectionTimeout)
+		}
+	}
+
+	f(100)
+
+	// the number of idle connections must be raised together with the total limit
+	f(1000)
+}

app/vmalert/remotewrite/init.go

@@ -34,10 +34,12 @@ var (
 	bearerTokenFile = flag.String("remoteWrite.bearerTokenFile", "", "Optional path to bearer token file to use for -remoteWrite.url.")
 
 	idleConnectionTimeout = flag.Duration("remoteWrite.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider settings this value less to the value of "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
+	maxIdleConnections    = flag.Int("remoteWrite.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to -remoteWrite.url for the vmalert-tool debug writer, which sends every series in a separate request. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 
-	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
-	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
-	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
+	maxQueueSize = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
+	maxBatchSize = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
+	concurrency  = flagutil.NewIntWithDynamicDefault("remoteWrite.concurrency", defaultConcurrency, "2*cgroup.AvailableCPUs()",
+		"Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
 	flushInterval = flag.Duration("remoteWrite.flushInterval", defaultFlushInterval, "Defines interval of flushes to remote write endpoint")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")

app/vmselect/clusternative/vmselect.go

@@ -1,7 +1,6 @@
 package clusternative
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"sync"
@@ -10,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -23,7 +23,7 @@ var (
 	maxTagValues                 = flag.Int("clusternative.maxTagValues", 100e3, "The maximum number of tag values returned per search at -clusternativeListenAddr")
 	maxTagValueSuffixesPerSearch = flag.Int("clusternative.maxTagValueSuffixesPerSearch", 100e3, "The maximum number of tag value suffixes returned "+
 		"from /metrics/find at -clusternativeListenAddr")
-	maxConcurrentRequests = flag.Int("clusternative.maxConcurrentRequests", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent vmselect requests "+
+	maxConcurrentRequests = flagutil.NewIntWithDynamicDefault("clusternative.maxConcurrentRequests", 2*cgroup.AvailableCPUs(), "2*cgroup.AvailableCPUs()", "The maximum number of concurrent vmselect requests "+
 		"the server can process at -clusternativeListenAddr. Default value depends on the number of available CPU cores. It shouldn't be high, since a single request usually saturates a CPU core at the underlying vmstorage nodes, "+
 		"and many concurrently executed requests may require high amounts of memory. See also -clusternative.maxQueueDuration")
 	maxQueueDuration = flag.Duration("clusternative.maxQueueDuration", 10*time.Second, "The maximum time the incoming query to -clusternativeListenAddr waits for execution "+
@@ -34,10 +34,8 @@ var (
 )
 
 // NewVMSelectServer starts new server at the given addr, which serves vmselect requests from netstorage.
-func NewVMSelectServer(ctx context.Context, addr string) (*vmselectapi.Server, error) {
-	api := &vmstorageAPI{
-		ctx: ctx,
-	}
+func NewVMSelectServer(addr string) (*vmselectapi.Server, error) {
+	api := &vmstorageAPI{}
 	limits := vmselectapi.Limits{
 		MaxConcurrentRequests:         *maxConcurrentRequests,
 		MaxConcurrentRequestsFlagName: "clusternative.maxConcurrentRequests",
@@ -48,25 +46,23 @@ func NewVMSelectServer(ctx context.Context, addr string) (*vmselectapi.Server, e
 }
 
 // vmstorageAPI impelements vmselectapi.API
-type vmstorageAPI struct {
-	ctx context.Context
-}
+type vmstorageAPI struct{}
 
 func (api *vmstorageAPI) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	bi := newBlockIterator(ctx, qt, true, sq)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	bi := newBlockIterator(qt, true, sq, dl)
 	return bi, nil
 }
 
 func (api *vmstorageAPI) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	res, err := netstorage.Tenants(ctx, qt, tr)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	res, err := netstorage.Tenants(qt, tr, dl)
 	return res, wrapClusterNativeError(err)
 }
 
 func (api *vmstorageAPI) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	metricNames, _, err := netstorage.SearchMetricNames(ctx, qt, true, sq)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	metricNames, _, err := netstorage.SearchMetricNames(qt, true, sq, dl)
 	return metricNames, wrapClusterNativeError(err)
 }
 
@@ -74,8 +70,8 @@ func (api *vmstorageAPI) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQ
 	if maxLabelValues <= 0 || maxLabelValues > *maxTagValues {
 		maxLabelValues = *maxTagValues
 	}
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	labelValues, _, err := netstorage.LabelValues(ctx, qt, true, labelName, sq, maxLabelValues)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	labelValues, _, err := netstorage.LabelValues(qt, true, labelName, sq, maxLabelValues, dl)
 	return labelValues, wrapClusterNativeError(err)
 }
 
@@ -84,8 +80,8 @@ func (api *vmstorageAPI) TagValueSuffixes(qt *querytracer.Tracer, accountID, pro
 	if maxSuffixes <= 0 || maxSuffixes > *maxTagValueSuffixesPerSearch {
 		maxSuffixes = *maxTagValueSuffixesPerSearch
 	}
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	suffixes, _, err := netstorage.TagValueSuffixes(ctx, qt, accountID, projectID, true, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	suffixes, _, err := netstorage.TagValueSuffixes(qt, accountID, projectID, true, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, dl)
 	return suffixes, wrapClusterNativeError(err)
 }
 
@@ -93,48 +89,48 @@ func (api *vmstorageAPI) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQu
 	if maxLabelNames <= 0 || maxLabelNames > *maxTagKeys {
 		maxLabelNames = *maxTagKeys
 	}
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	labelNames, _, err := netstorage.LabelNames(ctx, qt, true, sq, maxLabelNames)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	labelNames, _, err := netstorage.LabelNames(qt, true, sq, maxLabelNames, dl)
 	return labelNames, wrapClusterNativeError(err)
 }
 
 func (api *vmstorageAPI) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	seriesCount, _, err := netstorage.SeriesCount(ctx, qt, accountID, projectID, true)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	seriesCount, _, err := netstorage.SeriesCount(qt, accountID, projectID, true, dl)
 	return seriesCount, wrapClusterNativeError(err)
 }
 
 func (api *vmstorageAPI) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	tsdbStatus, _, err := netstorage.TSDBStatus(ctx, qt, true, sq, focusLabel, topN)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	tsdbStatus, _, err := netstorage.TSDBStatus(qt, true, sq, focusLabel, topN, dl)
 	return tsdbStatus, wrapClusterNativeError(err)
 }
 
 func (api *vmstorageAPI) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	deletedTotal, err := netstorage.DeleteSeries(ctx, qt, sq)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	deletedTotal, err := netstorage.DeleteSeries(qt, sq, dl)
 	return deletedTotal, wrapClusterNativeError(err)
 }
 
 func (api *vmstorageAPI) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	return wrapClusterNativeError(netstorage.RegisterMetricNames(ctx, qt, mrs))
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	return wrapClusterNativeError(netstorage.RegisterMetricNames(qt, mrs, dl))
 }
 
 func (api *vmstorageAPI) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	return wrapClusterNativeError(netstorage.ResetMetricNamesStats(ctx, qt))
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	return wrapClusterNativeError(netstorage.ResetMetricNamesStats(qt, dl))
 }
 
 func (api *vmstorageAPI) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, le, limit int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	statResult, err := netstorage.GetMetricNamesStats(ctx, qt, tt, le, limit, matchPattern)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	statResult, err := netstorage.GetMetricNamesStats(qt, tt, le, limit, matchPattern, dl)
 	return statResult, wrapClusterNativeError(err)
 }
 
 func (api *vmstorageAPI) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
-	ctx := searchutil.NewContextWithDeadlineTimestamp(api.ctx, deadline)
-	meta, _, err := netstorage.GetMetricsMetadata(ctx, qt, tt, true, limit, metricName)
+	dl := searchutil.DeadlineFromTimestamp(deadline)
+	meta, _, err := netstorage.GetMetricsMetadata(qt, tt, true, limit, metricName, dl)
 	return meta, wrapClusterNativeError(err)
 }
 
@@ -151,9 +147,9 @@ type workItem struct {
 	doneCh         chan struct{}
 }
 
-func newBlockIterator(ctx searchutil.Context, qt *querytracer.Tracer, denyPartialResponse bool, sq *storage.SearchQuery) *blockIterator {
+func newBlockIterator(qt *querytracer.Tracer, denyPartialResponse bool, sq *storage.SearchQuery, deadline searchutil.Deadline) *blockIterator {
 	bi := getBlockIterator()
-	workers, processBlocks := netstorage.PrepareProcessRawBlocks(ctx, qt, denyPartialResponse, sq)
+	workers, processBlocks := netstorage.PrepareProcessRawBlocks(qt, denyPartialResponse, sq, deadline)
 	bi.workCh = make(chan workItem, workers)
 	bi.wis = slicesutil.SetLength(bi.wis, workers)
 	for i := range bi.wis {

app/vmselect/graphite/eval.go

@@ -23,12 +23,12 @@ var maxGraphitePathExpressionLen = flag.Int("search.maxGraphitePathExpressionLen
 	"Longer expressions are truncated to prevent memory exhaustion on complex nested queries. Set to 0 to disable truncation.")
 
 type evalConfig struct {
-	ctx                 searchutil.Context
 	at                  *auth.Token
 	startTime           int64
 	endTime             int64
 	storageStep         int64
 	denyPartialResponse bool
+	deadline            searchutil.Deadline
 
 	currentTime time.Time
 
@@ -181,14 +181,14 @@ func evalMetricExpr(ec *evalConfig, me *graphiteql.MetricExpr) (nextSeriesFunc,
 }
 
 func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr graphiteql.Expr) (nextSeriesFunc, error) {
-	rss, _, err := netstorage.ProcessSearchQuery(ec.ctx, nil, ec.denyPartialResponse, sq)
+	rss, _, err := netstorage.ProcessSearchQuery(nil, ec.denyPartialResponse, sq, ec.deadline)
 	if err != nil {
 		return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
 	seriesCh := make(chan *series, cgroup.AvailableCPUs())
 	errCh := make(chan error, 1)
 	go func() {
-		err := rss.RunParallel(ec.ctx, nil, func(rs *netstorage.Result, _ uint) error {
+		err := rss.RunParallel(nil, func(rs *netstorage.Result, _ uint) error {
 			nameWithTags := getCanonicalPath(&rs.MetricName)
 			tags := unmarshalTags(nameWithTags)
 			s := &series{
@@ -201,9 +201,8 @@ func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr g
 			}
 			s.summarize(aggrAvg, ec.startTime, ec.endTime, ec.storageStep, 0)
 
-			deadline := ec.ctx.Deadline()
 			// A negative or zero duration will cause timer.C to return immediately
-			remainingTimeout := deadline.Deadline() - fasttime.UnixTimestamp()
+			remainingTimeout := ec.deadline.Deadline() - fasttime.UnixTimestamp()
 			t := timerpool.Get(time.Duration(remainingTimeout) * time.Second)
 			defer timerpool.Put(t)
 

app/vmselect/graphite/eval_test.go

@@ -1,7 +1,6 @@
 package graphite
 
 import (
-	"context"
 	"fmt"
 	"math"
 	"reflect"
@@ -10,13 +9,11 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphiteql"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 )
 
 func TestExecExprSuccess(t *testing.T) {
 	ec := &evalConfig{
-		ctx:         searchutil.NewContext(context.Background(), searchutil.NewDeadline(time.Now(), time.Minute, "")),
 		at:          &auth.Token{},
 		startTime:   120e3,
 		endTime:     210e3,

app/vmselect/graphite/metrics_api.go

@@ -28,7 +28,7 @@ var maxTagValueSuffixes = flag.Int("search.maxTagValueSuffixesPerSearch", 100e3,
 //
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
 func MetricsFindHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	format := r.FormValue("format")
 	if format == "" {
 		format = "treejson"
@@ -81,7 +81,7 @@ func MetricsFindHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit
 		MaxTimestamp: until,
 	}
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
-	paths, isPartial, err := metricsFind(ctx, at, denyPartialResponse, tr, label, "", query, delimiter[0], false)
+	paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, "", query, delimiter[0], false, deadline)
 	if err != nil {
 		return err
 	}
@@ -123,7 +123,7 @@ func deduplicatePaths(paths []string) []string {
 //
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-expand
 func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	queries := r.Form["query"]
 	if len(queries) == 0 {
 		return fmt.Errorf("missing `query` arg")
@@ -159,7 +159,7 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr
 	isPartialResponse := false
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
 	for _, query := range queries {
-		paths, isPartial, err := metricsFind(ctx, at, denyPartialResponse, tr, label, "", query, delimiter[0], true)
+		paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, "", query, delimiter[0], true, deadline)
 		if err != nil {
 			return err
 		}
@@ -208,11 +208,11 @@ func MetricsExpandHandler(startTime time.Time, at *auth.Token, w http.ResponseWr
 //
 // See https://graphite-api.readthedocs.io/en/latest/api.html#metrics-index-json
 func MetricsIndexHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	jsonp := r.FormValue("jsonp")
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
 	sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, 0, math.MaxInt64, nil, 0)
-	metricNames, isPartial, err := netstorage.LabelValues(ctx, nil, denyPartialResponse, "__name__", sq, 0)
+	metricNames, isPartial, err := netstorage.LabelValues(nil, denyPartialResponse, "__name__", sq, 0, deadline)
 	if err != nil {
 		return fmt.Errorf(`cannot obtain metric names: %w`, err)
 	}
@@ -229,12 +229,12 @@ func MetricsIndexHandler(startTime time.Time, at *auth.Token, w http.ResponseWri
 }
 
 // metricsFind searches for label values that match the given qHead and qTail.
-func metricsFind(ctx searchutil.Context, at *auth.Token, denyPartialResponse bool, tr storage.TimeRange, label, qHead, qTail string, delimiter byte,
-	isExpand bool) ([]string, bool, error) {
+func metricsFind(at *auth.Token, denyPartialResponse bool, tr storage.TimeRange, label, qHead, qTail string, delimiter byte,
+	isExpand bool, deadline searchutil.Deadline) ([]string, bool, error) {
 	n := strings.IndexAny(qTail, "*{[")
 	if n < 0 {
 		query := qHead + qTail
-		suffixes, isPartial, err := netstorage.TagValueSuffixes(ctx, nil, at.AccountID, at.ProjectID, denyPartialResponse, tr, label, query, delimiter, *maxTagValueSuffixes)
+		suffixes, isPartial, err := netstorage.TagValueSuffixes(nil, at.AccountID, at.ProjectID, denyPartialResponse, tr, label, query, delimiter, *maxTagValueSuffixes, deadline)
 		if err != nil {
 			return nil, false, err
 		}
@@ -254,7 +254,7 @@ func metricsFind(ctx searchutil.Context, at *auth.Token, denyPartialResponse boo
 	}
 	if n == len(qTail)-1 && strings.HasSuffix(qTail, "*") {
 		query := qHead + qTail[:len(qTail)-1]
-		suffixes, isPartial, err := netstorage.TagValueSuffixes(ctx, nil, at.AccountID, at.ProjectID, denyPartialResponse, tr, label, query, delimiter, *maxTagValueSuffixes)
+		suffixes, isPartial, err := netstorage.TagValueSuffixes(nil, at.AccountID, at.ProjectID, denyPartialResponse, tr, label, query, delimiter, *maxTagValueSuffixes, deadline)
 		if err != nil {
 			return nil, false, err
 		}
@@ -268,7 +268,7 @@ func metricsFind(ctx searchutil.Context, at *auth.Token, denyPartialResponse boo
 		return results, isPartial, nil
 	}
 	qHead += qTail[:n]
-	paths, isPartial, err := metricsFind(ctx, at, denyPartialResponse, tr, label, qHead, "*", delimiter, isExpand)
+	paths, isPartial, err := metricsFind(at, denyPartialResponse, tr, label, qHead, "*", delimiter, isExpand, deadline)
 	if err != nil {
 		return nil, false, err
 	}
@@ -292,7 +292,7 @@ func metricsFind(ctx searchutil.Context, at *auth.Token, denyPartialResponse boo
 			results = append(results, path)
 			continue
 		}
-		fullPaths, isPartialLocal, err := metricsFind(ctx, at, denyPartialResponse, tr, label, path, qTail, delimiter, isExpand)
+		fullPaths, isPartialLocal, err := metricsFind(at, denyPartialResponse, tr, label, path, qTail, delimiter, isExpand, deadline)
 		if err != nil {
 			return nil, false, err
 		}

app/vmselect/graphite/render_api.go

@@ -27,7 +27,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/stable/render_api.html
 func RenderHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	format := r.FormValue("format")
 	if format != "json" {
 		return fmt.Errorf("unsupported format=%q; supported values: json", format)
@@ -100,12 +100,12 @@ func RenderHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r
 	targets := r.Form["target"]
 	for _, target := range targets {
 		ec := &evalConfig{
-			ctx:                 ctx,
 			at:                  at,
 			startTime:           fromTime,
 			endTime:             untilTime,
 			storageStep:         storageStep,
 			denyPartialResponse: denyPartialResponse,
+			deadline:            deadline,
 			currentTime:         startTime,
 			xFilesFactor:        xFilesFactor,
 			etfs:                etfs,

app/vmselect/graphite/tags_api.go

@@ -31,7 +31,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb
 func TagsDelSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	paths := r.Form["path"]
 	totalDeleted := 0
 	var row graphiteparser.Row
@@ -60,7 +60,7 @@ func TagsDelSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseWr
 		}
 		tfss := joinTagFilterss(tfs, etfs)
 		sq := storage.NewSearchQuery(at.AccountID, at.ProjectID, 0, ct, tfss, 0)
-		n, err := netstorage.DeleteSeries(ctx, nil, sq)
+		n, err := netstorage.DeleteSeries(nil, sq, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot delete series for %q: %w", sq, err)
 		}
@@ -91,7 +91,7 @@ func TagsTagMultiSeriesHandler(startTime time.Time, at *auth.Token, w http.Respo
 }
 
 func registerMetrics(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request, isJSONResponse bool) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	paths := r.Form["path"]
 	var row graphiteparser.Row
 	var labels []prompb.Label
@@ -137,7 +137,7 @@ func registerMetrics(startTime time.Time, at *auth.Token, w http.ResponseWriter,
 		mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], at.AccountID, at.ProjectID, labels)
 		mr.Timestamp = ct
 	}
-	if err := netstorage.RegisterMetricNames(ctx, nil, mrs); err != nil {
+	if err := netstorage.RegisterMetricNames(nil, mrs, deadline); err != nil {
 		return fmt.Errorf("cannot register paths: %w", err)
 	}
 
@@ -165,7 +165,7 @@ var (
 //
 // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
 func TagsAutoCompleteValuesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
 		return err
@@ -192,7 +192,7 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, at *auth.Token, w http.R
 		// Escape special chars in tagPrefix as Graphite does.
 		// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228
 		filter := regexp.QuoteMeta(valuePrefix)
-		tagValues, isPartial, err = netstorage.GraphiteTagValues(ctx, nil, at.AccountID, at.ProjectID, denyPartialResponse, tag, filter, *maxGraphiteTagValuesPerSearch)
+		tagValues, isPartial, err = netstorage.GraphiteTagValues(nil, at.AccountID, at.ProjectID, denyPartialResponse, tag, filter, *maxGraphiteTagValuesPerSearch, deadline)
 		if err != nil {
 			return err
 		}
@@ -202,7 +202,7 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, at *auth.Token, w http.R
 		if err != nil {
 			return err
 		}
-		metricNames, isPartialResponse, err := netstorage.SearchMetricNames(ctx, nil, denyPartialResponse, sq)
+		metricNames, isPartialResponse, err := netstorage.SearchMetricNames(nil, denyPartialResponse, sq, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
 		}
@@ -258,7 +258,7 @@ var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_sec
 //
 // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support
 func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
 		return err
@@ -282,7 +282,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.Res
 		// Escape special chars in tagPrefix as Graphite does.
 		// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181
 		filter := regexp.QuoteMeta(tagPrefix)
-		labels, isPartial, err = netstorage.GraphiteTags(ctx, nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, *maxGraphiteTagKeysPerSearch)
+		labels, isPartial, err = netstorage.GraphiteTags(nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, *maxGraphiteTagKeysPerSearch, deadline)
 		if err != nil {
 			return err
 		}
@@ -292,7 +292,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.Res
 		if err != nil {
 			return err
 		}
-		metricNames, isPartialResponse, err := netstorage.SearchMetricNames(ctx, nil, denyPartialResponse, sq)
+		metricNames, isPartialResponse, err := netstorage.SearchMetricNames(nil, denyPartialResponse, sq, deadline)
 		if err != nil {
 			return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
 		}
@@ -344,7 +344,7 @@ var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_secon
 //
 // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
 func TagsFindSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
 		return err
@@ -362,7 +362,7 @@ func TagsFindSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseW
 		return err
 	}
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
-	metricNames, isPartial, err := netstorage.SearchMetricNames(ctx, nil, denyPartialResponse, sq)
+	metricNames, isPartial, err := netstorage.SearchMetricNames(nil, denyPartialResponse, sq, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err)
 	}
@@ -420,14 +420,14 @@ var tagsFindSeriesDuration = metrics.NewSummary(`vm_request_duration_seconds{pat
 //
 // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
 func TagValuesHandler(startTime time.Time, at *auth.Token, tagName string, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
 		return err
 	}
 	filter := r.FormValue("filter")
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
-	tagValues, isPartial, err := netstorage.GraphiteTagValues(ctx, nil, at.AccountID, at.ProjectID, denyPartialResponse, tagName, filter, *maxGraphiteTagValuesPerSearch)
+	tagValues, isPartial, err := netstorage.GraphiteTagValues(nil, at.AccountID, at.ProjectID, denyPartialResponse, tagName, filter, *maxGraphiteTagValuesPerSearch, deadline)
 	if err != nil {
 		return err
 	}
@@ -452,14 +452,14 @@ var tagValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/t
 //
 // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags
 func TagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
 		return err
 	}
 	filter := r.FormValue("filter")
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
-	labels, isPartial, err := netstorage.GraphiteTags(ctx, nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, *maxGraphiteTagKeysPerSearch)
+	labels, isPartial, err := netstorage.GraphiteTags(nil, at.AccountID, at.ProjectID, denyPartialResponse, filter, *maxGraphiteTagKeysPerSearch, deadline)
 	if err != nil {
 		return err
 	}

app/vmselect/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"embed"
 	"encoding/json"
 	"flag"
@@ -22,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/stats"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
@@ -102,8 +102,6 @@ func main() {
 	buildinfo.Init()
 	logger.Init()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 	logger.Infof("starting netstorage at storageNodes %s", *storageNodes)
 	startTime := time.Now()
 	storage.SetDedupInterval(*minScrapeInterval)
@@ -126,6 +124,7 @@ func main() {
 		fs.MustRemoveDirContents(tmpDataPath)
 		netstorage.InitTmpBlocksDir(tmpDataPath)
 		promql.InitRollupResultCache(*cacheDataPath + "/rollupResult")
+		appmetrics.MustCreateUncleanShutdownMarker(*cacheDataPath)
 	} else {
 		netstorage.InitTmpBlocksDir("")
 		promql.InitRollupResultCache("")
@@ -138,7 +137,7 @@ func main() {
 	var vmselectapiServer *vmselectapi.Server
 	if *clusternativeListenAddr != "" {
 		logger.Infof("starting vmselectapi server at %q", *clusternativeListenAddr)
-		s, err := clusternative.NewVMSelectServer(ctx, *clusternativeListenAddr)
+		s, err := clusternative.NewVMSelectServer(*clusternativeListenAddr)
 		if err != nil {
 			logger.Fatalf("cannot initialize vmselectapi server: %s", err)
 		}
@@ -165,7 +164,6 @@ func main() {
 		logger.Fatalf("cannot stop http service: %s", err)
 	}
 	logger.Infof("successfully shut down http service in %.3f seconds", time.Since(startTime).Seconds())
-	cancel()
 
 	if vmselectapiServer != nil {
 		logger.Infof("stopping vmselectapi server...")
@@ -178,6 +176,7 @@ func main() {
 	netstorage.MustStop()
 	if len(*cacheDataPath) > 0 {
 		promql.StopRollupResultCache()
+		appmetrics.MustRemoveUncleanShutdownMarker(*cacheDataPath)
 	}
 	logger.Infof("successfully stopped netstorage in %.3f seconds", time.Since(startTime).Seconds())
 
@@ -200,6 +199,7 @@ var (
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := strings.ReplaceAll(r.URL.Path, "//", "/")
+
 	if handleStaticAndSimpleRequests(w, r, path) {
 		return true
 	}

app/vmselect/netstorage/tenant_cache.go

@@ -21,7 +21,7 @@ var (
 )
 
 // TenantsCached returns the list of tenants available in the storage.
-func TenantsCached(ctx searchutil.Context, qt *querytracer.Tracer, tr storage.TimeRange, mayCache bool) ([]storage.TenantToken, error) {
+func TenantsCached(qt *querytracer.Tracer, tr storage.TimeRange, deadline searchutil.Deadline, mayCache bool) ([]storage.TenantToken, error) {
 	qtL := qt.NewChild("fetching tenants on timeRange=%s", tr.String())
 	defer qtL.Done()
 
@@ -41,7 +41,7 @@ func TenantsCached(ctx searchutil.Context, qt *querytracer.Tracer, tr storage.Ti
 		qtL.Printf("do not fetch list of tenants from cache")
 	}
 
-	tenants, err := Tenants(ctx, qtL, tr)
+	tenants, err := Tenants(qtL, tr, deadline)
 	if err != nil {
 		return nil, fmt.Errorf("cannot obtain tenants: %w", err)
 	}

app/vmselect/netstorage/tenant_filters.go

@@ -13,8 +13,8 @@ import (
 )
 
 // GetTenantTokensFromFilters returns the list of tenant tokens and the list of filters without tenant filters.
-func GetTenantTokensFromFilters(ctx searchutil.Context, qt *querytracer.Tracer, tr storage.TimeRange, tfs [][]storage.TagFilter, mayCache bool) ([]storage.TenantToken, [][]storage.TagFilter, error) {
-	tenants, err := TenantsCached(ctx, qt, tr, mayCache)
+func GetTenantTokensFromFilters(qt *querytracer.Tracer, tr storage.TimeRange, tfs [][]storage.TagFilter, deadline searchutil.Deadline, mayCache bool) ([]storage.TenantToken, [][]storage.TagFilter, error) {
+	tenants, err := TenantsCached(qt, tr, deadline, mayCache)
 	if err != nil {
 		return nil, nil, fmt.Errorf("cannot obtain tenants: %w", err)
 	}

app/vmselect/netstorage/tmp_blocks_file.go

@@ -63,6 +63,9 @@ type tmpBlocksFile struct {
 	r *fs.ReaderAt
 
 	offset uint64
+
+	// err stores the first error occurred while writing the temporary blocks file.
+	err error
 }
 
 func getTmpBlocksFile() *tmpBlocksFile {
@@ -83,6 +86,7 @@ func putTmpBlocksFile(tbf *tmpBlocksFile) {
 	tbf.f = nil
 	tbf.r = nil
 	tbf.offset = 0
+	tbf.err = nil
 	tmpBlocksFilePool.Put(tbf)
 }
 
@@ -109,8 +113,16 @@ var (
 //
 // It returns errors since the operation may fail on space shortage
 // and this must be handled.
+//
+// The tbf is left unusable after the first error, since a failed write cannot be undone:
+// the returned addresses are derived from tbf.offset before the data is flushed from tbf.buf
+// to the file, the failed flush may be partial, and the remaining buffer is dropped.
 func (tbf *tmpBlocksFile) WriteBlockData(b []byte, tbfIdx uint) (tmpBlockAddr, error) {
 	var addr tmpBlockAddr
+	if tbf.err != nil {
+		// Do not write anything to the tbf after the first failed write
+		return addr, tbf.err
+	}
 	addr.tbfIdx = tbfIdx
 	addr.offset = tbf.offset
 	addr.size = len(b)
@@ -125,7 +137,8 @@ func (tbf *tmpBlocksFile) WriteBlockData(b []byte, tbfIdx uint) (tmpBlockAddr, e
 	if tbf.f == nil {
 		f, err := os.CreateTemp(tmpBlocksDir, "")
 		if err != nil {
-			return addr, err
+			tbf.err = fmt.Errorf("cannot create temporary blocks file at %q: %w", tmpBlocksDir, err)
+			return addr, tbf.err
 		}
 		tbf.f = f
 		tmpBlocksFilesCreated.Inc()
@@ -133,7 +146,9 @@ func (tbf *tmpBlocksFile) WriteBlockData(b []byte, tbfIdx uint) (tmpBlockAddr, e
 	_, err := tbf.f.Write(tbf.buf)
 	tbf.buf = append(tbf.buf[:0], b...)
 	if err != nil {
-		return addr, fmt.Errorf("cannot write block to %q: %w", tbf.f.Name(), err)
+		// The blocks buffered at tbf.buf could be partially lost, mark the tbf as unusable.
+		tbf.err = fmt.Errorf("cannot write block to %q: %w", tbf.f.Name(), err)
+		return addr, tbf.err
 	}
 	return addr, nil
 }
@@ -144,12 +159,16 @@ func (tbf *tmpBlocksFile) Len() uint64 {
 }
 
 func (tbf *tmpBlocksFile) Finalize() error {
+	if tbf.err != nil {
+		return tbf.err
+	}
 	if tbf.f == nil {
 		return nil
 	}
 	fname := tbf.f.Name()
 	if _, err := tbf.f.Write(tbf.buf); err != nil {
-		return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
+		tbf.err = fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
+		return tbf.err
 	}
 	tbf.buf = tbf.buf[:0]
 	r := fs.NewReaderAt(tbf.f)
@@ -169,6 +188,10 @@ func (tbf *tmpBlocksFile) Finalize() error {
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockAt(dst *storage.Block, addr tmpBlockAddr) {
+	if tbf.err != nil {
+		// This should never happen, since Finalize() already returns the error for such a tbf.
+		logger.Panicf("BUG: cannot read block at %s from the temporary blocks file with the failed write: %s", addr, tbf.err)
+	}
 	var buf []byte
 	if tbf.r == nil {
 		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]

app/vmselect/netstorage/tmp_blocks_file_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 	"time"
@@ -29,6 +30,39 @@ func TestTmpBlocksFileSerial(t *testing.T) {
 	}
 }
 
+func TestTmpBlocksFileWriteFailure(t *testing.T) {
+	// Emulate the disk write failure by pointing tmpBlocksDir to a non-existing directory.
+	tmpBlocksDirOrig := tmpBlocksDir
+	tmpBlocksDir = filepath.Join(tmpBlocksDirOrig, "non-existing-dir")
+	defer func() {
+		tmpBlocksDir = tmpBlocksDirOrig
+	}()
+
+	tbf := getTmpBlocksFile()
+	defer putTmpBlocksFile(tbf)
+
+	// Write blocks until tbf.buf is flushed to the file. The flush must fail.
+	b := make([]byte, 64*1024)
+	var writeErr error
+	for range maxInmemoryTmpBlocksFile()/len(b) + 2 {
+		if _, err := tbf.WriteBlockData(b, 0); err != nil {
+			writeErr = err
+			break
+		}
+	}
+	if writeErr == nil {
+		t.Fatalf("expecting non-nil error from WriteBlockData")
+	}
+
+	if _, err := tbf.WriteBlockData(b, 0); err != writeErr {
+		t.Fatalf("expecting non-nil error from WriteBlockData after the failed write")
+	}
+
+	if err := tbf.Finalize(); err != writeErr {
+		t.Fatalf("expecting non-nil error from Finalize after the failed write")
+	}
+}
+
 func TestTmpBlocksFileConcurrent(t *testing.T) {
 	concurrency := 3
 	ch := make(chan error, concurrency)

app/vmselect/prometheus/prometheus.go

@@ -144,7 +144,7 @@ func FederateHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter,
 		return fmt.Errorf("cannot obtain search query: %w", err)
 	}
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
-	rss, isPartial, err := netstorage.ProcessSearchQuery(cp.ctx, nil, denyPartialResponse, sq)
+	rss, isPartial, err := netstorage.ProcessSearchQuery(nil, denyPartialResponse, sq, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 	}
@@ -172,7 +172,7 @@ func FederateHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter,
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
 	sw := newScalableWriter(bw)
-	err = rss.RunParallel(cp.ctx, nil, func(rs *netstorage.Result, workerID uint) error {
+	err = rss.RunParallel(nil, func(rs *netstorage.Result, workerID uint) error {
 		if err := bw.Error(); err != nil {
 			return err
 		}
@@ -229,12 +229,12 @@ func ExportCSVHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter
 		// Unconditionally deny partial response for the exported data,
 		// since users usually expect that the exported data is full.
 		denyPartialResponse := true
-		rss, _, err := netstorage.ProcessSearchQuery(cp.ctx, nil, denyPartialResponse, sq)
+		rss, _, err := netstorage.ProcessSearchQuery(nil, denyPartialResponse, sq, cp.deadline)
 		if err != nil {
 			return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 		}
 		go func() {
-			err := rss.RunParallel(cp.ctx, nil, func(rs *netstorage.Result, workerID uint) error {
+			err := rss.RunParallel(nil, func(rs *netstorage.Result, workerID uint) error {
 				if err := bw.Error(); err != nil {
 					return err
 				}
@@ -253,7 +253,7 @@ func ExportCSVHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter
 		}()
 	} else {
 		go func() {
-			err := netstorage.ExportBlocks(cp.ctx, nil, sq, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange, workerID uint) error {
+			err := netstorage.ExportBlocks(nil, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange, workerID uint) error {
 				if err := bw.Error(); err != nil {
 					return err
 				}
@@ -310,7 +310,7 @@ func ExportNativeHandler(startTime time.Time, at *auth.Token, w http.ResponseWri
 	_, _ = bw.Write(trBuf)
 
 	// Marshal native blocks.
-	err = netstorage.ExportBlocks(cp.ctx, nil, sq, func(mn *storage.MetricName, b *storage.Block, _ storage.TimeRange, workerID uint) error {
+	err = netstorage.ExportBlocks(nil, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, _ storage.TimeRange, workerID uint) error {
 		if err := bw.Error(); err != nil {
 			return err
 		}
@@ -455,13 +455,13 @@ func exportHandler(qt *querytracer.Tracer, at *auth.Token, w http.ResponseWriter
 		// Unconditionally deny partial response for the exported data,
 		// since users usually expect that the exported data is full.
 		denyPartialResponse := true
-		rss, _, err := netstorage.ProcessSearchQuery(cp.ctx, qt, denyPartialResponse, sq)
+		rss, _, err := netstorage.ProcessSearchQuery(qt, denyPartialResponse, sq, cp.deadline)
 		if err != nil {
 			return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
 		}
 		qtChild := qt.NewChild("background export format=%s", format)
 		go func() {
-			err := rss.RunParallel(cp.ctx, qtChild, func(rs *netstorage.Result, workerID uint) error {
+			err := rss.RunParallel(qtChild, func(rs *netstorage.Result, workerID uint) error {
 				if err := bw.Error(); err != nil {
 					return err
 				}
@@ -482,7 +482,7 @@ func exportHandler(qt *querytracer.Tracer, at *auth.Token, w http.ResponseWriter
 	} else {
 		qtChild := qt.NewChild("background export format=%s", format)
 		go func() {
-			err := netstorage.ExportBlocks(cp.ctx, qtChild, sq, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange, workerID uint) error {
+			err := netstorage.ExportBlocks(qtChild, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange, workerID uint) error {
 				if err := bw.Error(); err != nil {
 					return err
 				}
@@ -549,7 +549,7 @@ func DeleteHandler(startTime time.Time, at *auth.Token, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	cp.ctx = searchutil.GetContextForDelete(r, startTime)
+	cp.deadline = searchutil.GetDeadlineForDelete(r, startTime)
 
 	if !cp.IsDefaultTimeRange() {
 		return fmt.Errorf("delete API does not support specific time ranges using start and end args, the series can only be deleted completely")
@@ -558,7 +558,7 @@ func DeleteHandler(startTime time.Time, at *auth.Token, r *http.Request) error {
 	if err != nil {
 		return err
 	}
-	deletedCount, err := netstorage.DeleteSeries(cp.ctx, nil, sq)
+	deletedCount, err := netstorage.DeleteSeries(nil, sq, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot delete time series: %w", err)
 	}
@@ -649,7 +649,7 @@ var httpClient = &http.Client{
 
 // Tenants processes /admin/tenants request.
 func Tenants(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
-	ctx := searchutil.GetContextForStatusRequest(r, startTime)
+	deadline := searchutil.GetDeadlineForStatusRequest(r, startTime)
 	start, err := httputil.GetTime(r, "start", 0)
 	if err != nil {
 		return err
@@ -663,7 +663,7 @@ func Tenants(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter,
 		MinTimestamp: start,
 		MaxTimestamp: end,
 	}
-	tenants, err := netstorage.Tenants(ctx, qt, tr)
+	tenants, err := netstorage.Tenants(qt, tr, deadline)
 	if err != nil {
 		return err
 	}
@@ -702,7 +702,7 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.To
 		// Spec: https://github.com/prometheus/proposals/blob/main/proposals/0028-utf8.md
 		labelName = unescapePrometheusLabelName(labelName)
 	}
-	labelValues, isPartial, err := netstorage.LabelValues(cp.ctx, qt, denyPartialResponse, labelName, sq, limit)
+	labelValues, isPartial, err := netstorage.LabelValues(qt, denyPartialResponse, labelName, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain values for label %q: %w", labelName, err)
 	}
@@ -732,7 +732,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok
 	if err != nil {
 		return httpserver.InvalidParamError(err)
 	}
-	cp.ctx = searchutil.GetContextForStatusRequest(r, startTime)
+	cp.deadline = searchutil.GetDeadlineForStatusRequest(r, startTime)
 
 	date := fasttime.UnixDate()
 	dateStr := r.FormValue("date")
@@ -770,7 +770,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok
 	if err != nil {
 		return err
 	}
-	status, isPartial, err := netstorage.TSDBStatus(cp.ctx, qt, denyPartialResponse, sq, focusLabel, topN)
+	status, isPartial, err := netstorage.TSDBStatus(qt, denyPartialResponse, sq, focusLabel, topN, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain tsdb stats: %w", err)
 	}
@@ -806,7 +806,7 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token,
 	if err != nil {
 		return err
 	}
-	labels, isPartial, err := netstorage.LabelNames(cp.ctx, qt, denyPartialResponse, sq, limit)
+	labels, isPartial, err := netstorage.LabelNames(qt, denyPartialResponse, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain labels: %w", err)
 	}
@@ -850,7 +850,7 @@ func MetadataHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token
 
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
 
-	metadata, isPartial, err := netstorage.GetMetricsMetadata(cp.ctx, qt, tt, denyPartialResponse, limit, metricName)
+	metadata, isPartial, err := netstorage.GetMetricsMetadata(qt, tt, denyPartialResponse, limit, metricName, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot get metadata: %w", err)
 	}
@@ -881,7 +881,7 @@ func getSearchQuery(qt *querytracer.Tracer, at *auth.Token, cp *commonParams, ma
 	if at != nil {
 		return storage.NewSearchQuery(at.AccountID, at.ProjectID, cp.start, cp.end, cp.filterss, maxSeries), nil
 	}
-	tt, tfs, err := netstorage.GetTenantTokensFromFilters(cp.ctx, qt, storage.TimeRange{MinTimestamp: cp.start, MaxTimestamp: cp.end}, cp.filterss, true)
+	tt, tfs, err := netstorage.GetTenantTokensFromFilters(qt, storage.TimeRange{MinTimestamp: cp.start, MaxTimestamp: cp.end}, cp.filterss, cp.deadline, true)
 	if err != nil {
 		return nil, fmt.Errorf("cannot obtain tenant tokens: %w", err)
 	}
@@ -897,9 +897,9 @@ func SeriesCountHandler(startTime time.Time, at *auth.Token, w http.ResponseWrit
 	if at == nil {
 		return fmt.Errorf("multi-tenant request to /api/v1/series/count is not supported")
 	}
-	ctx := searchutil.GetContextForStatusRequest(r, startTime)
+	deadline := searchutil.GetDeadlineForStatusRequest(r, startTime)
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
-	n, isPartial, err := netstorage.SeriesCount(ctx, nil, at.AccountID, at.ProjectID, denyPartialResponse)
+	n, isPartial, err := netstorage.SeriesCount(nil, at.AccountID, at.ProjectID, denyPartialResponse, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %w", err)
 	}
@@ -940,7 +940,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token,
 		return err
 	}
 	denyPartialResponse := httputil.GetDenyPartialResponse(r)
-	metricNames, isPartial, err := netstorage.SearchMetricNames(cp.ctx, qt, denyPartialResponse, sq)
+	metricNames, isPartial, err := netstorage.SearchMetricNames(qt, denyPartialResponse, sq, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
 	}
@@ -966,7 +966,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w
 	defer queryDuration.UpdateDuration(startTime)
 
 	ct := startTime.UnixNano() / 1e6
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	noCache := httputil.GetBool(r, "nocache")
 	query := r.FormValue("query")
 	if len(query) == 0 {
@@ -1018,7 +1018,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w
 		filterss := searchutil.JoinTagFilterss(tagFilterss, etfs)
 
 		cp := &commonParams{
-			ctx:      ctx,
+			deadline: deadline,
 			start:    start,
 			end:      end,
 			filterss: filterss,
@@ -1067,13 +1067,13 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w
 		queryOffset = 0
 	}
 	ec := &promql.EvalConfig{
-		Context:             ctx,
 		Start:               start,
 		End:                 start,
 		Step:                step,
 		MaxPointsPerSeries:  *maxPointsPerTimeseries,
 		MaxSeries:           *maxUniqueTimeseries,
 		QuotedRemoteAddr:    httpserver.GetQuotedRemoteAddr(r),
+		Deadline:            deadline,
 		NoCache:             noCache,
 		LookbackDelta:       lookbackDelta,
 		RoundDigits:         getRoundDigits(r),
@@ -1085,7 +1085,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w
 
 		DenyPartialResponse: httputil.GetDenyPartialResponse(r),
 	}
-	err = populateAuthTokens(ctx, qt, ec, at)
+	err = populateAuthTokens(qt, ec, at, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot populate auth tokens: %w", err)
 	}
@@ -1169,7 +1169,7 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok
 
 func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Token, w http.ResponseWriter, query string,
 	start, end, step, lookbackDelta int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	noCache := httputil.GetBool(r, "nocache")
 	optimizeRepeatedBinaryOpSubexprs := httputil.GetBool(r, "optimize_repeated_binary_op_subexprs")
 	if start > end {
@@ -1183,13 +1183,13 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok
 	}
 
 	ec := &promql.EvalConfig{
-		Context:                          ctx,
 		Start:                            start,
 		End:                              end,
 		Step:                             step,
 		MaxPointsPerSeries:               *maxPointsPerTimeseries,
 		MaxSeries:                        *maxUniqueTimeseries,
 		QuotedRemoteAddr:                 httpserver.GetQuotedRemoteAddr(r),
+		Deadline:                         deadline,
 		NoCache:                          noCache,
 		OptimizeRepeatedBinaryOpSubexprs: optimizeRepeatedBinaryOpSubexprs,
 		LookbackDelta:                    lookbackDelta,
@@ -1202,8 +1202,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok
 
 		DenyPartialResponse: httputil.GetDenyPartialResponse(r),
 	}
-
-	if err := populateAuthTokens(ctx, qt, ec, at); err != nil {
+	if err := populateAuthTokens(qt, ec, at, deadline); err != nil {
 		return fmt.Errorf("cannot populate auth tokens: %w", err)
 	}
 	qs := promql.NewQueryStats(query, at, ec)
@@ -1241,13 +1240,13 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, at *auth.Tok
 	return nil
 }
 
-func populateAuthTokens(ctx searchutil.Context, qt *querytracer.Tracer, ec *promql.EvalConfig, at *auth.Token) error {
+func populateAuthTokens(qt *querytracer.Tracer, ec *promql.EvalConfig, at *auth.Token, deadline searchutil.Deadline) error {
 	if at != nil {
 		ec.AuthTokens = []*auth.Token{at}
 		return nil
 	}
 
-	tt, tfs, err := netstorage.GetTenantTokensFromFilters(ctx, qt, storage.TimeRange{MinTimestamp: ec.Start, MaxTimestamp: ec.End}, ec.EnforcedTagFilterss, ec.MayCache())
+	tt, tfs, err := netstorage.GetTenantTokensFromFilters(qt, storage.TimeRange{MinTimestamp: ec.Start, MaxTimestamp: ec.End}, ec.EnforcedTagFilterss, deadline, ec.MayCache())
 	if err != nil {
 		return fmt.Errorf("cannot obtain tenant tokens for the given search query: %w", err)
 	}
@@ -1418,7 +1417,7 @@ func QueryStatsHandler(at *auth.Token, w http.ResponseWriter, r *http.Request) e
 //
 // timeout, start, end, match[], extra_label, extra_filters[]
 type commonParams struct {
-	ctx              searchutil.Context
+	deadline         searchutil.Deadline
 	start            int64
 	end              int64
 	currentTimestamp int64
@@ -1442,7 +1441,7 @@ func getExportParams(r *http.Request, startTime time.Time) (*commonParams, error
 	if err != nil {
 		return nil, err
 	}
-	cp.ctx = searchutil.GetContextForExport(r, startTime)
+	cp.deadline = searchutil.GetDeadlineForExport(r, startTime)
 	return cp, nil
 }
 
@@ -1454,7 +1453,7 @@ func getCommonParamsForLabelsAPI(r *http.Request, startTime time.Time, requireNo
 	if cp.start == 0 {
 		cp.start = cp.end - defaultStep
 	}
-	cp.ctx = searchutil.GetContextForLabelsAPI(r, startTime)
+	cp.deadline = searchutil.GetDeadlineForLabelsAPI(r, startTime)
 	return cp, nil
 }
 
@@ -1471,7 +1470,7 @@ func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch
 }
 
 func getCommonParamsInternal(r *http.Request, startTime time.Time, requireNonEmptyMatch, isLabelsAPI bool) (*commonParams, error) {
-	ctx := searchutil.GetContextForQuery(r, startTime)
+	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	start, err := httputil.GetTime(r, "start", 0)
 	if err != nil {
 		return nil, err
@@ -1511,7 +1510,7 @@ func getCommonParamsInternal(r *http.Request, startTime time.Time, requireNonEmp
 	}
 
 	cp := &commonParams{
-		ctx:              ctx,
+		deadline:         deadline,
 		start:            start,
 		end:              end,
 		currentTimestamp: ct,

app/vmselect/promql/eval.go

@@ -114,10 +114,7 @@ func alignStartEnd(start, end, step int64) (int64, int64) {
 
 // EvalConfig is the configuration required for query evaluation via Exec
 type EvalConfig struct {
-	Context searchutil.Context
-
-	AuthTokens []*auth.Token
-
+	AuthTokens    []*auth.Token
 	IsMultiTenant bool
 
 	Start int64
@@ -134,6 +131,8 @@ type EvalConfig struct {
 	// QuotedRemoteAddr contains quoted remote address.
 	QuotedRemoteAddr string
 
+	Deadline searchutil.Deadline
+
 	// Whether the response must not be cached.
 	NoCache bool
 
@@ -178,7 +177,6 @@ type EvalConfig struct {
 // copyEvalConfig returns src copy.
 func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	var ec EvalConfig
-	ec.Context = src.Context
 	ec.AuthTokens = src.AuthTokens
 	ec.IsMultiTenant = src.IsMultiTenant
 	ec.Start = src.Start
@@ -186,6 +184,7 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.Step = src.Step
 	ec.MaxSeries = src.MaxSeries
 	ec.MaxPointsPerSeries = src.MaxPointsPerSeries
+	ec.Deadline = src.Deadline
 	ec.NoCache = src.NoCache
 	ec.OptimizeRepeatedBinaryOpSubexprs = src.OptimizeRepeatedBinaryOpSubexprs
 	ec.LookbackDelta = src.LookbackDelta
@@ -1878,7 +1877,7 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	} else {
 		sq = storage.NewSearchQuery(ec.AuthTokens[0].AccountID, ec.AuthTokens[0].ProjectID, minTimestamp, ec.End, tfss, ec.MaxSeries)
 	}
-	rss, isPartial, err := netstorage.ProcessSearchQuery(ec.Context, qt, ec.DenyPartialResponse, sq)
+	rss, isPartial, err := netstorage.ProcessSearchQuery(qt, ec.DenyPartialResponse, sq, ec.Deadline)
 	if err != nil {
 		return nil, err
 	}
@@ -1948,9 +1947,9 @@ func evalRollupFuncNoCache(qt *querytracer.Tracer, ec *EvalConfig, funcName stri
 	// Evaluate rollup
 	keepMetricNames := getKeepMetricNames(expr)
 	if iafc != nil {
-		return evalRollupWithIncrementalAggregate(ec.Context, qt, funcName, keepMetricNames, iafc, rss, rcs, preFunc, sharedTimestamps)
+		return evalRollupWithIncrementalAggregate(qt, funcName, keepMetricNames, iafc, rss, rcs, preFunc, sharedTimestamps)
 	}
-	return evalRollupNoIncrementalAggregate(ec.Context, qt, funcName, keepMetricNames, rss, rcs, preFunc, sharedTimestamps)
+	return evalRollupNoIncrementalAggregate(qt, funcName, keepMetricNames, rss, rcs, preFunc, sharedTimestamps)
 }
 
 var (
@@ -1973,14 +1972,14 @@ func maxSilenceInterval() int64 {
 	return d
 }
 
-func evalRollupWithIncrementalAggregate(ctx searchutil.Context, qt *querytracer.Tracer, funcName string, keepMetricNames bool,
+func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool,
 	iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
 ) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs)
 	defer qt.Done()
 	var samplesScannedTotal atomic.Uint64
-	err := rss.RunParallel(ctx, qt, func(rs *netstorage.Result, workerID uint) error {
+	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
 		ts := getTimeseries()
@@ -2014,7 +2013,7 @@ func evalRollupWithIncrementalAggregate(ctx searchutil.Context, qt *querytracer.
 	return tss, nil
 }
 
-func evalRollupNoIncrementalAggregate(ctx searchutil.Context, qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
+func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig,
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64,
 ) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
@@ -2024,7 +2023,7 @@ func evalRollupNoIncrementalAggregate(ctx searchutil.Context, qt *querytracer.Tr
 	tsw := getTimeseriesByWorkerID()
 	seriesByWorkerID := tsw.byWorkerID
 	seriesLen := rss.Len()
-	err := rss.RunParallel(ctx, qt, func(rs *netstorage.Result, workerID uint) error {
+	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
 		for _, rc := range rcs {

app/vmselect/promql/exec_test.go

@@ -1,7 +1,6 @@
 package promql
 
 import (
-	"context"
 	"math"
 	"testing"
 	"time"
@@ -66,7 +65,6 @@ func TestExecSuccess(t *testing.T) {
 	f := func(q string, resultExpected []netstorage.Result) {
 		t.Helper()
 		ec := &EvalConfig{
-			Context: searchutil.NewContext(context.Background(), searchutil.NewDeadline(time.Now(), time.Minute, "")),
 			AuthTokens: []*auth.Token{{
 				AccountID: accountID,
 				ProjectID: projectID,
@@ -77,6 +75,7 @@ func TestExecSuccess(t *testing.T) {
 			Step:               step,
 			MaxPointsPerSeries: 1e4,
 			MaxSeries:          1000,
+			Deadline:           searchutil.NewDeadline(time.Now(), time.Minute, ""),
 			RoundDigits:        100,
 		}
 		for range 5 {
@@ -10468,7 +10467,6 @@ func TestExecError(t *testing.T) {
 	f := func(q string) {
 		t.Helper()
 		ec := &EvalConfig{
-			Context: searchutil.NewContext(context.Background(), searchutil.NewDeadline(time.Now(), time.Minute, "")),
 			AuthTokens: []*auth.Token{{
 				AccountID: 123,
 				ProjectID: 567,
@@ -10478,6 +10476,7 @@ func TestExecError(t *testing.T) {
 			Step:               100,
 			MaxPointsPerSeries: 1e4,
 			MaxSeries:          1000,
+			Deadline:           searchutil.NewDeadline(time.Now(), time.Minute, ""),
 			RoundDigits:        100,
 		}
 		for range 4 {

app/vmselect/promql/transform.go

@@ -2,6 +2,7 @@ package promql
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"math"
 	"math/rand"
@@ -2566,6 +2567,11 @@ func isDecimalChar(ch byte) bool {
 func mustParseNum(s string) float64 {
 	f, err := strconv.ParseFloat(s, 64)
 	if err != nil {
+		if errors.Is(err, strconv.ErrRange) {
+			// The number is too large to fit into float64; ParseFloat returns ±Inf in this case.
+			// Use ±Inf for sorting purposes — it is semantically correct.
+			return f
+		}
 		logger.Panicf("BUG: unexpected error when parsing the number %q: %s", s, err)
 	}
 	return f

app/vmselect/promql/transform_test.go

@@ -386,4 +386,12 @@ func TestNumericLess(t *testing.T) {
 	f("12.9", "12.56", false)
 	f("12.56", "12.9", true)
 	f("12.9", "12.9", false)
+
+	// 309-digit numbers - must not panic (regression test for GHSA-9g98-8jgr-x2vv)
+	big := strings.Repeat("9", 309)
+	f(big, "1", false)
+	f("1", big, true)
+	f(big, big, false)
+	f("-"+big, big, true)
+	f(big, "-"+big, false)
 }

app/vmselect/searchutil/searchutil.go

@@ -1,7 +1,6 @@
 package searchutil
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"net/http"
@@ -39,39 +38,34 @@ func GetMaxQueryDuration(r *http.Request) time.Duration {
 	return d
 }
 
-// GetDeadlineForQuery returns context for the given query r.
-func GetContextForQuery(r *http.Request, startTime time.Time) Context {
+// GetDeadlineForQuery returns deadline for the given query r.
+func GetDeadlineForQuery(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxQueryDuration.Milliseconds()
-	deadline := getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxQueryDuration")
-	return NewContext(r.Context(), deadline)
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxQueryDuration")
 }
 
-// GetContextForStatusRequest returns context for the given request to /api/v1/status/*.
-func GetContextForStatusRequest(r *http.Request, startTime time.Time) Context {
+// GetDeadlineForStatusRequest returns deadline for the given request to /api/v1/status/*.
+func GetDeadlineForStatusRequest(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxStatusRequestDuration.Milliseconds()
-	deadline := getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxStatusRequestDuration")
-	return NewContext(r.Context(), deadline)
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxStatusRequestDuration")
 }
 
-// GetContextForExport returns context for the given request to /api/v1/export.
-func GetContextForExport(r *http.Request, startTime time.Time) Context {
+// GetDeadlineForExport returns deadline for the given request to /api/v1/export.
+func GetDeadlineForExport(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxExportDuration.Milliseconds()
-	deadline := getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxExportDuration")
-	return NewContext(r.Context(), deadline)
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxExportDuration")
 }
 
-// GetContextForLabelsAPI returns context for the given request to /api/v1/labels, /api/v1/label/.../values or /api/v1/series
-func GetContextForLabelsAPI(r *http.Request, startTime time.Time) Context {
+// GetDeadlineForLabelsAPI returns deadline for the given request to /api/v1/labels, /api/v1/label/.../values or /api/v1/series
+func GetDeadlineForLabelsAPI(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxLabelsAPIDuration.Milliseconds()
-	deadline := getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxLabelsAPIDuration")
-	return NewContext(r.Context(), deadline)
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxLabelsAPIDuration")
 }
 
-// GetDeadlineForDelete returns context for the given request to /api/v1/admin/tsdb/delete_series.
-func GetContextForDelete(r *http.Request, startTime time.Time) Context {
+// GetDeadlineForDelete returns deadline for the given request to /api/v1/admin/tsdb/delete_series.
+func GetDeadlineForDelete(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxDeleteDuration.Milliseconds()
-	deadline := getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxDeleteDuration")
-	return NewContext(r.Context(), deadline)
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxDeleteDuration")
 }
 
 func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) Deadline {
@@ -86,68 +80,6 @@ func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64
 	return NewDeadline(startTime, timeout, flagHint)
 }
 
-// Context defines search context with deadline
-type Context struct {
-	parent   context.Context
-	deadline Deadline
-}
-
-// NewContext return new context for given parent context and deadline
-func NewContext(ctx context.Context, deadline Deadline) Context {
-	if ctx == nil {
-		ctx = context.Background()
-	}
-	return Context{
-		parent:   ctx,
-		deadline: deadline,
-	}
-}
-
-// NewContextWithDeadlineTimestamp return new context for given parent context and timestamp of deadline
-func NewContextWithDeadlineTimestamp(ctx context.Context, timestamp uint64) Context {
-	deadline := DeadlineFromTimestamp(timestamp)
-	return Context{
-		parent:   ctx,
-		deadline: deadline,
-	}
-}
-
-// Deadline returns context deadline
-func (ctx *Context) Deadline() Deadline {
-	return ctx.deadline
-}
-
-// IsDone returns true if context is cancelled or deadline exceeded
-func (ctx *Context) IsDone() bool {
-	if ctx.deadline.Exceeded() {
-		return true
-	}
-	if ctx.canceled() {
-		return true
-	}
-	return false
-}
-
-func (ctx *Context) canceled() bool {
-	select {
-	case <-ctx.parent.Done():
-		return true
-	default:
-		return false
-	}
-}
-
-// Err return context error if there is any
-func (ctx *Context) Err() error {
-	if ctx.deadline.Exceeded() {
-		return fmt.Errorf("context deadline timeout: %s: %w", ctx.deadline.String(), context.DeadlineExceeded)
-	}
-	if ctx.canceled() {
-		return ctx.parent.Err()
-	}
-	return nil
-}
-
 // Deadline contains deadline with the corresponding timeout for pretty error messages.
 type Deadline struct {
 	deadline uint64

app/vmselect/searchutil/searchutil_test.go

@@ -276,31 +276,27 @@ func tagFiltersToString(tfs []storage.TagFilter) string {
 	return string(b)
 }
 
-func TestGetContextDeadline(t *testing.T) {
-	f := func(got Context, exp Deadline) {
-		t.Helper()
-		// got is a function parameter and therefore addressable,
-		// so the pointer-receiver Deadline() method resolves here.
-		gotDeadline := got.Deadline()
-		if gotDeadline.Deadline() != exp.Deadline() {
-			t.Fatalf("expected deadline %d; got %d instead", exp.Deadline(), gotDeadline.Deadline())
+func TestGetDeadline(t *testing.T) {
+	f := func(got, exp Deadline) {
+		if got.Deadline() != exp.Deadline() {
+			t.Fatalf("expected to have %v; got %v instead", exp, got)
 		}
 	}
 
 	start := time.Now()
-	expDeadline := func(d time.Duration) Deadline {
-		return NewDeadline(start, d, "")
+	expDeadline := func(deadline time.Duration) Deadline {
+		return NewDeadline(start, deadline, "")
 	}
 
 	r, _ := http.NewRequest("GET", "", nil)
-	f(GetContextForExport(r, start), expDeadline(*maxExportDuration))
-	f(GetContextForLabelsAPI(r, start), expDeadline(*maxLabelsAPIDuration))
-	f(GetContextForStatusRequest(r, start), expDeadline(*maxStatusRequestDuration))
-	f(GetContextForQuery(r, start), expDeadline(*maxQueryDuration))
+	f(GetDeadlineForExport(r, start), expDeadline(*maxExportDuration))
+	f(GetDeadlineForLabelsAPI(r, start), expDeadline(*maxLabelsAPIDuration))
+	f(GetDeadlineForStatusRequest(r, start), expDeadline(*maxStatusRequestDuration))
+	f(GetDeadlineForQuery(r, start), expDeadline(*maxQueryDuration))
 
 	r, _ = http.NewRequest("GET", "http://foo?timeout=1s", nil)
-	f(GetContextForExport(r, start), expDeadline(time.Second))
-	f(GetContextForLabelsAPI(r, start), expDeadline(time.Second))
-	f(GetContextForStatusRequest(r, start), expDeadline(time.Second))
-	f(GetContextForQuery(r, start), expDeadline(time.Second))
+	f(GetDeadlineForExport(r, start), expDeadline(time.Second))
+	f(GetDeadlineForLabelsAPI(r, start), expDeadline(time.Second))
+	f(GetDeadlineForStatusRequest(r, start), expDeadline(time.Second))
+	f(GetDeadlineForQuery(r, start), expDeadline(time.Second))
 }

app/vmselect/stats/stats.go

@@ -44,7 +44,7 @@ func MetricNamesStatsHandler(startTime time.Time, at *auth.Token, qt *querytrace
 			return fmt.Errorf("match_pattern=%q must be valid regex: %w", matchPattern, err)
 		}
 	}
-	ctx := searchutil.GetContextForStatusRequest(r, startTime)
+	deadline := searchutil.GetDeadlineForStatusRequest(r, startTime)
 	var tt *storage.TenantToken
 	if at != nil {
 		tt = &storage.TenantToken{
@@ -52,7 +52,7 @@ func MetricNamesStatsHandler(startTime time.Time, at *auth.Token, qt *querytrace
 			ProjectID: at.ProjectID,
 		}
 	}
-	stats, err := netstorage.GetMetricNamesStats(ctx, qt, tt, limit, le, matchPattern)
+	stats, err := netstorage.GetMetricNamesStats(qt, tt, limit, le, matchPattern, deadline)
 	if err != nil {
 		return err
 	}
@@ -62,8 +62,8 @@ func MetricNamesStatsHandler(startTime time.Time, at *auth.Token, qt *querytrace
 
 // ResetMetricNamesStatsHandler resets metric names usage state
 func ResetMetricNamesStatsHandler(startTime time.Time, qt *querytracer.Tracer, r *http.Request) error {
-	ctx := searchutil.GetContextForStatusRequest(r, startTime)
-	if err := netstorage.ResetMetricNamesStats(ctx, qt); err != nil {
+	deadline := searchutil.GetDeadlineForStatusRequest(r, startTime)
+	if err := netstorage.ResetMetricNamesStats(qt, deadline); err != nil {
 		return err
 	}
 	return nil

app/vmstorage/main.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/VictoriaMetrics/metrics"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -53,7 +54,7 @@ var (
 		"Configured value must always be lower than the graceful shutdown period configured by the orchestration platform (terminationGracePeriodSeconds for Kubernetes). "+
 		"See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#improving-re-routing-performance-during-restart")
 	vmselectAddr                  = flag.String("vmselectAddr", ":8401", "TCP address to accept connections from vmselect services")
-	vmselectMaxConcurrentRequests = flag.Int("search.maxConcurrentRequests", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent vmselect requests "+
+	vmselectMaxConcurrentRequests = flagutil.NewIntWithDynamicDefault("search.maxConcurrentRequests", 2*cgroup.AvailableCPUs(), "2*cgroup.AvailableCPUs()", "The maximum number of concurrent vmselect requests "+
 		"the vmstorage can process at -vmselectAddr. It shouldn't be high, since a single request usually saturates a CPU core, and many concurrently executed requests "+
 		"may require high amounts of memory. See also -search.maxQueueDuration")
 	vmselectMaxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the incoming vmselect request waits for execution "+
@@ -211,6 +212,7 @@ func main() {
 	storageMetrics := metrics.NewSet()
 	storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
 	metrics.RegisterSet(storageMetrics)
+	appmetrics.MustCreateUncleanShutdownMarker(*storageDataPath)
 
 	protoparserutil.StartUnmarshalWorkers()
 
@@ -265,6 +267,7 @@ func main() {
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())
 
 	fs.MustStopDirRemover()
+	appmetrics.MustRemoveUncleanShutdownMarker(*storageDataPath)
 	logger.Infof("the vmstorage has been stopped")
 }
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -1,4 +1,4 @@
-import { FC, useRef } from "preact/compat";
+import { forwardRef, useImperativeHandle, useRef } from "preact/compat";
 import ServerConfigurator from "./ServerConfigurator/ServerConfigurator";
 import { ArrowDownIcon, SettingsIcon } from "../../Main/Icons";
 import Button from "../../Main/Button/Button";
@@ -21,7 +21,11 @@ export interface ChildComponentHandle {
   handleApply: () => void;
 }
 
-const GlobalSettings: FC = () => {
+export interface GlobalSettingsHandle {
+  open: () => void;
+}
+
+const GlobalSettings = forwardRef<GlobalSettingsHandle>((_, ref) => {
   const { isMobile } = useDeviceDetect();
 
   const appModeEnable = getAppModeEnable();
@@ -74,6 +78,10 @@ const GlobalSettings: FC = () => {
     },
   ].filter(control => control.show);
 
+  useImperativeHandle(ref, () => ({
+    open: handleOpen,
+  }));
+
   return <>
     {isMobile ? (
       <div
@@ -139,6 +147,6 @@ const GlobalSettings: FC = () => {
       </Modal>
     )}
   </>;
-};
+});
 
 export default GlobalSettings;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TimeZonePreview/TimeZonePreview.tsx

@@ -0,0 +1,52 @@
+import { FC } from "preact/compat";
+import Button from "../../../Main/Button/Button";
+import { useTimeState } from "../../../../state/time/TimeStateContext";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import { getUTCByTimezone } from "../../../../utils/time";
+import { useMemo } from "react";
+import { ArrowDownIcon, PlanetIcon } from "../../../Main/Icons";
+
+type Props = {
+  onOpenSettings?: () => void;
+}
+
+const TimeZonePreview: FC<Props> = ({ onOpenSettings }) => {
+  const { isMobile } = useDeviceDetect();
+
+  const { timezone } = useTimeState();
+  const utcOffset = useMemo(() => getUTCByTimezone(timezone), [timezone]);
+
+  const handleOpenSettings = () => {
+    onOpenSettings && onOpenSettings();
+  };
+
+
+  if (isMobile) {
+    return (
+      <button
+        className="vm-mobile-option"
+        onClick={handleOpenSettings}
+      >
+        <span className="vm-mobile-option__icon"><PlanetIcon/></span>
+        <div className="vm-mobile-option-text">
+          <span className="vm-mobile-option-text__label">Time zone</span>
+          <span className="vm-mobile-option-text__value">{utcOffset}</span>
+        </div>
+        <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
+      </button>
+    );
+  }
+
+  return (
+    <Button
+      className="vm-header-button"
+      onClick={handleOpenSettings}
+      startIcon={<PlanetIcon/>}
+    >
+      {utcOffset}
+    </Button>
+  );
+};
+
+
+export default TimeZonePreview;

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -113,6 +113,8 @@ const StepConfigurator: FC = () => {
     setError("");
   }, [defaultStep, prevDefaultStep, value, graphDispatch]);
 
+  const textValue = isAutoStep ? `auto (${customStep})` : customStep;
+
   return (
     <div
       className="vm-step-control"
@@ -126,7 +128,7 @@ const StepConfigurator: FC = () => {
           <span className="vm-mobile-option__icon"><TimelineIcon/></span>
           <div className="vm-mobile-option-text">
             <span className="vm-mobile-option-text__label">Step</span>
-            <span className="vm-mobile-option-text__value">{customStep}</span>
+            <span className="vm-mobile-option-text__value">{textValue}</span>
           </div>
           <span className="vm-mobile-option__arrow"><ArrowDownIcon/></span>
         </div>
@@ -138,7 +140,7 @@ const StepConfigurator: FC = () => {
           startIcon={<TimelineIcon/>}
           onClick={toggleOpenOptions}
         >
-          Step: {isAutoStep ? `auto (${customStep})` : customStep}
+          Step: {textValue}
         </Button>
       )}
       <Popper

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector.tsx

@@ -19,7 +19,11 @@ import useBoolean from "../../../../hooks/useBoolean";
 import useWindowSize from "../../../../hooks/useWindowSize";
 import usePrevious from "../../../../hooks/usePrevious";
 
-export const TimeSelector: FC = () => {
+type Props = {
+  onOpenSettings?: () => void;
+}
+
+export const TimeSelector: FC<Props> = ({ onOpenSettings }) => {
   const { isMobile } = useDeviceDetect();
   const { isDarkTheme } = useAppState();
   const wrapperRef = useRef<HTMLDivElement>(null);
@@ -53,7 +57,7 @@ export const TimeSelector: FC = () => {
     setFrom(formatDateForNativeInput(dateFromSeconds(start)));
   }, [timezone, start]);
 
-  const setDuration = ({ duration, until, id }: {duration: string, until: Date, id: string}) => {
+  const setDuration = ({ duration, until, id }: { duration: string, until: Date, id: string }) => {
     dispatch({ type: "SET_RELATIVE_TIME", payload: { duration, until, id } });
     handleCloseOptions();
   };
@@ -75,16 +79,23 @@ export const TimeSelector: FC = () => {
 
   const setTimeAndClosePicker = () => {
     if (from && until) {
-      dispatch({ type: "SET_PERIOD", payload: {
-        from: dayjs.tz(from).toDate(),
-        to: dayjs.tz(until).toDate()
-      } });
+      dispatch({
+        type: "SET_PERIOD", payload: {
+          from: dayjs.tz(from).toDate(),
+          to: dayjs.tz(until).toDate()
+        }
+      });
     }
     handleCloseOptions();
   };
 
   const onSwitchToNow = () => dispatch({ type: "RUN_QUERY_TO_NOW" });
 
+  const handleOpenSettings = () => {
+    onOpenSettings && onOpenSettings();
+    handleCloseOptions();
+  };
+
   const onCancelClick = () => {
     setUntil(formatDateForNativeInput(dateFromSeconds(end)));
     setFrom(formatDateForNativeInput(dateFromSeconds(start)));
@@ -140,6 +151,7 @@ export const TimeSelector: FC = () => {
         </Tooltip>
       )}
     </div>
+
     <Popper
       open={openOptions}
       buttonRef={buttonRef}
@@ -179,13 +191,17 @@ export const TimeSelector: FC = () => {
               onEnter={setTimeAndClosePicker}
             />
           </div>
-          <div className="vm-time-selector-left-timezone">
-            <div className="vm-time-selector-left-timezone__title">{activeTimezone.region}</div>
-            <div className="vm-time-selector-left-timezone__utc">{activeTimezone.utc}</div>
-          </div>
+          <button
+            type="button"
+            className="vm-time-selector-left-timezone"
+            onClick={handleOpenSettings}
+          >
+            <span className="vm-time-selector-left-timezone__title">{activeTimezone.region}</span>
+            <span className="vm-time-selector-left-timezone__utc">{activeTimezone.utc}</span>
+          </button>
           <Button
             variant="text"
-            startIcon={<AlarmIcon />}
+            startIcon={<AlarmIcon/>}
             onClick={onSwitchToNow}
           >
             switch to now

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/style.scss

@@ -40,8 +40,13 @@
       gap: $padding-small;
       font-size: $font-size-small;
       margin-bottom: $padding-small;
+      color: $color-text;
+      cursor: pointer;
 
-      &__title {}
+      &:hover {
+        color: $color-primary;
+        text-decoration: underline;
+      }
 
       &__utc {
         display: inline-flex;

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -634,6 +634,17 @@ export const DebugIcon = () => (
   </svg>
 );
 
+export const PlanetIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2M4 12c0-.61.08-1.21.21-1.78L8.99 15v1c0 1.1.9 2 2 2v1.93C7.06 19.43 4 16.07 4 12m13.89 5.4c-.26-.81-1-1.4-1.9-1.4h-1v-3c0-.55-.45-1-1-1h-6v-2h2c.55 0 1-.45 1-1V7h2c1.1 0 2-.9 2-2v-.41C17.92 5.77 20 8.65 20 12c0 2.08-.81 3.98-2.11 5.4"
+    ></path>
+  </svg>
+);
+
 export const SystemIcon = () => (
   <svg
     viewBox="0 0 24 24"

app/vmui/packages/vmui/src/layouts/Header/HeaderControls/style.scss

@@ -11,6 +11,7 @@
   &_mobile {
     display: grid;
     grid-template-columns: 1fr;
+    gap: 0;
     padding: 0;
     flex-grow: initial;
 

app/vmui/packages/vmui/src/layouts/MainLayout/ControlsMainLayout.tsx

@@ -6,9 +6,11 @@ import StepConfigurator from "../../components/Configurators/StepConfigurator/St
 import { TimeSelector } from "../../components/Configurators/TimeRangeSettings/TimeSelector/TimeSelector";
 import CardinalityDatePicker from "../../components/Configurators/CardinalityDatePicker/CardinalityDatePicker";
 import { ExecutionControls } from "../../components/Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls";
-import GlobalSettings from "../../components/Configurators/GlobalSettings/GlobalSettings";
+import GlobalSettings, { GlobalSettingsHandle } from "../../components/Configurators/GlobalSettings/GlobalSettings";
 import ShortcutKeys from "../../components/Main/ShortcutKeys/ShortcutKeys";
 import { ControlsProps } from "../Header/HeaderControls/HeaderControls";
+import { useRef } from "react";
+import TimeZonePreview from "../../components/Configurators/GlobalSettings/TimeZonePreview/TimeZonePreview";
 
 const ControlsMainLayout: FC<ControlsProps> = ({
   displaySidebar,
@@ -17,6 +19,7 @@ const ControlsMainLayout: FC<ControlsProps> = ({
   accountIds,
   closeModal,
 }) => {
+  const settingsRef = useRef<GlobalSettingsHandle>(null);
 
   return (
     <div
@@ -27,14 +30,15 @@ const ControlsMainLayout: FC<ControlsProps> = ({
     >
       {headerSetup?.tenant && <TenantsConfiguration accountIds={accountIds || []}/>}
       {headerSetup?.stepControl && <StepConfigurator/>}
-      {headerSetup?.timeSelector && <TimeSelector/>}
+      {headerSetup?.timeSelector && <TimeSelector onOpenSettings={() => settingsRef.current?.open()}/>}
       {headerSetup?.cardinalityDatePicker && <CardinalityDatePicker/>}
+      <TimeZonePreview onOpenSettings={() => settingsRef.current?.open()}/>
       {headerSetup?.executionControls && <ExecutionControls
         tooltip={headerSetup?.executionControls?.tooltip}
         useAutorefresh={headerSetup?.executionControls?.useAutorefresh}
         closeModal={closeModal}
       />}
-      <GlobalSettings/>
+      <GlobalSettings ref={settingsRef}/>
       {!displaySidebar && <ShortcutKeys/>}
     </div>
   );

app/vmui/packages/vmui/src/styles/components/mobile-option.scss

@@ -1,11 +1,12 @@
 @use "src/styles/variables" as *;
 
 .vm-mobile-option {
-  display: flex;
+  display: grid;
+  grid-template-columns: auto 1fr auto;
   align-items: center;
   justify-content: flex-start;
-  gap: $padding-small;
-  padding: calc($padding-medium/2) 0;
+  gap: $padding-global;
+  padding: $padding-global $padding-small;
   width: 100%;
   user-select: none;
 
@@ -17,14 +18,33 @@
   }
 
   &__icon {
-    width: 22px;
-    height: 22px;
+    position: relative;
+    display: flex;
+    width: 40px;
+    height: 40px;
     color: $color-primary;
+
+    &:after {
+      content: "";
+      position: absolute;
+      top: 0;
+      left: 0;
+      width: 100%;
+      height: 100%;
+      opacity: 0.1;
+      background-color: currentColor;
+      border-radius: $border-radius-medium;
+    }
+
+    svg {
+      width: 21px;
+      height: auto;
+    }
   }
 
   &__arrow {
-    width: 14px;
-    height: 14px;
+    width: 20px;
+    height: 20px;
     transform: rotate(-90deg);
     color: $color-primary;
   }
@@ -32,11 +52,13 @@
   &-text {
     display: grid;
     align-items: center;
-    gap: 2px;
+    height: 100%;
+    gap: calc($padding-small / 2);
     flex-grow: 1;
+    text-align: left;
 
     &__label {
-      font-weight: bold;
+      font-weight: 600;
     }
 
     &__value {

dashboards/backupmanager.json

@@ -59,6 +59,19 @@
         },
         "type": "dashboard"
       },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds"
+        },
+        "enable": true,
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
+        "hide": true,
+        "iconColor": "dark-blue",
+        "name": "version",
+        "textFormat": "{{version}}",
+        "titleFormat": "Version change"
+      },
       {
         "datasource": {
           "type": "prometheus",

dashboards/victoriametrics-cluster.json

@@ -37,7 +37,7 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version change",

dashboards/victoriametrics.json

@@ -37,7 +37,7 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version",

dashboards/vm/backupmanager.json

@@ -60,6 +60,19 @@
         },
         "type": "dashboard"
       },
+      {
+        "datasource": {
+          "type": "victoriametrics-metrics-datasource",
+          "uid": "$ds"
+        },
+        "enable": true,
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
+        "hide": true,
+        "iconColor": "dark-blue",
+        "name": "version",
+        "textFormat": "{{version}}",
+        "titleFormat": "Version change"
+      },
       {
         "datasource": {
           "type": "victoriametrics-metrics-datasource",

dashboards/vm/victoriametrics-cluster.json

@@ -38,7 +38,7 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version change",

dashboards/vm/victoriametrics.json

@@ -38,7 +38,7 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version",

dashboards/vm/vmagent.json

@@ -26,11 +26,11 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version",
-        "textFormat": "{{short_version}}",
+        "textFormat": "{{version}}",
         "titleFormat": "Version change"
       },
       {

dashboards/vm/vmalert.json

@@ -26,11 +26,11 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version",
-        "textFormat": "{{short_version}}",
+        "textFormat": "{{version}}",
         "titleFormat": "Version change"
       },
       {

dashboards/vm/vmauth.json

@@ -54,6 +54,19 @@
         },
         "type": "dashboard"
       },
+      {
+        "datasource": {
+          "type": "victoriametrics-metrics-datasource",
+          "uid": "$ds"
+        },
+        "enable": true,
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
+        "hide": true,
+        "iconColor": "dark-blue",
+        "name": "version",
+        "textFormat": "{{version}}",
+        "titleFormat": "Version change"
+      },
       {
         "datasource": {
           "type": "victoriametrics-metrics-datasource",

dashboards/vmagent.json

@@ -25,11 +25,11 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version",
-        "textFormat": "{{short_version}}",
+        "textFormat": "{{version}}",
         "titleFormat": "Version change"
       },
       {

dashboards/vmalert.json

@@ -25,11 +25,11 @@
           "uid": "$ds"
         },
         "enable": true,
-        "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(short_version))",
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
         "hide": true,
         "iconColor": "dark-blue",
         "name": "version",
-        "textFormat": "{{short_version}}",
+        "textFormat": "{{version}}",
         "titleFormat": "Version change"
       },
       {

dashboards/vmauth.json

@@ -53,6 +53,19 @@
         },
         "type": "dashboard"
       },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$ds"
+        },
+        "enable": true,
+        "expr": "sum by(version) (\n    label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n    OR\n    vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n) \nunless \n(\n    sum by(version) (\n        label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\")\n        OR\n        vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"}\n    ) offset $__interval\n)",
+        "hide": true,
+        "iconColor": "dark-blue",
+        "name": "version",
+        "textFormat": "{{version}}",
+        "titleFormat": "Version change"
+      },
       {
         "datasource": {
           "type": "prometheus",

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.149.0
+    image: victoriametrics/vmagent:v1.150.0
     depends_on:
       - "vmauth"
     ports:
@@ -42,14 +42,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.149.0-cluster
+    image: victoriametrics/vmstorage:v1.150.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.149.0-cluster
+    image: victoriametrics/vmstorage:v1.150.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -59,7 +59,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.149.0-cluster
+    image: victoriametrics/vminsert:v1.150.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -68,7 +68,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.149.0-cluster
+    image: victoriametrics/vminsert:v1.150.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -80,7 +80,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.149.0-cluster
+    image: victoriametrics/vmselect:v1.150.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -90,7 +90,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.149.0-cluster
+    image: victoriametrics/vmselect:v1.150.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -105,7 +105,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.149.0
+    image: victoriametrics/vmauth:v1.150.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -119,7 +119,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.149.0
+    image: victoriametrics/vmalert:v1.150.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.149.0
+    image: victoriametrics/vmagent:v1.150.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.149.0
+    image: victoriametrics/victoria-metrics:v1.150.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.149.0
+    image: victoriametrics/vmalert:v1.150.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts-health.yml

@@ -16,6 +16,19 @@ groups:
             Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
             It might be crashlooping.
 
+      - alert: UncleanShutdown
+        expr: vm_app_prev_shutdown_unclean == 1 and time() - vm_app_start_timestamp < 600
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }} on instance {{ $labels.instance }} started after an unclean shutdown"
+          description: |
+            The previous process run didn't shut down cleanly. Check the logs for OOM, SIGKILL,
+            a host failure, or another unexpected termination. In Kubernetes, a pod may be forcefully
+            killed with SIGKILL if the shutdown takes longer than terminationGracePeriodSeconds.
+            This alert stops firing 10 minutes after startup.
+            See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443 for more details.
+
       - alert: ServiceDown
         expr: up{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"} == 0
         for: 2m

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.149.0
+    image: victoriametrics/vmagent:v1.150.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.149.0
+    image: victoriametrics/victoria-metrics:v1.150.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.149.0
+    image: victoriametrics/vmalert:v1.150.0
     depends_on:
       - "victoriametrics"
     ports:

docs/Makefile

@@ -90,12 +90,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/victoria_metrics_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/victoria_metrics_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent insert requests/ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of concurrent search requests\./ s/(default [0-9]\+)/(default vmselect.getDefaultMaxConcurrentRequests())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of CPU cores a single query can use\./ s/(default [0-9]\+)/(default netstorage.defaultMaxWorkersPerQuery())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/victoria_metrics_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/victoria_metrics_common_flags.md
 
 docs-update-vmauth-flags:
 ifndef TAG
@@ -119,9 +116,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmauth_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmauth_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmauth_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmauth_common_flags.md
 
 docs-update-vmagent-flags:
 ifndef TAG
@@ -145,11 +142,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmagent_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmagent_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent insert requests/ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmagent_common_flags.md
-	sed -i '/The number of concurrent queues to each -remoteWrite.url./ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmagent_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmagent_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmagent_common_flags.md
 
 docs-update-vmalert-flags:
 ifndef TAG
@@ -173,10 +168,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmalert_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmalert_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/Defines number of writers for concurrent writing into remote write endpoint./ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmalert_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmalert_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmalert_common_flags.md
 
 docs-update-vmselect-flags:
 ifndef TAG

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -10,9 +10,9 @@ sitemap:
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.149.0)
-  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.149.0)
-  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.149.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.150.0)
+  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.150.0)
+  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.150.0)
 - [Grafana](https://grafana.com/) (v12.2.0)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
@@ -328,7 +328,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.149.0
+    image: victoriametrics/vmagent:v1.150.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -345,7 +345,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.149.0
+    image: victoriametrics/victoria-metrics:v1.150.0
     ports:
       - 8428:8428
     volumes:
@@ -378,7 +378,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.149.0
+    image: victoriametrics/vmalert:v1.150.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -248,23 +248,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
 # compose.yaml
 services:
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.149.0
+    image: victoriametrics/victoria-metrics:v1.150.0
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.149.0-cluster
+    image: victoriametrics/vmstorage:v1.150.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.149.0-cluster
+    image: victoriametrics/vminsert:v1.150.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.149.0-cluster
+    image: victoriametrics/vmselect:v1.150.0-cluster
     command:
       - -storageNode=vmstorage:8401
 
   vmagent:
-    image: victoriametrics/vmagent:v1.149.0
+    image: victoriametrics/vmagent:v1.150.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -316,7 +316,7 @@ Now add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: docker.io/victoriametrics/vmauth:v1.149.0
+    image: docker.io/victoriametrics/vmauth:v1.150.0
     ports:
       - 8427:8427
     volumes:

docs/guides/multi-regional-setup-dedicated-regions/README.md

@@ -6,80 +6,237 @@ build:
 sitemap:
   disable: true
 ---
-### Scenario
 
-Let's cover the case. You have multiple regions with workloads and want to collect metrics.
+## Overview {#scenario}
 
-The monitoring setup is in the dedicated regions as shown below:
+This guide shows how to run VictoriaMetrics across many regions in high-availability mode. Each workload runs a local vmagent and sends metrics to dedicated monitoring deployments, so metric data is duplicated and available even if one monitoring region is down.
 
-![Multi-regional setup with VictoriaMetrics: Dedicated regions for monitoring](setup.webp)
+Use this architecture when you need region-level resilience and want monitoring to keep working even if one region becomes unavailable.
 
-Every workload region (Earth, Mars, Venus) has a vmagent that sends data to multiple regions with a monitoring setup. 
-The monitoring setup (Ground Control 1,2) contains VictoriaMetrics Time Series Database(TSDB) cluster or single.
+This setup gives you:
 
-Using this schema, you can achieve:
+* High availability of metric data across regions.
+* A single global query endpoint.
+* Simpler disaster recovery.
 
-* Global Querying View
-* Querying all metrics from one monitoring installation
-* High Availability
-* You can lose one region, but your experience will be the same.
-* Of course, that means you duplicate your traffic twice.
+The trade-off is that you store and send the same data twice, so storage and compute requirements are increased.
+
+## Architecture
+
+The example architecture separates workloads into three regions, called Earth, Mars, and Venus. These represent the systems you want to monitor (e.g., your applications or your infrastructure). For monitoring, there are two separate regions, Ground Control 1 and 2, each running its own VictoriaMetrics deployment. The workload regions (the planets) run a local vmagent that forwards the same metrics to the two dedicated Ground Control regions.
+
+![Multi-regional setup with VictoriaMetrics: Dedicated regions for monitoring](setup-1.webp)
+{width="700"}
+
+The role of the Ground Controls can be filled by VictoriaMetrics in [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) or [cluster mode](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
+
+## High Availability
+
+The architecture provides high availability by storing two full copies of the data: one in Ground Control 1 and the other in Ground Control 2. Since both store the same data, losing one region doesn't result in a monitoring outage. You can still run queries, view dashboards, and receive alerts.
+
+vmagent keeps a separate persistent queue for each `-remoteWrite.url` destination. If one Ground Control region is unavailable, vmagent continues sending data to the other region. The samples for the unavailable region stay in the file-based queue, and vmagent delivers them after the region recovers. The queue size is limited by disk space available to the vmagent or group of vmagents. This helps restore consistency across both regions.
+
+This setup provides two logical copies of the data in separate monitoring regions. That lets you fail over to the healthy region if one region becomes unavailable, or spread read load across both regions if needed.
 
 ### How to write the data to Ground Control regions
 
-* You need to pass two `-remoteWrite.url` command-line options to `vmagent`:
+Run one or more vmagent nodes in each workload region and configure them to send metrics to both Ground Control regions. This gives each workload region a local write path and keeps delivery going if one monitoring region is unavailable. 
+
+For example, a vmagent that sends data to two single-node VictoriaMetrics instances looks like this:
 
 ```sh
 /path/to/vmagent-prod \
-  -remoteWrite.url=<ground-control-1-remote-write> \
-  -remoteWrite.url=<ground-control-2-remote-write>
+  -remoteWrite.url=https://ground-control-1:8428/api/v1/write \
+  -remoteWrite.url=https://ground-control-2:8428/api/v1/write
 ```
 
-* If you scrape data from Prometheus-compatible targets, then please specify `-promscrape.config` parameter as well.
+For a VictoriaMetrics cluster, use the following URLs for [`accountID=0`](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
 
-Here is a Quickstart guide for [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#quick-start)
+```sh
+/path/to/vmagent-prod \
+  -remoteWrite.url=https://ground-control-1-vminsert:8480/insert/0/prometheus/api/v1/write \
+  -remoteWrite.url=https://ground-control-2-vminsert:8480/insert/0/prometheus/api/v1/write
+```
+For more details, see [data ingestion with vmagent](https://docs.victoriametrics.com/victoriametrics/data-ingestion/vmagent/).
+vmagent [alerting rules and dashboards](https://docs.victoriametrics.com/vmagent/index.html#monitoring) help to monitor 
+the health state of each configured destination and its queue size.
 
 ### How to read the data from Ground Control regions
 
-You can use one of the following options:
+You can read data from Ground Control regions in a few different ways. The best option depends on your needs and operational complexity:
 
-1. Multi-level [vmselect setup](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup) in cluster setup, top-level vmselect(s) reads data from cluster-level vmselects
-   * Returns data in one of the clusters is unavailable  
-   * Merges data from both sources. You need to turn on [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) to remove duplicates 
-1. Regional endpoints - use one regional endpoint as default and switch to another if there is an issue.
-1. Load balancer - that sends queries to a particular region. The benefit and disadvantage of this setup is that it's simple.
-1. Promxy - proxy that reads data from multiple Prometheus-like sources. It allows reading data more intelligently to cover the region's unavailability out of the box. It doesn't support MetricsQL yet (please check this issue).
-1. Global vmselect in cluster setup - you can set up an additional subset of vmselects that knows about all storages in all regions.
-   * The [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) in 1ms on the vmselect side must be turned on. This setup allows you to query data using MetricsQL.
-   * The downside is that vmselect waits for a response from all storages in all regions.
+* Choose region via load balancer: put a load balancer in front of both Ground Control regions. Route traffic to a preferred region, with automatic failover to the other region in case of failure.
+* Merge results from multiple regions via vmselect: run a dedicated vmselect that would be configured to read from both regions and merge the results.
 
+You can read more about choosing the right architecture in the [VictoriaMetrics topologies guide](https://docs.victoriametrics.com/guides/vm-architectures/).
 
-### High Availability
+#### Load balancer
 
-The data is duplicated twice, and every region contains a full copy of the data. That means one region can be offline.
+Use a load balancer when you want one stable query endpoint in front of your Ground Control regions. In this setup, dashboards and tools send queries to a single URL, and vmauth routes each request to one available region.
 
-You don't need to set up a replication factor using the VictoriaMetrics cluster.
+The following diagram shows [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) performing the role of [load balancer for HA setups](https://docs.victoriametrics.com/vmauth/index.html#high-availability).
 
-### Alerting
+![Diagram shows vmauth between Grafana and Ground Control regions](load-balancer-vmauth.webp)
+{width="700"}
 
-You can set up vmalert in each Ground control region that evaluates recording and alerting rules. As every region contains a full copy of the data, you don't need to synchronize recording rules from one region to another.
+This approach is faster than [merging results with vmselect](#vmselect), because each query goes to only one region. It can also reduce query latency by roughly half compared with a topology that reads and merges data from both regions.
 
-For alert deduplication, please use [cluster mode in Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability).
+The main downside is that vmauth does not know whether a recovered region has already finished replaying delayed data from the vmagent queue. If you send queries to that region too early, recent data may still be incomplete. In that case, it is better to wait until the region catches up before routing traffic there.
 
-We also recommend adopting the list of [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
-for VictoriaMetrics components.
+For VictoriaMetrics single node, you can vmauth it with the following configuration:
 
-### Monitoring
+```yaml
+unauthorized_user:
+  url_prefix:
+    - "http://ground-control-1:8428"
+    - "http://ground-control-2:8428"
+  load_balancing_policy: first_available
+```
 
-An additional VictoriaMetrics single can be set up in every region, scraping metrics from the main TSDB.
+On the VictoriaMetrics cluster, the URLs must point to the Ground Control vmselect nodes. For example:
 
-You also may evaluate the option to send these metrics to the neighbour region to achieve HA.
+```yaml
+unauthorized_user:
+  url_prefix:
+    - "http://ground-control-1-vmselect:8481"
+    - "http://ground-control-2-vmselect:8481"
+  load_balancing_policy: first_available
+```
 
-Additional context
-* VictoriaMetrics Single - [https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
-* VictoriaMetrics Cluster - [https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
+The examples above show how to load balance requests without authentication. You can optionally configure authentication in several ways; for more details, read the [vmauth authorization section](https://docs.victoriametrics.com/victoriametrics/vmauth/#authorization).
 
+To start vmauth with your configuration, use the `-auth.config` flag. For example:
 
-### What more can we do?
+```sh
+/path/to/vmauth-prod -auth.config=/path/to/auth.yaml
+```
+
+You can test that queries work with curl:
+
+```sh
+# single node
+curl http://vmauth-node:8427/api/v1/query?query=up
+
+# cluster
+curl http://vmauth-node:8427/select/0/prometheus/api/v1/query?query=up
+```
+
+For an example of this topology in Kubernetes, see the [`VMDistributed` resource](https://docs.victoriametrics.com/helm/victoriametrics-k8s-stack/#vmdistributed-enabled).
+
+#### vmselect
+
+> This option requires that Ground Control regions are deployed in one of these modes:
+> - As a [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
+> - Or as VictoriaMetrics [single-node with multitenant support enabled](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy). In other words, VictoriaMetrics should be started with the optional `-vmselectAddr=:8401` command line flag to enable the vmselect RPC server.
+
+In this setup, each Ground Control region has its own local vmselect. A top-level vmselect queries these instead of connecting directly to vmstorage nodes.
+
+![Diagram shows top-level vmselect connecting to the regional vmselect nodes in each Ground Control cluster](top-level-vmselect.webp)
+{width="700"}
+
+This option is useful when direct access to vmstorage nodes is not practical or desirable. For example, when running on Kubernetes, the vmstorage services don't provide an HTTP query endpoint by default.
+
+To enable this setup, each Ground Control regional vmselect must listen for requests from the top layer by setting the `-clusternativeListenAddr` flag. The top-level vmselect must then use `-storageNode` to point to the regional vmselect nodes and must set a [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) interval to handle duplicated data.
+
+For example, here's how we can run the local cluster vmselect nodes and a top-level vmselect node:
+
+```sh
+# Ground Control 1 cluster vmselect
+/path/to/vmselect-prod \
+  -storageNode=ground-control-1-vmstorage-1:8401,ground-control-1-vmstorage-2:8401 \
+  -clusternativeListenAddr=:8401
+
+# Ground Control 2 cluster vmselect
+/path/to/vmselect-prod \
+  -storageNode=ground-control-2-vmstorage-1:8401,ground-control-2-vmstorage-2:8401 \
+  -clusternativeListenAddr=:8401
+
+# Top-level vmselect
+/path/to/vmselect-prod \
+  -storageNode=ground-control-1-vmselect:8401,ground-control-2-vmselect:8401 \
+  -dedup.minScrapeInterval=1ms \
+  -replicationFactor=2
+```
+
+This option provides a single query endpoint for both Ground Control regions. If one region becomes unavailable, the global vmselect can still query the healthy region, so dashboards and queries can continue to work.
+
+The main trade-off is performance. In a two-level vmselect topology, queries pass through two query layers, so they usually take longer than using regional endpoints directly, or through a load balancer. The benefit is that the topology is easy to understand; it keeps working if one region is lost, and it can merge data from both regions while one region is still catching up after recovery.
+
+## Alerting
+
+Run a vmalert node in each Ground Control region and point it to the local VictoriaMetrics endpoint. Since each region stores the same data, you can deploy the same alerting and recording rules in every region without needing cross-region rule synchronization. Send alerts to an [Alertmanager cluster](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability) to deduplicate firing alerts.
+
+![Diagram showing vmalert nodes running in each Ground Control region. An Alertmanager cluster connects to each vmalert and deduplicates notifications](vmalert-alertmanager.webp)
+{width="700"}
+
+A simple vmalert example for a single-node VictoriaMetrics looks like this:
+
+```sh
+/path/to/vmalert \
+  -rule=/path/to/rules.yaml \
+  -datasource.url=http://ground-control-1:8428 \
+  -notifier.url=http://alertmanager-1:9093 \
+  -notifier.url=http://alertmanager-2:9093
+```
+
+In VictoriaMetrics cluster mode, point `-datasource.url` to the regional vmselect endpoint. For example:
+
+```sh
+/path/to/vmalert \
+  -rule=/path/to/rules.yaml \
+  -datasource.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
+  -notifier.url=http://alertmanager-1:9093,http://alertmanager-2:9093
+```
+
+If you want vmalert to preserve alert state and recording rule results across restarts, configure `-remoteWrite.url` and `-remoteRead.url` to point to VictoriaMetrics as well. For example, for a VictoriaMetrics cluster:
+
+```sh
+/path/to/vmalert \
+  -rule=/path/to/rules.yaml \
+  -datasource.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
+  -remoteRead.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
+  -remoteWrite.url=http://ground-control-1-vminsert:8480/insert/0/prometheus \
+  -notifier.url=http://alertmanager-1:9093,http://alertmanager-2:9093
+```
+
+We recommend using the list of [VictoriaMetrics alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts).
+
+## Monitoring
+
+You can monitor Ground Control instances themselves using a separate monitoring path. In this setup, each region runs its own monitoring instance that scrapes metrics from the Ground Control components.
+
+![Diagram of the original setup with monitoring of monitoring added. Each region has a dedicated VictoriaMetrics instance dedicated to monitoring the main TSDB](setup-mom-1.webp)
+{width="700"}
+
+You can optionally duplicate the monitored metrics to the neighboring region for extra resilience. That way, if a whole Ground Control region goes down, you still have access to the telemetry of the downed VictoriaMetrics instance, which can help you troubleshoot and restore service more easily.
+
+Refer to the following pages on how to monitor your VictoriaMetrics deployments:
+
+* [How to monitor VictoriaMetrics single node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
+* [How to monitor a VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
+
+## What more can we do?
+
+You can deploy extra vmagent instances in Ground Control regions and use them as regional ingestion proxies. This places the write endpoint closer to storage and adds another disk-backed buffer, which improves resilience when storage is temporarily unavailable.
+
+![Diagram of the original setup where a vmagent node runs in front of each Ground Control region](setup-vmagent-1.webp)
+{width="700"}
+
+This pattern is useful when you want more reliable delivery, local relabeling, or a cleaner separation between cross-region traffic and local storage ingestion.
+
+For a Ground Control running VictoriaMetrics single node, you can run vmagent as follows:
+
+```sh
+# vmagent next to Ground Control 1
+/path/to/vmagent-prod \
+  -remoteWrite.url=http://ground-control-1:8428/api/v1/write
+```
+
+If running in cluster mode, use this instead:
+
+```sh
+# vmagent next to Ground Control 1 for cluster mode
+/path/to/vmagent-prod \
+  -remoteWrite.url=http://ground-control-1-vminsert:8480/insert/0/prometheus/api/v1/write
+```
 
-Setup vmagents in Ground Control regions. That allows it to accept data close to storage and add more reliability if storage is temporarily offline.

docs/guides/vmagent-openid-configuration/README.md

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
 # compose.yaml
 services:
   vmstorage:
-    image: victoriametrics/vmstorage:v1.149.0-cluster
+    image: victoriametrics/vmstorage:v1.150.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.149.0-cluster
+    image: victoriametrics/vminsert:v1.150.0-cluster
     command:
       - -storageNode=vmstorage:8400
 
   vmselect:
-    image: victoriametrics/vmselect:v1.149.0-cluster
+    image: victoriametrics/vmselect:v1.150.0-cluster
     command:
       - -storageNode=vmstorage:8401
     ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
 # compose.yaml
 services:
   vmauth:
-    image: victoriametrics/vmauth:v1.149.0-enterprise
+    image: victoriametrics/vmauth:v1.150.0-enterprise
     ports:
       - 8427:8427
     volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
 # compose.yaml
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.149.0
+    image: victoriametrics/vmagent:v1.150.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:

docs/guides/vmalert-datasource-managed-alerts-grafana/README.md

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
 # compose.yml
 services:
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.149.0
+    image: victoriametrics/victoria-metrics:v1.150.0
     command:
       - "--storageDataPath=/victoria-metrics-data"
       - "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
       - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
 
   vmalert:
-    image: victoriametrics/vmalert:v1.149.0
+    image: victoriametrics/vmalert:v1.150.0
     depends_on:
       - victoriametrics
       - alertmanager

docs/victoriametrics/LTS-releases.md

@@ -28,5 +28,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
 
 ## Currently supported LTS release lines
 
-- v1.148.x - the latest one is [v1.148.1 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.1)
-- v1.136.x - the latest one is [v1.136.15 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.15)
+- v1.148.x - the latest one is [v1.148.2 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.2)
+- v1.136.x - the latest one is [v1.136.16 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.16)

docs/victoriametrics/Quick-Start.md

@@ -54,8 +54,8 @@ and unpack it. It contains a single `victoria-metrics-prod` binary.
 For example, on Linux with `amd64` architecture:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.149.0/victoria-metrics-linux-amd64-v1.149.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.149.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.150.0/victoria-metrics-linux-amd64-v1.150.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.150.0.tar.gz
 ```
 
 The binary is self-contained and requires no installation - it is ready to run as is.
@@ -230,9 +230,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.149.0
+docker pull victoriametrics/victoria-metrics:v1.150.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.149.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.150.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

docs/victoriametrics/README.md

@@ -1266,47 +1266,106 @@ See also [resource usage limits at VictoriaMetrics cluster](https://docs.victori
 
 ## High availability
 
-The general approach for achieving high availability is the following:
+VictoriaMetrics supports high availability for both writes and reads by combining replication with multiple instances.
 
-* To run two identically configured VictoriaMetrics instances in distinct datacenters (availability zones);
-* To store the collected data simultaneously into these instances via [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) or Prometheus.
-* To query the first VictoriaMetrics instance and to fail over to the second instance when the first instance becomes temporarily unavailable.
-  This can be done via [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) according to [these docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#high-availability).
+### High availability for writes
 
-Such a setup guarantees that the collected data isn't lost when one of VictoriaMetrics instance becomes unavailable.
-The collected data continues to be written to the available VictoriaMetrics instance, so it should be available for querying.
-Both [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and Prometheus buffer the collected data locally if they cannot send it
-to the configured remote storage. So the collected data will be written to the temporarily unavailable VictoriaMetrics instance
-after it becomes available.
+You can achieve **high availability for writes** using replication:
 
-If you use [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) for storing the data into VictoriaMetrics,
-then it can be configured with multiple `-remoteWrite.url` command-line flags, where every flag points to the VictoriaMetrics
-instance in a particular availability zone, in order to replicate the collected data to all the VictoriaMetrics instances.
-For example, the following command instructs `vmagent` to replicate data to `vm-az1` and `vm-az2` instances of VictoriaMetrics:
+* Run two or more identically configured VictoriaMetrics instances in distinct datacenters (availability zones);
+* Replicate collected metrics simultaneously into all these instances via one or more [vmagents](https://docs.victoriametrics.com/victoriametrics/vmagent/).
 
+In this setup, configure vmagent [to replicate data](https://docs.victoriametrics.com/victoriametrics/vmagent/#replication-and-high-availability)
+to each remote destination:
 ```sh
 /path/to/vmagent \
-  -remoteWrite.url=http://<vm-az1>:8428/api/v1/write \
-  -remoteWrite.url=http://<vm-az2>:8428/api/v1/write
+  -remoteWrite.url=https://victoriametrics-1:8428/api/v1/write \
+  -remoteWrite.url=https://victoriametrics-2:8428/api/v1/write
 ```
 
-If you use Prometheus for collecting and writing the data to VictoriaMetrics,
-then the following [`remote_write`](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) section
-in Prometheus config can be used for replicating the collected data to `vm-az1` and `vm-az2` VictoriaMetrics instances:
+Each `--remoteWrite.url` creates its own replication queue. The queue temporarily stores data on disk while a remote destination is unavailable.
+See more about [on-disk persistence in vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#on-disk-persistence).
 
-```yaml
-remote_write:
-  - url: http://<vm-az1>:8428/api/v1/write
-  - url: http://<vm-az2>:8428/api/v1/write
+When the remote destination becomes available, vmagent drains the queue and restores data consistency across destinations.
+
+> The max size of the on-disk queue can be increased by [horizontally sharding vmagents](https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets).
+> To achieve high availability for vmagent itself, run multiple identically configured vmagent replicas.
+> In this case, the load on the remote destinations will increase proportionally to the number of vmagent replicas. The duplicated data in remote destinations
+> has to be [deduplicated](https://docs.victoriametrics.com/victoriametrics/#deduplication) on the VictoriaMetrics side.
+
+### High availability for reads
+
+You can achieve **high availability for reads** by choosing one of the following options:
+
+- Load balancer: Use a load balancer to ensure read operations are always routed to an available VictoriaMetrics instance.
+- Top-level vmselect: Use vmselect to query all available VictoriaMetrics instances and merge the results
+
+**Load balancer for reads**
+
+In this mode, we use a load balancer to query the main VictoriaMetrics instance and fail over to a secondary instance if the first one becomes temporarily unavailable.
+
+This can be done using [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) configured in [high-availability mode](https://docs.victoriametrics.com/victoriametrics/vmauth/#high-availability).
+
+```mermaid
+flowchart LR
+    Client["Query Client<br/>Grafana/vmalert"]
+
+    VMAUTH["vmauth<br/>Load Balancer / Failover"]
+
+    VM1["VictoriaMetrics-1<br/>Primary read target"]
+    VM2["VictoriaMetrics-2<br/>Failover read target"]
+
+    Client -->|"Read query"| VMAUTH
+
+    VMAUTH -->|"1. Send queries"| VM1
+    VMAUTH -.->|"2. Fail over if VM1<br/>is unavailable"| VM2
 ```
 
-It is recommended to use [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) instead of Prometheus for highly loaded setups,
-since it uses lower amounts of RAM, CPU and network bandwidth than Prometheus.
+This is the most cost-efficient option because it queries only one VictoriaMetrics instance at a time.
 
-If you use identically configured [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) instances for collecting the same data
-and sending it to VictoriaMetrics, then do not forget enabling [deduplication](#deduplication) at VictoriaMetrics side.
+The downside is that when one instance goes down and then comes back up, the load balancer may immediately start sending
+read queries to the recovering instance, even though it hasn't caught up with vmagent's queue yet and may return incomplete results.
 
-See [VMDistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/) Kubernetes operator resource for an example.
+This shortcoming can be mitigated during sequential upgrades by removing the catching-up instance from the vmauth configuration until the vmagent queues are drained. During sequential upgrades, this mechanism is automatically applied when using the [Kubernetes VMDistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/) resource. After an outage, you must remove the recovered instance manually until its vmagent queues are drained.
+
+Another option is to use top-level vmselect as described below.
+
+**Top-level vmselect for reads**
+
+In this option, we use a top-level [vmselect](https://docs.victoriametrics.com/victoriametrics/vmselect/) to query all
+remote destinations simultaneously and merge the results.
+
+This option is only possible if VictoriaMetrics single-node instances are configured with the `-vmselectAddr` flag.
+See more details in the [VictoriaMetrics multi-tenancy section](https://docs.victoriametrics.com/victoriametrics/#multi-tenancy).
+
+```mermaid
+flowchart LR
+    Client["Query Client<br/>Grafana / vmalert"]
+
+    VMSELECT["vmselect<br/>Query all destinations<br/>
+     <code><pre>-dedup.minScrapeInterval=1ms<br/>-replicationFactor=2</pre></code>"]
+
+    VM1["VictoriaMetrics-1<br/>Single-node<br/><code>-vmselectAddr=:8401</code>"]
+    VM2["VictoriaMetrics-2<br/>Single-node<br/><code>-vmselectAddr=:8401</code>"]
+
+    Client -->|"Read query"| VMSELECT
+
+    VMSELECT --> VM1
+    VMSELECT --> VM2
+
+    VMSELECT -->|"Merged and deduplicated results"| Client
+```
+
+This option requires extra resources on vmselect because it queries all remote destinations simultaneously and merges
+their responses before returning the final result.
+
+The benefit is that it can handle data gaps across destinations by merging responses from all VictoriaMetrics instances (as long as at least one instance has all the data without gaps).
+Thus, a single recovering instance can't cause incomplete results, as gaps will be filled with samples from the healthy instance.
+
+Since vmselect fetches replicated data from VictoriaMetrics instances, it must be deduplicated before processing.
+Configure vmselect with `-dedup.minScrapeInterval=1ms` to remove duplicated samples during merging.
+Also set `-replicationFactor=N` on vmselect, where `N` equals the number of remote storage destinations, so that queries
+can tolerate the unavailability of up to `N-1` destinations.
 
 ## Deduplication
 

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,11 +26,18 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): cancel query requests if client closes connection. See [#11355](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11355).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose the `vm_app_prev_shutdown_unclean` gauge. It is set to `1` when the previous process run didn't shut down cleanly. Added the `UncleanShutdown` [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml), which fires for 10 minutes after an unclean shutdown is detected. See [#8443](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show the selected time zone UTC offset next to the date/time controls and allow opening time zone settings from it. See [#11332](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11332).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/), and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): show how the default value is calculated for command-line flags which derive it from the number of available CPU cores. For example, `-maxConcurrentInserts` now prints `(default 16 = 2*cgroup.AvailableCPUs())` in `-help` output instead of `(default 16)`. Updated flags: `-search.maxConcurrentRequests`, `-search.maxWorkersPerQuery`, `-fs.maxConcurrency`, `-remoteWrite.concurrency`, `-remoteWrite.queues`. See [#9680](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680). Thanks to @Vandit1604 for contribution.
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix infinite loop in the OpenTelemetry Firehose ingestion endpoint (`/opentelemetry/api/v1/push`) when receiving a malformed record with an incomplete varint in the `data` field. Previously this caused the goroutine to spin forever, permanently consuming CPU until the process was restarted.
+* BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): reuse connections to `-remoteWrite.url` when writing the results of recording rules and alerts. Previously every series was sent over a new connection, which left a lot of sockets in `TIME_WAIT` state and could exhaust the ephemeral port range. The number of idle connections can be tuned via the new `-remoteWrite.maxIdleConnections` command-line flag. Thanks @evkuzin for contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent process crash in `sort_by_label_numeric()` and `sort_by_label_numeric_desc()` when a label value contains a number with 309 or more digits. See [#11423](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11423).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fail the query request directly when there is not enough disk space to store temporary search results. Previously, such queries could lead to vmselect crash. See [#4688](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4688).
 
 ## [v1.150.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.150.0)
 
-Release candidate
+Released at 2026-08-17
 
 **Update Note 1:** `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: default value of `-enableMultitenancyViaHeaders` command-line flag has changed from `false` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default. With this change, mentioned components will start supporting URLs with omitted tenant ID in the path: `https://<vmselect>:8481/select/prometheus/api/v1/query` will become a valid URL. To disable multitenancy via headers and simplified URLs set `--enableMultitenancyViaHeaders=false` on vmagent, vminsert and vmselect. 
 
@@ -81,6 +88,27 @@ Released at 2026-08-05
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix incorrect [sum_samples_total](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#sum_samples_total) results when `enable_windows: true` is set. See [#11261](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11261). Thanks to @beyond-infra for contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): accept scientific notation with sub-second precision (e.g. `1.784144612388E9`) for timestamp args such as `start` and `end` in `/api/v1/query_range` and `--vm-native-filter-time-start` and `--vm-native-filter-time-end` in `vmctl`. Previously, values with this pattern were rejected, which is incompatible with Prometheus. See [#11268](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11268). Thanks to @STiFLeR7 for contribution.
 
+## [v1.148.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.2)
+
+Released at 2026-08-14
+
+**v1.148.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.148.x line will be supported for at least 12 months since [v1.148.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11480) release**
+
+* SECURITY: upgrade Go builder from Go1.26.5 to Go1.26.6. See [the list of issues addressed in Go1.26.6](https://github.com/golang/go/issues?q=milestone%3AGo1.26.6%20label%3ACherryPickApproved).
+
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly parse small fractional Unix timestamps in timestamp args such as `start` and `end` in `/api/v1/query_range` and `--vm-native-filter-time-start` and `--vm-native-filter-time-end` in `vmctl`. Previously, fractional Unix timestamps with the integer part below `9223372` were interpreted with the wrong unit, for example `12.0` was parsed as `12000` seconds instead of `12` seconds. See [#11324](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11324).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): persist the previous working set cache during graceful shutdown when it is likely to contain the active working set. This prevents saving an empty or cold current cache right after split-mode cache rotation, which could otherwise slow down ingestion or queries after restart until the cache warms up again. See [#11299](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11299).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly assign scrape target IP address at IPv6-only networks for [docker_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#docker_sd_configs). See [#10965](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10965).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): avoid suggesting the unrelated `-enableTCP6` command-line flag when scraping a target over a Unix domain socket fails.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): respect the custom query step specified via `g0.step_input` when opening a URL. Previously, it could be reset to the automatically calculated step and potentially cause dashboards to freeze. See [#11137](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11137).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): change the HTTP response code for [Prometheus querying API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-usage) requests from `422 Unprocessable Entity` to `400 Bad Request` when request parameters are missing or incorrect. See [#11330](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11330).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): skip labels with empty name at [/api/v1/import](https://docs.victoriametrics.com/victoriametrics/#how-to-import-data-in-json-line-format). Previously such a label replaced the metric name, so a series sent with `"metric":{"__name__":"foo","":"bar"}` was stored under the name `bar`. Other ingestion protocols already skip such labels. See [#4962](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4962).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): rename `vmalert_rule_group_results_limit` back to `vmalert_group_rule_results_limit`. The metric was introduced in [v1.147.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11470) but was accidentally given the wrong name. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): properly update group-level `eval_delay` and `eval_alignment` for existing groups during runtime when config reload is triggered periodically or manually via `/-/reload`. Previously, these settings weren't updated after config reload during runtime. See [#11374](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11374).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): scale the default `-search.maxConcurrentRequests` with the number of available CPU cores instead of capping it at 16. See [#11191](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11191). Thanks to @Dhru1Tanna for contribution.
+
 ## [v1.148.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.1)
 
 Released at 2026-07-31
@@ -423,6 +451,24 @@ It enables back `Discovered targets` debug UI by default.
 * BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
 
+## [v1.136.16](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.16)
+
+Released at 2026-08-14
+
+**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
+
+* SECURITY: upgrade Go builder from Go1.26.5 to Go1.26.6. See [the list of issues addressed in Go1.26.6](https://github.com/golang/go/issues?q=milestone%3AGo1.26.6%20label%3ACherryPickApproved).
+
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): persist the previous working set cache during graceful shutdown when it is likely to contain the active working set. This prevents saving an empty or cold current cache right after split-mode cache rotation, which could otherwise slow down ingestion or queries after restart until the cache warms up again. See [#11299](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11299).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly assign scrape target IP address at IPv6-only networks for [docker_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#docker_sd_configs). See [#10965](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10965).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): respect the custom query step specified via `g0.step_input` when opening a URL. Previously, it could be reset to the automatically calculated step and potentially cause dashboards to freeze. See [#11137](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11137).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): change the HTTP response code for [Prometheus querying API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-usage) requests from `422 Unprocessable Entity` to `400 Bad Request` when request parameters are missing or incorrect. See [#11330](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11330).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): skip labels with empty name at [/api/v1/import](https://docs.victoriametrics.com/victoriametrics/#how-to-import-data-in-json-line-format). Previously such a label replaced the metric name, so a series sent with `"metric":{"__name__":"foo","":"bar"}` was stored under the name `bar`. Other ingestion protocols already skip such labels. See [#4962](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4962).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): properly update group-level `eval_delay` and `eval_alignment` for existing groups during runtime when config reload is triggered periodically or manually via `/-/reload`. Previously, these settings weren't updated after config reload during runtime. See [#11374](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11374).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): scale the default `-search.maxConcurrentRequests` with the number of available CPU cores instead of capping it at 16. See [#11191](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11191). Thanks to @Dhru1Tanna for contribution.
+
 ## [v1.136.15](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.15)
 
 Released at 2026-07-31

docs/victoriametrics/changelog/CHANGELOG_2025.md

@@ -115,6 +115,8 @@ Released at 2025-11-04
 
 Released at 2025-10-31
 
+**Update Note 1:** [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): reject responses with the [matrix](https://prometheus.io/docs/prometheus/latest/querying/basics/#expression-language-data-types) data type during normal rule evaluation, since vmalert expects the result to contain only a single sample or floating-point value as the rule value, not a matrix, which can contain a range of data points. Such responses could be generated by incorrect rule expressions such as `max_over_time(some_metric_filter > 90)[10m:]`, where `[10m:]` should be passed to `max_over_time` instead as `max_over_time((some_metric_filter > 90)[10m:])`.
+
 * FEATURE: `vminsert` and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce new RPC protocol for insert-storage communication. See this PR [#9820](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9820) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): explicitly check response type for [range queries](https://docs.victoriametrics.com/keyConcepts.html#range-query) during [replay](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling) and return error on type mismatch. This change should reduce confusions like in [#9779](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9779).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): allow providing multiple filters for [remote-read migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) via multiple `--remote-read-filter-label` and `--remote-read-filter-label-value` flags. This is useful in order to narrow down the data being migrated by using more precise filters. See this PR [#9917](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9917) for details.

docs/victoriametrics/enterprise.md

@@ -122,7 +122,7 @@ It is allowed to run Enterprise components in [cases listed here](https://docs.v
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
 [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
 and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.149.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.150.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -140,8 +140,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.149.0/victoria-metrics-linux-amd64-v1.149.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.149.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.150.0/victoria-metrics-linux-amd64-v1.150.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.150.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -156,7 +156,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.149.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.150.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
@@ -166,13 +166,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.149.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.150.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.149.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.150.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -182,7 +182,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.149.0
+    image: victoriametrics/victoria-metrics:v1.150.0
     ports:
       - 8428:8428
     volumes:
@@ -214,7 +214,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.149.0-enterprise
+    tag: v1.150.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -225,7 +225,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.149.0-enterprise
+    tag: v1.150.0-enterprise
 
 license:
   secret:
@@ -275,7 +275,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.149.0-enterprise
+    tag: v1.150.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -292,7 +292,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.149.0-enterprise
+    tag: v1.150.0-enterprise
 ```
 
 Example secret with license key:
@@ -343,7 +343,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.149.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.150.0-enterprise.tar.gz`
 
 Includes:
 
@@ -352,7 +352,7 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.149.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.150.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## What Happens to Licensed Components When a License Expires
 

docs/victoriametrics/integrations/grafana/_index.md

@@ -50,7 +50,7 @@ If you don't see an option to create a data source - try contacting system admin
 Create [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure/)
 in Grafana. Follow the same connection instructions as for [VictoriaMetrics datasource](#VictoriaMetrics-datasource).
 
-In the "Type and version" section set the type to "Prometheus" and the version to at least "2.24.x".
+In the "Performance" section set the Prometheus type to "Prometheus" and the Prometheus version to at least "2.24.x".
 This allows Grafana to use a more efficient API to get label values:
 
 ![Datasource](datasource-prometheus.webp)

docs/victoriametrics/scrape_config_examples.md

@@ -36,8 +36,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.149.0/victoria-metrics-linux-amd64-v1.149.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.149.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.150.0/victoria-metrics-linux-amd64-v1.150.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.150.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -151,8 +151,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.149.0/victoria-metrics-linux-amd64-v1.149.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.149.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.150.0/victoria-metrics-linux-amd64-v1.150.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.150.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/victoria_metrics_common_flags.md

@@ -342,6 +342,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs for details (default 30s)
   -promscrape.kumaSDCheckInterval duration
      Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kuma_sd_configs for details (default 30s)
+  -promscrape.linodeSDCheckInterval duration
+     Interval for checking for changes in Linode. This works only if linode_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#linode_sd_configs for details (default 1m0s)
   -promscrape.marathonSDCheckInterval duration
      Interval for checking for changes in Marathon REST API. This works only if marathon_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#marathon_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int

docs/victoriametrics/vmagent_common_flags.md

@@ -35,7 +35,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
   -enableMetadata
      Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
   -enableMultitenancyViaHeaders
-     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy
+     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy (default true)
   -enableMultitenantHandlers
      Whether to process incoming data via multitenant insert handlers according to https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format . By default incoming data is processed via single-node insert handlers according to https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data .See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy for details
   -enableTCP6
@@ -302,6 +302,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs for details (default 30s)
   -promscrape.kumaSDCheckInterval duration
      Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kuma_sd_configs for details (default 30s)
+  -promscrape.linodeSDCheckInterval duration
+     Interval for checking for changes in Linode. This works only if linode_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#linode_sd_configs for details (default 1m0s)
   -promscrape.marathonSDCheckInterval duration
      Interval for checking for changes in Marathon REST API. This works only if marathon_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#marathon_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int

docs/victoriametrics/vmalert.md

@@ -1127,13 +1127,18 @@ Or for all rules within the [group](#groups) {{% available_from "v1.117.0" %}}.
 Just set `debug: true` in configuration and vmalert will start printing additional log messages:
 
 ```sh
-2022-09-15T13:35:41.155Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:41+02:00: query returned 0 series (elapsed: 5.896041ms, isPartial: false)
-2022-09-15T13:35:56.149Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29+by%28instance%29+%3E+0&step=15s&time=1663248945"
-2022-09-15T13:35:56.178Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: query returned 1 series (elapsed: 28.368208ms, isPartial: false)
-2022-09-15T13:35:56.178Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29&step=15s&time=1663248945"
-2022-09-15T13:35:56.179Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} created in state PENDING
+2026-08-20T08:21:29.464Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A20%3A00Z"
+2026-08-20T08:21:29.465Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:20:00Z: query returned 0 series (series_fetched: 0, elapsed: 1.075166ms, isPartial: false)
 ...
-2022-09-15T13:36:56.153Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:36:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} PENDING => FIRING: 1m0s since becoming active at 2022-09-15 15:35:56.126006 +0200 CEST m=+39.384575417
+2026-08-20T08:22:29.466Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A21%3A00Z"
+2026-08-20T08:22:29.468Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: query returned 2 series (series_fetched: 2, elapsed: 2.055916ms, isPartial: false)
+2026-08-20T08:22:29.469Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: alert 4671711516378822929 {alertgroup="vm-health",alertname="ServiceDown",instance="victoriametrics:8428",job="victoriametrics",severity="critical"} created in state PENDING
+2026-08-20T08:22:29.469Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: alert 6230585559362831632 {alertgroup="vm-health",alertname="ServiceDown",instance="vmagent:8429",job="vmagent",severity="critical"} created in state PENDING
+...
+2026-08-20T08:23:29.463Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A22%3A00Z"
+2026-08-20T08:23:29.465Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: query returned 2 series (series_fetched: 2, elapsed: 1.391416ms, isPartial: false)
+2026-08-20T08:23:29.466Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: alert 4671711516378822929 {alertgroup="vm-health",alertname="ServiceDown",instance="victoriametrics:8428",job="victoriametrics",severity="critical"} PENDING => FIRING: 1m0s since becoming active at 2026-08-20 08:21:00 +0000 UTC
+2026-08-20T08:23:29.466Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: alert 6230585559362831632 {alertgroup="vm-health",alertname="ServiceDown",instance="vmagent:8429",job="vmagent",severity="critical"} PENDING => FIRING: 1m0s since becoming active at 2026-08-20 08:21:00 +0000 UTC
 ```
 
 Sensitive info is stripped from the `curl` examples - see [security](#security) section for more details.

docs/victoriametrics/vmalert_common_flags.md

@@ -370,6 +370,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
      Defines a duration for idle (keep-alive connections) to exist. Consider settings this value less to the value of "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors. (default 50s)
   -remoteWrite.maxBatchSize int
      Defines max number of timeseries to be flushed at once (default 10000)
+  -remoteWrite.maxIdleConnections int
+     Defines the number of idle (keep-alive connections) to -remoteWrite.url for the vmalert-tool debug writer, which sends every series in a separate request. Too low a value may result in a high number of sockets in TIME_WAIT state. (default 100)
   -remoteWrite.maxQueueSize int
      Defines the max number of pending datapoints to remote write endpoint (default 100000)
   -remoteWrite.oauth2.clientID string

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.149.0/vmutils-darwin-arm64-v1.149.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.150.0/vmutils-darwin-arm64-v1.150.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.149.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.150.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

docs/victoriametrics/vminsert_common_flags.md

@@ -39,7 +39,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -enableMetadata
      Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
   -enableMultitenancyViaHeaders
-     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers
+     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers (default true)
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable

docs/victoriametrics/vmselect_common_flags.md

@@ -44,7 +44,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
   -enableMetadata
      Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
   -enableMultitenancyViaHeaders
-     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers
+     Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers (default true)
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable
@@ -228,6 +228,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
      Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs for details (default 30s)
   -promscrape.kumaSDCheckInterval duration
      Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kuma_sd_configs for details (default 30s)
+  -promscrape.linodeSDCheckInterval duration
+     Interval for checking for changes in Linode. This works only if linode_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#linode_sd_configs for details (default 1m0s)
   -promscrape.marathonSDCheckInterval duration
      Interval for checking for changes in Marathon REST API. This works only if marathon_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#marathon_sd_configs for details (default 30s)
   -promscrape.maxDroppedTargets int

lib/appmetrics/appmetrics.go

@@ -65,6 +65,9 @@ func writePrometheusMetrics(w io.Writer) {
 	// Export start time and uptime in seconds
 	metrics.WriteGaugeUint64(w, "vm_app_start_timestamp", uint64(startTime.Unix()))
 	metrics.WriteGaugeUint64(w, "vm_app_uptime_seconds", uint64(time.Since(startTime).Seconds()))
+	if uncleanShutdownEnabled.Load() {
+		metrics.WriteGaugeUint64(w, "vm_app_prev_shutdown_unclean", uncleanShutdown)
+	}
 
 	// Export flags as metrics.
 	isSetMap := make(map[string]bool)

lib/appmetrics/osmetrics.go

@@ -13,13 +13,13 @@ type osInfo struct {
 	release string
 }
 
-var os osInfo
+var hostOS osInfo
 var initOSOnce sync.Once
 
 func writeOSMetrics(w io.Writer) {
 	initOSOnce.Do(initOS)
 
-	if os.name != "" {
-		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_os_info{os=%q, release=%q}`, os.name, os.release), 1)
+	if hostOS.name != "" {
+		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_os_info{os=%q, release=%q}`, hostOS.name, hostOS.release), 1)
 	}
 }

lib/appmetrics/osmetrics_darwin.go

@@ -8,7 +8,7 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "darwin"}
+	hostOS = osInfo{name: "darwin"}
 
 	out, err := exec.Command("sysctl", "-n", "kern.osrelease").Output()
 	if err != nil {
@@ -16,5 +16,5 @@ func initOS() {
 		return
 	}
 
-	os.release = strings.TrimSpace(string(out))
+	hostOS.release = strings.TrimSpace(string(out))
 }

lib/appmetrics/osmetrics_linux.go

@@ -7,7 +7,7 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "linux"}
+	hostOS = osInfo{name: "linux"}
 
 	var uname syscall.Utsname
 	if err := syscall.Uname(&uname); err != nil {
@@ -22,5 +22,5 @@ func initOS() {
 		}
 		ur = append(ur, byte(v))
 	}
-	os.release = string(ur)
+	hostOS.release = string(ur)
 }

lib/appmetrics/osmetrics_windows.go

@@ -8,12 +8,12 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "windows"}
+	hostOS = osInfo{name: "windows"}
 
 	ver := windows.RtlGetVersion()
 	if ver == nil {
 		logger.Warnf("vm_os_info metric will miss release info since windows.RtlGetVersion returned nil version")
 		return
 	}
-	os.release = fmt.Sprintf("%d.%d.%d", ver.MajorVersion, ver.MinorVersion, ver.BuildNumber)
+	hostOS.release = fmt.Sprintf("%d.%d.%d", ver.MajorVersion, ver.MinorVersion, ver.BuildNumber)
 }

lib/appmetrics/unclean_shutdown.go

@@ -0,0 +1,53 @@
+package appmetrics
+
+import (
+	"os"
+	"path/filepath"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// UncleanShutdownMarkerFilename is the marker file used to detect a previous unclean shutdown.
+const UncleanShutdownMarkerFilename = ".vm_app_running"
+
+var (
+	uncleanShutdownEnabled atomic.Bool
+	uncleanShutdown        uint64
+)
+
+// MustCreateUncleanShutdownMarker creates an UncleanShutdownMarkerFilename marker file in the given dirPath.
+// Must be called once on program startup and paired with a single MustRemoveUncleanShutdownMarker call on exit.
+//
+// If the marker file already exists on startup, it indicates a previous unclean shutdown and uncleanShutdown is set to 1.
+func MustCreateUncleanShutdownMarker(dirPath string) {
+	if !uncleanShutdownEnabled.CompareAndSwap(false, true) {
+		logger.Fatalf("BUG: unclean shutdown marker was already initialized. It could only be called once")
+	}
+	marker := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+	f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
+	if err == nil {
+		fs.MustClose(f)
+		return
+	}
+	if os.IsExist(err) {
+		uncleanShutdown = 1
+		logger.Warnf("Previous shutdown was unclean since file %q exists. Please check logs and investigate the reason of unclean shutdown", marker)
+		return
+	}
+	logger.Panicf("FATAL: cannot create unclean shutdown marker %q: %s", marker, err)
+}
+
+// MustRemoveUncleanShutdownMarker removes the UncleanShutdownMarkerFilename marker file created by MustCreateUncleanShutdownMarker.
+// Must be called once, as late as possible before program exit.
+func MustRemoveUncleanShutdownMarker(dirPath string) {
+	if !uncleanShutdownEnabled.Load() {
+		logger.Fatalf("BUG: unclean shutdown marker was not initialized with MustCreateUncleanShutdownMarker call")
+	}
+	marker := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+	if err := os.Remove(marker); err != nil {
+		logger.Fatalf("FATAL: cannot remove unclean shutdown marker %q: %s", marker, err)
+	}
+	fs.MustSyncPath(dirPath)
+}

lib/appmetrics/unclean_shutdown_test.go

@@ -0,0 +1,60 @@
+package appmetrics
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestUncleanShutdownLifecycle(t *testing.T) {
+
+	t.Cleanup(func() {
+		uncleanShutdownEnabled.Store(false)
+	})
+	dirPath := t.TempDir()
+	markerPath := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+
+	// unclean logic is disabled. the unclean shutdown metric should not be exposed
+	var bb bytes.Buffer
+	writePrometheusMetrics(&bb)
+	if strings.Contains(bb.String(), "vm_app_prev_shutdown_unclean") {
+		t.Fatalf("unexpected unclean shutdown metric before starting the marker")
+	}
+
+	// clean start, the metric must report 0
+	MustCreateUncleanShutdownMarker(dirPath)
+	mustContainUncleanShutdownMetric(t, 0)
+	if _, err := os.Stat(markerPath); err != nil {
+		t.Fatalf("cannot stat the running marker after the first start: %s", err)
+	}
+	MustRemoveUncleanShutdownMarker(dirPath)
+	if _, err := os.Stat(markerPath); !os.IsNotExist(err) {
+		t.Fatalf("unexpected running marker after a clean shutdown; got error %v; want os.ErrNotExist", err)
+	}
+	uncleanShutdownEnabled.Store(false)
+
+	// simulate prev unclean shutdown, the metric must report 1
+	if err := os.WriteFile(markerPath, nil, 0600); err != nil {
+		t.Fatalf("cannot create test marker: %s", err)
+	}
+	MustCreateUncleanShutdownMarker(dirPath)
+	mustContainUncleanShutdownMetric(t, 1)
+	MustRemoveUncleanShutdownMarker(dirPath)
+	if _, err := os.Stat(markerPath); !os.IsNotExist(err) {
+		t.Fatalf("unexpected running marker after a clean shutdown; got error %v; want os.ErrNotExist", err)
+	}
+}
+
+func mustContainUncleanShutdownMetric(t *testing.T, value uint64) {
+	t.Helper()
+
+	var bb bytes.Buffer
+	writePrometheusMetrics(&bb)
+	want := "vm_app_prev_shutdown_unclean " + strconv.FormatUint(value, 10) + "\n"
+	if !strings.Contains(bb.String(), want) {
+		t.Fatalf("missing %q in the exported app metrics", want)
+	}
+}

lib/backup/fscommon/fscommon.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/backupnames"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
@@ -107,7 +108,7 @@ func appendFilesInternal(dst []string, d *os.File) ([]string, error) {
 }
 
 func isSpecialFile(name string) bool {
-	return name == "flock.lock" || name == backupnames.RestoreInProgressFilename || name == backupnames.RestoreMarkFileName || strings.HasSuffix(name, ".tmp")
+	return name == "flock.lock" || name == appmetrics.UncleanShutdownMarkerFilename || name == backupnames.RestoreInProgressFilename || name == backupnames.RestoreMarkFileName || strings.HasSuffix(name, ".tmp")
 }
 
 // RemoveEmptyDirs recursively removes empty directories under the given dir.

lib/flagutil/array.go

@@ -39,8 +39,29 @@ func NewArrayBool(name, description string) *ArrayBool {
 }
 
 // NewArrayInt returns new ArrayInt with the given name, defaultValue and description.
+//
+// -help shows defaultValue as a plain number. Use NewArrayIntWithDynamicDefault when
+// defaultValue is calculated at runtime.
 func NewArrayInt(name string, defaultValue int, description string) *ArrayInt {
-	description += fmt.Sprintf(" (default %d)", defaultValue)
+	return newArrayInt(name, defaultValue, strconv.Itoa(defaultValue), description)
+}
+
+// NewArrayIntWithDynamicDefault returns new ArrayInt with the given name, defaultValue and description.
+//
+// Use it instead of NewArrayInt when defaultValue is calculated at runtime.
+// See NewIntWithDynamicDefault for why such a value needs a hint.
+func NewArrayIntWithDynamicDefault(name string, defaultValue int, defaultValueHint, description string) *ArrayInt {
+	if defaultValueHint == "" {
+		panic(fmt.Sprintf("BUG: missing defaultValueHint for -%s", name))
+	}
+	return newArrayInt(name, defaultValue, fmt.Sprintf("%d = %s", defaultValue, defaultValueHint), description)
+}
+
+// newArrayInt registers an int array flag, which shows defaultValueText as its default in -help.
+//
+// Array flags keep the default in the description, since flag.Var hides an empty DefValue.
+func newArrayInt(name string, defaultValue int, defaultValueText, description string) *ArrayInt {
+	description += fmt.Sprintf(" (default %s)", defaultValueText)
 	description += "\nSupports `array` of values separated by comma or specified via multiple flags."
 	description += "\nEmpty values are set to default value."
 	a := &ArrayInt{

lib/flagutil/flag.go

@@ -7,6 +7,25 @@ import (
 	"strings"
 )
 
+// NewIntWithDynamicDefault returns a new int flag with the given name, defaultValue and description.
+//
+// Use it instead of flag.Int when defaultValue is calculated at runtime, for example
+// from the number of CPU cores. Such a value differs per machine, so -help shows both
+// the value and defaultValueHint, for example "16 = 2 * availableCPUs".
+//
+// Only -help output changes. The flag value stays defaultValue.
+func NewIntWithDynamicDefault(name string, defaultValue int, defaultValueHint, description string) *int {
+	if defaultValueHint == "" {
+		panic(fmt.Sprintf("BUG: missing defaultValueHint for -%s", name))
+	}
+	p := flag.Int(name, defaultValue, description)
+
+	// DefValue is only the text shown by -help: "default value (as text); for usage message".
+	flag.Lookup(name).DefValue = fmt.Sprintf("%d = %s", defaultValue, defaultValueHint)
+
+	return p
+}
+
 // WriteFlags writes all the explicitly set flags to w.
 func WriteFlags(w io.Writer) {
 	flag.Visit(func(f *flag.Flag) {

lib/flagutil/flag_test.go

@@ -0,0 +1,51 @@
+package flagutil
+
+import (
+	"flag"
+	"strings"
+	"testing"
+)
+
+// The flags are registered at package level, since flag registration panics when it repeats.
+var (
+	fooFlagIntDynamicDefault      = NewIntWithDynamicDefault("fooFlagIntDynamicDefault", 42, "2 * availableCPUs", "test")
+	fooFlagArrayIntDynamicDefault = NewArrayIntWithDynamicDefault("fooFlagArrayIntDynamicDefault", 42, "2 * availableCPUs", "test")
+	fooFlagArrayIntPlainDefault   = NewArrayInt("fooFlagArrayIntPlainDefault", 42, "test")
+)
+
+func TestNewIntWithDynamicDefaultSuccess(t *testing.T) {
+	// -help must show the value together with the hint.
+	f := flag.Lookup("fooFlagIntDynamicDefault")
+	if f.DefValue != "42 = 2 * availableCPUs" {
+		t.Fatalf("unexpected DefValue; got %q; want %q", f.DefValue, "42 = 2 * availableCPUs")
+	}
+
+	// the flag value must stay the calculated one.
+	if *fooFlagIntDynamicDefault != 42 {
+		t.Fatalf("unexpected flag value; got %d; want %d", *fooFlagIntDynamicDefault, 42)
+	}
+}
+
+func TestNewArrayIntWithDynamicDefaultSuccess(t *testing.T) {
+	// array flags keep the default in the description, so the hint must go there.
+	f := flag.Lookup("fooFlagArrayIntDynamicDefault")
+	if !strings.Contains(f.Usage, "(default 42 = 2 * availableCPUs)") {
+		t.Fatalf("missing the hint in the flag description; got %q", f.Usage)
+	}
+
+	// the default value must stay the calculated one.
+	if n := fooFlagArrayIntDynamicDefault.GetOptionalArg(0); n != 42 {
+		t.Fatalf("unexpected default value; got %d; want %d", n, 42)
+	}
+}
+
+func TestNewArrayIntKeepsPlainDefault(t *testing.T) {
+	// NewArrayInt must keep showing a plain number, since it shares the body with the dynamic one.
+	f := flag.Lookup("fooFlagArrayIntPlainDefault")
+	if !strings.Contains(f.Usage, "(default 42)") {
+		t.Fatalf("unexpected flag description; got %q", f.Usage)
+	}
+	if n := fooFlagArrayIntPlainDefault.GetOptionalArg(0); n != 42 {
+		t.Fatalf("unexpected default value; got %d; want %d", n, 42)
+	}
+}

lib/fs/fsutil/concurrency.go

@@ -1,14 +1,15 @@
 package fsutil
 
 import (
-	"flag"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
-var maxConcurrency = flag.Int("fs.maxConcurrency", getDefaultConcurrency(), "The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency "+
-	"on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph")
+var maxConcurrency = flagutil.NewIntWithDynamicDefault("fs.maxConcurrency", getDefaultConcurrency(), "fsutil.getDefaultConcurrency()",
+	"The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency "+
+		"on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph")
 
 func getDefaultConcurrency() int {
 	n := min(16*cgroup.AvailableCPUs(), 256)

lib/protoparser/opentelemetry/firehose/parser.go

@@ -37,12 +37,12 @@ func ProcessRequestBody(b []byte) ([]byte, error) {
 	for _, r := range req.Records {
 		for len(r.Data) > 0 {
 			messageLength, varIntLength := binary.Uvarint(r.Data)
-			if varIntLength > binary.MaxVarintLen32 {
-				return nil, fmt.Errorf("failed to parse OpenTelemetry message: invalid variant")
+			if varIntLength <= 0 || varIntLength > binary.MaxVarintLen32 {
+				return nil, fmt.Errorf("failed to parse OpenTelemetry message: invalid varint (n=%d)", varIntLength)
 			}
 			totalLength := varIntLength + int(messageLength)
-			if totalLength > len(r.Data) {
-				return nil, fmt.Errorf("failed to parse OpenTelemetry message: insufficient length of buffer")
+			if totalLength <= 0 || totalLength > len(r.Data) {
+				return nil, fmt.Errorf("failed to parse OpenTelemetry message: invalid message length")
 			}
 			dst = append(dst, r.Data[varIntLength:totalLength]...)
 			r.Data = r.Data[totalLength:]

lib/protoparser/opentelemetry/firehose/parser_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
@@ -241,6 +242,30 @@ func TestProcessRequestBody(t *testing.T) {
 	}
 }
 
+// TestProcessRequestBodyIncompleteVarint verifies that an incomplete varint (0x80)
+// returns an error instead of spinning forever (GHSA-89v2-864p-v3xc).
+func TestProcessRequestBodyIncompleteVarint(t *testing.T) {
+	// "gA==" is base64 for a single 0x80 byte, i.e. an incomplete varint.
+	// binary.Uvarint returns (0, 0) for this input, which previously caused an
+	// infinite zero-progress loop inside ProcessRequestBody.
+	data := []byte(`{"records":[{"data":"gA=="}]}`)
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := ProcessRequestBody(data)
+		done <- err
+	}()
+
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("expected error for incomplete varint input, got nil")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("ProcessRequestBody did not return within 5s - infinite loop on incomplete varint input")
+	}
+}
+
 func formatTimeseries(tss []prompb.TimeSeries) string {
 	var labels promutil.Labels
 	var a []string

lib/storage/storage_timing_test.go

@@ -918,9 +918,29 @@ func variableTimeRange() []dataConfig {
 		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2025, 1, 1, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	tr1w := TimeRange{
+	tr2d := TimeRange{
 		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
-		MaxTimestamp: time.Date(2025, 1, 7, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2025, 1, 2, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	tr4d := TimeRange{
+		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2025, 1, 4, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	tr8d := TimeRange{
+		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2025, 1, 8, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	tr16d := TimeRange{
+		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2025, 1, 16, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	tr32d := TimeRange{
+		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2025, 1, 32, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	tr64d := TimeRange{
+		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2025, 1, 64, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
 	tr1m := TimeRange{
 		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
@@ -930,13 +950,13 @@ func variableTimeRange() []dataConfig {
 		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2025, 2, 28, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	tr6m := TimeRange{
+	tr4m := TimeRange{
 		MinTimestamp: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
-		MaxTimestamp: time.Date(2025, 5, 31, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2025, 4, 30, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	trNames := []string{"1d", "1w", "1m", "2m", "6m"}
+	trNames := []string{"1d", "2d", "4d", "8d", "16d", "32d", "64d", "1m", "2m", "4m"}
 	var cfgs []dataConfig
-	for i, tr := range []TimeRange{tr1d, tr1w, tr2m, tr1m, tr6m} {
+	for i, tr := range []TimeRange{tr1d, tr2d, tr4d, tr8d, tr16d, tr32d, tr64d, tr1m, tr2m, tr4m} {
 		cfgs = append(cfgs, dataConfig{
 			name:             fmt.Sprintf("VariableTimeRange/%s", trNames[i]),
 			numSeries:        100_000,

lib/writeconcurrencylimiter/concurrencylimiter.go

@@ -10,16 +10,18 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	maxConcurrentInserts = flag.Int("maxConcurrentInserts", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent insert requests. "+
-		"Set higher value when clients send data over slow networks. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. "+
-		"See also -insert.maxQueueDuration")
+	maxConcurrentInserts = flagutil.NewIntWithDynamicDefault("maxConcurrentInserts", 2*cgroup.AvailableCPUs(), "2*cgroup.AvailableCPUs()",
+		"The maximum number of concurrent insert requests. "+
+			"Set higher value when clients send data over slow networks. "+
+			"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. "+
+			"See also -insert.maxQueueDuration")
 	maxQueueDuration = flag.Duration("insert.maxQueueDuration", time.Minute, "The maximum duration to wait in the queue when -maxConcurrentInserts "+
 		"concurrent insert requests are executed")
 )