app/vmselect: fail the query request directly when there is not enough disk space

Previously, vmselect may crash if there is not enough free disk space to store tmp files from vmstorage. vmselect stores data blocks received from vmstorage if response size exceeds in-memory limit.
 It reduced vmselect availability, because it makes impossible to serve small queries. And huge query may crash vmselect for all users.

 So this commit adds a error check instead. If there is not enough disk space, vmselect will stop query execution and return error back to user.

fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4688

README.md

@@ -1,6 +1,6 @@
 # VictoriaMetrics
 
-[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
+[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?logo=github&labelColor=gray&color=gray&label=Release)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)

app/victoria-metrics/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -22,7 +23,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer/push"
 )
 
 var (
@@ -35,9 +35,10 @@ var (
 		"This can be changed with -promscrape.config.strictParse=false command-line flag")
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmsingle can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate.")
-	vmselectMaxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
-		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
-		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
+	vmselectMaxConcurrentRequests = flagutil.NewIntWithDynamicDefault("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "vmselect.getDefaultMaxConcurrentRequests()",
+		"The maximum number of concurrent search requests. "+
+			"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
+			"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
 	vmselectMaxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
 )
@@ -91,7 +92,9 @@ func main() {
 	}
 	logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
 	startTime := time.Now()
+
 	vmstorage.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
+	appmetrics.MustCreateUncleanShutdownMarker(vmstorage.DataPath())
 	vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
 	vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
 	vminsert.Init()
@@ -104,10 +107,8 @@ func main() {
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
-	push.Init()
 	sig := procutil.WaitForSigterm()
 	logger.Infof("received signal %s", sig)
-	push.Stop()
 	pushmetrics.Stop()
 
 	stopSelfScraper()
@@ -123,6 +124,7 @@ func main() {
 
 	vmstorage.Stop()
 	vmselect.Stop()
+	appmetrics.MustRemoveUncleanShutdownMarker(vmstorage.DataPath())
 
 	logger.Infof("the VictoriaMetrics has been stopped in %.3f seconds", time.Since(startTime).Seconds())
 }

app/vmagent/remotewrite/remotewrite.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/cespare/xxhash/v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
@@ -62,9 +63,10 @@ var (
 		"See also -remoteWrite.maxDiskUsagePerURL and -remoteWrite.disableOnDiskQueue")
 	keepDanglingQueues = flag.Bool("remoteWrite.keepDanglingQueues", false, "Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. "+
 		"Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.")
-	queues = flagutil.NewArrayInt("remoteWrite.queues", cgroup.AvailableCPUs()*2, "The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
-		"isn't enough for sending high volume of collected data to remote storage. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
+	queues = flagutil.NewArrayIntWithDynamicDefault("remoteWrite.queues", cgroup.AvailableCPUs()*2, "2*cgroup.AvailableCPUs()",
+		"The number of concurrent queues to each -remoteWrite.url. Set more queues if default number of queues "+
+			"isn't enough for sending high volume of collected data to remote storage. "+
+			"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage")
 	inmemoryQueues = flagutil.NewArrayInt("remoteWrite.inmemoryQueues", 0, "The number of additional workers per each -remoteWrite.url, which send only recently ingested data from the in-memory queue, "+
 		"while the file-based queue at -remoteWrite.tmpDataPath is drained by workers configured via -remoteWrite.queues. "+
 		"This reduces delivery lag for fresh samples when the file-based queue contains a backlog accumulated during remote storage outages.")
@@ -233,6 +235,7 @@ func Init() {
 	initStreamAggrConfigGlobal()
 
 	initRemoteWriteCtxs(*remoteWriteURLs)
+	appmetrics.MustCreateUncleanShutdownMarker(*tmpDataPath)
 
 	disableOnDiskQueues := []bool(*disableOnDiskQueue)
 	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
@@ -391,6 +394,8 @@ func Stop() {
 	if sl := dailySeriesLimiter; sl != nil {
 		sl.MustStop()
 	}
+
+	appmetrics.MustRemoveUncleanShutdownMarker(*tmpDataPath)
 }
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url

app/vmalert/remotewrite/init.go

@@ -36,9 +36,10 @@ var (
 	idleConnectionTimeout = flag.Duration("remoteWrite.idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist. Consider settings this value less to the value of "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
 	maxIdleConnections    = flag.Int("remoteWrite.maxIdleConnections", 100, `Defines the number of idle (keep-alive connections) to -remoteWrite.url for the vmalert-tool debug writer, which sends every series in a separate request. Too low a value may result in a high number of sockets in TIME_WAIT state.`)
 
-	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
-	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
-	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
+	maxQueueSize = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
+	maxBatchSize = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
+	concurrency  = flagutil.NewIntWithDynamicDefault("remoteWrite.concurrency", defaultConcurrency, "2*cgroup.AvailableCPUs()",
+		"Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
 	flushInterval = flag.Duration("remoteWrite.flushInterval", defaultFlushInterval, "Defines interval of flushes to remote write endpoint")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")

app/vmselect/main.go

@@ -26,7 +26,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer/push"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmalertproxy"
 )
@@ -118,15 +117,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	// Handle non-trivial dynamic requests, which may take big amounts of time and resources.
 	startTime := time.Now()
 	defer requestDuration.UpdateDuration(startTime)
-	tracerEnabled := httputil.GetBool(r, "trace") || push.IsEnabled()
+	tracerEnabled := httputil.GetBool(r, "trace")
 	qt := querytracer.New(tracerEnabled, "%s", r.URL.Path)
-	if push.IsEnabled() {
-		defer func() {
-			if time.Since(startTime) >= push.MinTraceDuration() {
-				push.Push(qt)
-			}
-		}()
-	}
 
 	// Limit the number of concurrent queries.
 	select {

app/vmselect/netstorage/netstorage.go

@@ -20,6 +20,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
@@ -30,11 +31,12 @@ var (
 	maxSamplesPerSeries = flag.Int("search.maxSamplesPerSeries", 30e6, "The maximum number of raw samples a single query can scan per each time series. This option allows limiting memory usage")
 	maxSamplesPerQuery  = flag.Int("search.maxSamplesPerQuery", 1e9, "The maximum number of raw samples a single query can process across all time series. "+
 		"This protects from heavy queries, which select unexpectedly high number of raw samples. See also -search.maxSamplesPerSeries")
-	maxWorkersPerQuery = flag.Int("search.maxWorkersPerQuery", defaultMaxWorkersPerQuery, "The maximum number of CPU cores a single query can use. "+
-		"The default value should work good for most cases. "+
-		"The flag can be set to lower values for improving performance of big number of concurrently executed queries. "+
-		"The flag can be set to bigger values for improving performance of heavy queries, which scan big number of time series (>10K) and/or big number of samples (>100M). "+
-		"There is no sense in setting this flag to values bigger than the number of CPU cores available on the system")
+	maxWorkersPerQuery = flagutil.NewIntWithDynamicDefault("search.maxWorkersPerQuery", defaultMaxWorkersPerQuery, "netstorage.defaultMaxWorkersPerQuery()",
+		"The maximum number of CPU cores a single query can use. "+
+			"The default value should work good for most cases. "+
+			"The flag can be set to lower values for improving performance of big number of concurrently executed queries. "+
+			"The flag can be set to bigger values for improving performance of heavy queries, which scan big number of time series (>10K) and/or big number of samples (>100M). "+
+			"There is no sense in setting this flag to values bigger than the number of CPU cores available on the system")
 )
 
 // Result is a single timeseries result.

app/vmselect/netstorage/tmp_blocks_file.go

@@ -62,6 +62,9 @@ type tmpBlocksFile struct {
 	r *fs.ReaderAt
 
 	offset uint64
+
+	// err stores the first error occurred while writing the temporary blocks file.
+	err error
 }
 
 func getTmpBlocksFile() *tmpBlocksFile {
@@ -82,6 +85,7 @@ func putTmpBlocksFile(tbf *tmpBlocksFile) {
 	tbf.f = nil
 	tbf.r = nil
 	tbf.offset = 0
+	tbf.err = nil
 	tmpBlocksFilePool.Put(tbf)
 }
 
@@ -109,6 +113,10 @@ var (
 // and this must be handled.
 func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	var addr tmpBlockAddr
+	if tbf.err != nil {
+		// Do not write anything to the tbf after the first failed write
+		return addr, tbf.err
+	}
 	addr.offset = tbf.offset
 	addr.size = len(b)
 	tbf.offset += uint64(addr.size)
@@ -122,7 +130,8 @@ func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	if tbf.f == nil {
 		f, err := os.CreateTemp(tmpBlocksDir, "")
 		if err != nil {
-			return addr, err
+			tbf.err = fmt.Errorf("cannot create temporary blocks file at %q: %w", tmpBlocksDir, err)
+			return addr, tbf.err
 		}
 		tbf.f = f
 		tmpBlocksFilesCreated.Inc()
@@ -130,7 +139,9 @@ func (tbf *tmpBlocksFile) WriteBlockRefData(b []byte) (tmpBlockAddr, error) {
 	_, err := tbf.f.Write(tbf.buf)
 	tbf.buf = append(tbf.buf[:0], b...)
 	if err != nil {
-		return addr, fmt.Errorf("cannot write block to %q: %w", tbf.f.Name(), err)
+		// The blocks buffered at tbf.buf could be partially lost, mark the tbf as unusable.
+		tbf.err = fmt.Errorf("cannot write block to %q: %w", tbf.f.Name(), err)
+		return addr, tbf.err
 	}
 	return addr, nil
 }
@@ -141,12 +152,16 @@ func (tbf *tmpBlocksFile) Len() uint64 {
 }
 
 func (tbf *tmpBlocksFile) Finalize() error {
+	if tbf.err != nil {
+		return tbf.err
+	}
 	if tbf.f == nil {
 		return nil
 	}
 	fname := tbf.f.Name()
 	if _, err := tbf.f.Write(tbf.buf); err != nil {
-		return fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
+		tbf.err = fmt.Errorf("cannot write the remaining %d bytes to %q: %w", len(tbf.buf), fname, err)
+		return tbf.err
 	}
 	tbf.buf = tbf.buf[:0]
 	r := fs.NewReaderAt(tbf.f)
@@ -166,6 +181,10 @@ func (tbf *tmpBlocksFile) Finalize() error {
 }
 
 func (tbf *tmpBlocksFile) MustReadBlockRefAt(partRef storage.PartRef, addr tmpBlockAddr) storage.BlockRef {
+	if tbf.err != nil {
+		// This should never happen, since Finalize() already returns the error for such a tbf.
+		logger.Panicf("BUG: cannot read block at %s from the temporary blocks file with the failed write: %s", addr, tbf.err)
+	}
 	var buf []byte
 	if tbf.r == nil {
 		buf = tbf.buf[addr.offset : addr.offset+uint64(addr.size)]

deployment/docker/rules/alerts-health.yml

@@ -16,6 +16,19 @@ groups:
             Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
             It might be crashlooping.
 
+      - alert: UncleanShutdown
+        expr: vm_app_prev_shutdown_unclean == 1 and time() - vm_app_start_timestamp < 600
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.job }} on instance {{ $labels.instance }} started after an unclean shutdown"
+          description: |
+            The previous process run didn't shut down cleanly. Check the logs for OOM, SIGKILL,
+            a host failure, or another unexpected termination. In Kubernetes, a pod may be forcefully
+            killed with SIGKILL if the shutdown takes longer than terminationGracePeriodSeconds.
+            This alert stops firing 10 minutes after startup.
+            See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443 for more details.
+
       - alert: ServiceDown
         expr: up{job=~".*(victoriametrics|vmselect|vminsert|vmstorage|vmagent|vmalert|vmsingle|vmalertmanager|vmauth).*"} == 0
         for: 2m

docs/Makefile

@@ -90,12 +90,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/victoria_metrics_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/victoria_metrics_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent insert requests/ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of concurrent search requests\./ s/(default [0-9]\+)/(default vmselect.getDefaultMaxConcurrentRequests())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of CPU cores a single query can use\./ s/(default [0-9]\+)/(default netstorage.defaultMaxWorkersPerQuery())/' docs/victoriametrics/victoria_metrics_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/victoria_metrics_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/victoria_metrics_common_flags.md
 
 docs-update-vmauth-flags:
 ifndef TAG
@@ -119,9 +116,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmauth_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmauth_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmauth_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmauth_common_flags.md
 
 docs-update-vmagent-flags:
 ifndef TAG
@@ -145,11 +142,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmagent_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmagent_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/The maximum number of concurrent insert requests/ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmagent_common_flags.md
-	sed -i '/The number of concurrent queues to each -remoteWrite.url./ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmagent_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmagent_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmagent_common_flags.md
 
 docs-update-vmalert-flags:
 ifndef TAG
@@ -173,10 +168,9 @@ endif
 	sed -i 's/\t/ /g' docs/victoriametrics/vmalert_common_flags.md
 	sed -i 's/\t/ /g' docs/victoriametrics/vmalert_enterprise_flags.md
 
-	# adjust flags with dynamic default values
-	# remove after https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680 implemented
-	sed -i '/Defines number of writers for concurrent writing into remote write endpoint./ s/(default [0-9]\+)/(default 2*cgroup.AvailableCPUs())/' docs/victoriametrics/vmalert_common_flags.md
-	sed -i '/The maximum number of concurrent goroutines to work with files;/ s/(default [0-9]\+)/(default fsutil.getDefaultConcurrency())/' docs/victoriametrics/vmalert_common_flags.md
+	# hide the machine-specific value of dynamic defaults, keeping the formula.
+	# the flagutil.New*WithDynamicDefault constructors print them as "(default <value> = <formula>)".
+	sed -i 's/(default [0-9]\+ = \(.*\))$$/(default \1)/' docs/victoriametrics/vmalert_common_flags.md
 
 docs-update-vmselect-flags:
 ifndef TAG

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,11 +26,14 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose the `vm_app_prev_shutdown_unclean` gauge. It is set to `1` when the previous process run didn't shut down cleanly. Added the `UncleanShutdown` [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml), which fires for 10 minutes after an unclean shutdown is detected. See [#8443](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show the selected time zone UTC offset next to the date/time controls and allow opening time zone settings from it. See [#11332](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11332).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/), and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): show how the default value is calculated for command-line flags which derive it from the number of available CPU cores. For example, `-maxConcurrentInserts` now prints `(default 16 = 2*cgroup.AvailableCPUs())` in `-help` output instead of `(default 16)`. Updated flags: `-search.maxConcurrentRequests`, `-search.maxWorkersPerQuery`, `-fs.maxConcurrency`, `-remoteWrite.concurrency`, `-remoteWrite.queues`. See [#9680](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680). Thanks to @Vandit1604 for contribution.
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix infinite loop in the OpenTelemetry Firehose ingestion endpoint (`/opentelemetry/api/v1/push`) when receiving a malformed record with an incomplete varint in the `data` field. Previously this caused the goroutine to spin forever, permanently consuming CPU until the process was restarted.
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): reuse connections to `-remoteWrite.url` when writing the results of recording rules and alerts. Previously every series was sent over a new connection, which left a lot of sockets in `TIME_WAIT` state and could exhaust the ephemeral port range. The number of idle connections can be tuned via the new `-remoteWrite.maxIdleConnections` command-line flag. Thanks @evkuzin for contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent process crash in `sort_by_label_numeric()` and `sort_by_label_numeric_desc()` when a label value contains a number with 309 or more digits. See [#11423](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11423).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fail the query request directly when there is not enough disk space to store temporary search results. Previously, such queries could lead to vmselect crash. See [#4688](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4688).
 
 ## [v1.150.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.150.0)
 

docs/victoriametrics/integrations/grafana/_index.md

@@ -50,7 +50,7 @@ If you don't see an option to create a data source - try contacting system admin
 Create [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure/)
 in Grafana. Follow the same connection instructions as for [VictoriaMetrics datasource](#VictoriaMetrics-datasource).
 
-In the "Type and version" section set the type to "Prometheus" and the version to at least "2.24.x".
+In the "Performance" section set the Prometheus type to "Prometheus" and the Prometheus version to at least "2.24.x".
 This allows Grafana to use a more efficient API to get label values:
 
 ![Datasource](datasource-prometheus.webp)

docs/victoriametrics/vmalert.md

@@ -1127,13 +1127,18 @@ Or for all rules within the [group](#groups) {{% available_from "v1.117.0" %}}.
 Just set `debug: true` in configuration and vmalert will start printing additional log messages:
 
 ```sh
-2022-09-15T13:35:41.155Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:41+02:00: query returned 0 series (elapsed: 5.896041ms, isPartial: false)
-2022-09-15T13:35:56.149Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29+by%28instance%29+%3E+0&step=15s&time=1663248945"
-2022-09-15T13:35:56.178Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: query returned 1 series (elapsed: 28.368208ms, isPartial: false)
-2022-09-15T13:35:56.178Z  DEBUG datasource request: executing POST request with params "denyPartialResponse=true&query=sum%28vm_tcplistener_conns%7Binstance%3D%22localhost%3A8429%22%7D%29&step=15s&time=1663248945"
-2022-09-15T13:35:56.179Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:35:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} created in state PENDING
+2026-08-20T08:21:29.464Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A20%3A00Z"
+2026-08-20T08:21:29.465Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:20:00Z: query returned 0 series (series_fetched: 0, elapsed: 1.075166ms, isPartial: false)
 ...
-2022-09-15T13:36:56.153Z  DEBUG alerting rule "TestGroup":"Conns" (2601299393013563564) at 2022-09-15T15:36:56+02:00: alert 10705778000901301787 {alertgroup="TestGroup",alertname="Conns",cluster="east-1",instance="localhost:8429",replica="a"} PENDING => FIRING: 1m0s since becoming active at 2022-09-15 15:35:56.126006 +0200 CEST m=+39.384575417
+2026-08-20T08:22:29.466Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A21%3A00Z"
+2026-08-20T08:22:29.468Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: query returned 2 series (series_fetched: 2, elapsed: 2.055916ms, isPartial: false)
+2026-08-20T08:22:29.469Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: alert 4671711516378822929 {alertgroup="vm-health",alertname="ServiceDown",instance="victoriametrics:8428",job="victoriametrics",severity="critical"} created in state PENDING
+2026-08-20T08:22:29.469Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:21:00Z: alert 6230585559362831632 {alertgroup="vm-health",alertname="ServiceDown",instance="vmagent:8429",job="vmagent",severity="critical"} created in state PENDING
+...
+2026-08-20T08:23:29.463Z        info    VictoriaMetrics/app/vmalert/datasource/client.go:262    DEBUG datasource request: executing POST request with params "http://victoriametrics:8428/api/v1/query?query=up%7Bjob%3D~%22.%2A%28victoriametrics%7Cvmselect%7Cvminsert%7Cvmstorage%7Cvmagent%7Cvmalert%7Cvmsingle%7Cvmalertmanager%7Cvmauth%29.%2A%22%7D&step=300s&time=2026-08-20T08%3A22%3A00Z"
+2026-08-20T08:23:29.465Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: query returned 2 series (series_fetched: 2, elapsed: 1.391416ms, isPartial: false)
+2026-08-20T08:23:29.466Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: alert 4671711516378822929 {alertgroup="vm-health",alertname="ServiceDown",instance="victoriametrics:8428",job="victoriametrics",severity="critical"} PENDING => FIRING: 1m0s since becoming active at 2026-08-20 08:21:00 +0000 UTC
+2026-08-20T08:23:29.466Z        info    VictoriaMetrics/app/vmalert/rule/alerting.go:273        DEBUG alerting rule "/etc/alerts/alerts-health.yml", "vm-health":"ServiceDown" (1340947595484135783) at 2026-08-20T08:22:00Z: alert 6230585559362831632 {alertgroup="vm-health",alertname="ServiceDown",instance="vmagent:8429",job="vmagent",severity="critical"} PENDING => FIRING: 1m0s since becoming active at 2026-08-20 08:21:00 +0000 UTC
 ```
 
 Sensitive info is stripped from the `curl` examples - see [security](#security) section for more details.

lib/appmetrics/appmetrics.go

@@ -65,6 +65,9 @@ func writePrometheusMetrics(w io.Writer) {
 	// Export start time and uptime in seconds
 	metrics.WriteGaugeUint64(w, "vm_app_start_timestamp", uint64(startTime.Unix()))
 	metrics.WriteGaugeUint64(w, "vm_app_uptime_seconds", uint64(time.Since(startTime).Seconds()))
+	if uncleanShutdownEnabled.Load() {
+		metrics.WriteGaugeUint64(w, "vm_app_prev_shutdown_unclean", uncleanShutdown)
+	}
 
 	// Export flags as metrics.
 	isSetMap := make(map[string]bool)

lib/appmetrics/osmetrics.go

@@ -13,13 +13,13 @@ type osInfo struct {
 	release string
 }
 
-var os osInfo
+var hostOS osInfo
 var initOSOnce sync.Once
 
 func writeOSMetrics(w io.Writer) {
 	initOSOnce.Do(initOS)
 
-	if os.name != "" {
-		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_os_info{os=%q, release=%q}`, os.name, os.release), 1)
+	if hostOS.name != "" {
+		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vm_os_info{os=%q, release=%q}`, hostOS.name, hostOS.release), 1)
 	}
 }

lib/appmetrics/osmetrics_darwin.go

@@ -8,7 +8,7 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "darwin"}
+	hostOS = osInfo{name: "darwin"}
 
 	out, err := exec.Command("sysctl", "-n", "kern.osrelease").Output()
 	if err != nil {
@@ -16,5 +16,5 @@ func initOS() {
 		return
 	}
 
-	os.release = strings.TrimSpace(string(out))
+	hostOS.release = strings.TrimSpace(string(out))
 }

lib/appmetrics/osmetrics_linux.go

@@ -7,7 +7,7 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "linux"}
+	hostOS = osInfo{name: "linux"}
 
 	var uname syscall.Utsname
 	if err := syscall.Uname(&uname); err != nil {
@@ -22,5 +22,5 @@ func initOS() {
 		}
 		ur = append(ur, byte(v))
 	}
-	os.release = string(ur)
+	hostOS.release = string(ur)
 }

lib/appmetrics/osmetrics_windows.go

@@ -8,12 +8,12 @@ import (
 )
 
 func initOS() {
-	os = osInfo{name: "windows"}
+	hostOS = osInfo{name: "windows"}
 
 	ver := windows.RtlGetVersion()
 	if ver == nil {
 		logger.Warnf("vm_os_info metric will miss release info since windows.RtlGetVersion returned nil version")
 		return
 	}
-	os.release = fmt.Sprintf("%d.%d.%d", ver.MajorVersion, ver.MinorVersion, ver.BuildNumber)
+	hostOS.release = fmt.Sprintf("%d.%d.%d", ver.MajorVersion, ver.MinorVersion, ver.BuildNumber)
 }

lib/appmetrics/unclean_shutdown.go

@@ -0,0 +1,53 @@
+package appmetrics
+
+import (
+	"os"
+	"path/filepath"
+	"sync/atomic"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+// UncleanShutdownMarkerFilename is the marker file used to detect a previous unclean shutdown.
+const UncleanShutdownMarkerFilename = ".vm_app_running"
+
+var (
+	uncleanShutdownEnabled atomic.Bool
+	uncleanShutdown        uint64
+)
+
+// MustCreateUncleanShutdownMarker creates an UncleanShutdownMarkerFilename marker file in the given dirPath.
+// Must be called once on program startup and paired with a single MustRemoveUncleanShutdownMarker call on exit.
+//
+// If the marker file already exists on startup, it indicates a previous unclean shutdown and uncleanShutdown is set to 1.
+func MustCreateUncleanShutdownMarker(dirPath string) {
+	if !uncleanShutdownEnabled.CompareAndSwap(false, true) {
+		logger.Fatalf("BUG: unclean shutdown marker was already initialized. It could only be called once")
+	}
+	marker := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+	f, err := os.OpenFile(marker, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
+	if err == nil {
+		fs.MustClose(f)
+		return
+	}
+	if os.IsExist(err) {
+		uncleanShutdown = 1
+		logger.Warnf("Previous shutdown was unclean since file %q exists. Please check logs and investigate the reason of unclean shutdown", marker)
+		return
+	}
+	logger.Panicf("FATAL: cannot create unclean shutdown marker %q: %s", marker, err)
+}
+
+// MustRemoveUncleanShutdownMarker removes the UncleanShutdownMarkerFilename marker file created by MustCreateUncleanShutdownMarker.
+// Must be called once, as late as possible before program exit.
+func MustRemoveUncleanShutdownMarker(dirPath string) {
+	if !uncleanShutdownEnabled.Load() {
+		logger.Fatalf("BUG: unclean shutdown marker was not initialized with MustCreateUncleanShutdownMarker call")
+	}
+	marker := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+	if err := os.Remove(marker); err != nil {
+		logger.Fatalf("FATAL: cannot remove unclean shutdown marker %q: %s", marker, err)
+	}
+	fs.MustSyncPath(dirPath)
+}

lib/appmetrics/unclean_shutdown_test.go

@@ -0,0 +1,60 @@
+package appmetrics
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestUncleanShutdownLifecycle(t *testing.T) {
+
+	t.Cleanup(func() {
+		uncleanShutdownEnabled.Store(false)
+	})
+	dirPath := t.TempDir()
+	markerPath := filepath.Join(dirPath, UncleanShutdownMarkerFilename)
+
+	// unclean logic is disabled. the unclean shutdown metric should not be exposed
+	var bb bytes.Buffer
+	writePrometheusMetrics(&bb)
+	if strings.Contains(bb.String(), "vm_app_prev_shutdown_unclean") {
+		t.Fatalf("unexpected unclean shutdown metric before starting the marker")
+	}
+
+	// clean start, the metric must report 0
+	MustCreateUncleanShutdownMarker(dirPath)
+	mustContainUncleanShutdownMetric(t, 0)
+	if _, err := os.Stat(markerPath); err != nil {
+		t.Fatalf("cannot stat the running marker after the first start: %s", err)
+	}
+	MustRemoveUncleanShutdownMarker(dirPath)
+	if _, err := os.Stat(markerPath); !os.IsNotExist(err) {
+		t.Fatalf("unexpected running marker after a clean shutdown; got error %v; want os.ErrNotExist", err)
+	}
+	uncleanShutdownEnabled.Store(false)
+
+	// simulate prev unclean shutdown, the metric must report 1
+	if err := os.WriteFile(markerPath, nil, 0600); err != nil {
+		t.Fatalf("cannot create test marker: %s", err)
+	}
+	MustCreateUncleanShutdownMarker(dirPath)
+	mustContainUncleanShutdownMetric(t, 1)
+	MustRemoveUncleanShutdownMarker(dirPath)
+	if _, err := os.Stat(markerPath); !os.IsNotExist(err) {
+		t.Fatalf("unexpected running marker after a clean shutdown; got error %v; want os.ErrNotExist", err)
+	}
+}
+
+func mustContainUncleanShutdownMetric(t *testing.T, value uint64) {
+	t.Helper()
+
+	var bb bytes.Buffer
+	writePrometheusMetrics(&bb)
+	want := "vm_app_prev_shutdown_unclean " + strconv.FormatUint(value, 10) + "\n"
+	if !strings.Contains(bb.String(), want) {
+		t.Fatalf("missing %q in the exported app metrics", want)
+	}
+}

lib/backup/fscommon/fscommon.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/backupnames"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
@@ -107,7 +108,7 @@ func appendFilesInternal(dst []string, d *os.File) ([]string, error) {
 }
 
 func isSpecialFile(name string) bool {
-	return name == "flock.lock" || name == backupnames.RestoreInProgressFilename || name == backupnames.RestoreMarkFileName || strings.HasSuffix(name, ".tmp")
+	return name == "flock.lock" || name == appmetrics.UncleanShutdownMarkerFilename || name == backupnames.RestoreInProgressFilename || name == backupnames.RestoreMarkFileName || strings.HasSuffix(name, ".tmp")
 }
 
 // RemoveEmptyDirs recursively removes empty directories under the given dir.

lib/flagutil/array.go

@@ -39,8 +39,29 @@ func NewArrayBool(name, description string) *ArrayBool {
 }
 
 // NewArrayInt returns new ArrayInt with the given name, defaultValue and description.
+//
+// -help shows defaultValue as a plain number. Use NewArrayIntWithDynamicDefault when
+// defaultValue is calculated at runtime.
 func NewArrayInt(name string, defaultValue int, description string) *ArrayInt {
-	description += fmt.Sprintf(" (default %d)", defaultValue)
+	return newArrayInt(name, defaultValue, strconv.Itoa(defaultValue), description)
+}
+
+// NewArrayIntWithDynamicDefault returns new ArrayInt with the given name, defaultValue and description.
+//
+// Use it instead of NewArrayInt when defaultValue is calculated at runtime.
+// See NewIntWithDynamicDefault for why such a value needs a hint.
+func NewArrayIntWithDynamicDefault(name string, defaultValue int, defaultValueHint, description string) *ArrayInt {
+	if defaultValueHint == "" {
+		panic(fmt.Sprintf("BUG: missing defaultValueHint for -%s", name))
+	}
+	return newArrayInt(name, defaultValue, fmt.Sprintf("%d = %s", defaultValue, defaultValueHint), description)
+}
+
+// newArrayInt registers an int array flag, which shows defaultValueText as its default in -help.
+//
+// Array flags keep the default in the description, since flag.Var hides an empty DefValue.
+func newArrayInt(name string, defaultValue int, defaultValueText, description string) *ArrayInt {
+	description += fmt.Sprintf(" (default %s)", defaultValueText)
 	description += "\nSupports `array` of values separated by comma or specified via multiple flags."
 	description += "\nEmpty values are set to default value."
 	a := &ArrayInt{

lib/flagutil/flag.go

@@ -7,6 +7,25 @@ import (
 	"strings"
 )
 
+// NewIntWithDynamicDefault returns a new int flag with the given name, defaultValue and description.
+//
+// Use it instead of flag.Int when defaultValue is calculated at runtime, for example
+// from the number of CPU cores. Such a value differs per machine, so -help shows both
+// the value and defaultValueHint, for example "16 = 2 * availableCPUs".
+//
+// Only -help output changes. The flag value stays defaultValue.
+func NewIntWithDynamicDefault(name string, defaultValue int, defaultValueHint, description string) *int {
+	if defaultValueHint == "" {
+		panic(fmt.Sprintf("BUG: missing defaultValueHint for -%s", name))
+	}
+	p := flag.Int(name, defaultValue, description)
+
+	// DefValue is only the text shown by -help: "default value (as text); for usage message".
+	flag.Lookup(name).DefValue = fmt.Sprintf("%d = %s", defaultValue, defaultValueHint)
+
+	return p
+}
+
 // WriteFlags writes all the explicitly set flags to w.
 func WriteFlags(w io.Writer) {
 	flag.Visit(func(f *flag.Flag) {

lib/flagutil/flag_test.go

@@ -0,0 +1,51 @@
+package flagutil
+
+import (
+	"flag"
+	"strings"
+	"testing"
+)
+
+// The flags are registered at package level, since flag registration panics when it repeats.
+var (
+	fooFlagIntDynamicDefault      = NewIntWithDynamicDefault("fooFlagIntDynamicDefault", 42, "2 * availableCPUs", "test")
+	fooFlagArrayIntDynamicDefault = NewArrayIntWithDynamicDefault("fooFlagArrayIntDynamicDefault", 42, "2 * availableCPUs", "test")
+	fooFlagArrayIntPlainDefault   = NewArrayInt("fooFlagArrayIntPlainDefault", 42, "test")
+)
+
+func TestNewIntWithDynamicDefaultSuccess(t *testing.T) {
+	// -help must show the value together with the hint.
+	f := flag.Lookup("fooFlagIntDynamicDefault")
+	if f.DefValue != "42 = 2 * availableCPUs" {
+		t.Fatalf("unexpected DefValue; got %q; want %q", f.DefValue, "42 = 2 * availableCPUs")
+	}
+
+	// the flag value must stay the calculated one.
+	if *fooFlagIntDynamicDefault != 42 {
+		t.Fatalf("unexpected flag value; got %d; want %d", *fooFlagIntDynamicDefault, 42)
+	}
+}
+
+func TestNewArrayIntWithDynamicDefaultSuccess(t *testing.T) {
+	// array flags keep the default in the description, so the hint must go there.
+	f := flag.Lookup("fooFlagArrayIntDynamicDefault")
+	if !strings.Contains(f.Usage, "(default 42 = 2 * availableCPUs)") {
+		t.Fatalf("missing the hint in the flag description; got %q", f.Usage)
+	}
+
+	// the default value must stay the calculated one.
+	if n := fooFlagArrayIntDynamicDefault.GetOptionalArg(0); n != 42 {
+		t.Fatalf("unexpected default value; got %d; want %d", n, 42)
+	}
+}
+
+func TestNewArrayIntKeepsPlainDefault(t *testing.T) {
+	// NewArrayInt must keep showing a plain number, since it shares the body with the dynamic one.
+	f := flag.Lookup("fooFlagArrayIntPlainDefault")
+	if !strings.Contains(f.Usage, "(default 42)") {
+		t.Fatalf("unexpected flag description; got %q", f.Usage)
+	}
+	if n := fooFlagArrayIntPlainDefault.GetOptionalArg(0); n != 42 {
+		t.Fatalf("unexpected default value; got %d; want %d", n, 42)
+	}
+}

lib/fs/fsutil/concurrency.go

@@ -1,14 +1,15 @@
 package fsutil
 
 import (
-	"flag"
 	"sync"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
-var maxConcurrency = flag.Int("fs.maxConcurrency", getDefaultConcurrency(), "The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency "+
-	"on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph")
+var maxConcurrency = flagutil.NewIntWithDynamicDefault("fs.maxConcurrency", getDefaultConcurrency(), "fsutil.getDefaultConcurrency()",
+	"The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency "+
+		"on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph")
 
 func getDefaultConcurrency() int {
 	n := min(16*cgroup.AvailableCPUs(), 256)

lib/querytracer/otlp.go

@@ -1,122 +0,0 @@
-package querytracer
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"time"
-)
-
-// OTLPTrace is an intermediate representation of a finished tracer tree
-// suitable for conversion to OTLP protobuf spans.
-type OTLPTrace struct {
-	TraceID string
-	Spans   []OTLPSpan
-}
-
-// OTLPSpan represents a single span ready for OTLP export.
-type OTLPSpan struct {
-	TraceID      string
-	SpanID       string
-	ParentSpanID string // empty for root
-	Name         string
-	StartNano    uint64
-	EndNano      uint64
-	Events       []OTLPEvent
-}
-
-// OTLPEvent is a timestamped annotation within a span (from Printf leaf nodes).
-type OTLPEvent struct {
-	TimeNano uint64
-	Name     string
-}
-
-// ToOTLPTrace converts the finished tracer tree into a flat list of OTLP spans.
-// It must be called after Done/Donef.
-func (t *Tracer) ToOTLPTrace() *OTLPTrace {
-	if t == nil {
-		return nil
-	}
-	traceID := newTraceID()
-	tr := &OTLPTrace{TraceID: traceID}
-	collectSpans(tr, t, traceID, "", t.startTime)
-	return tr
-}
-
-// collectSpans recursively walks the tracer tree and populates tr.Spans.
-// parentSpanID is empty for the root.
-// prevTime is used only when approximating timestamps for JSON-embedded spans.
-func collectSpans(tr *OTLPTrace, t *Tracer, traceID, parentSpanID string, prevTime time.Time) {
-	if t.span != nil {
-		collectJSONSpans(tr, t.span, traceID, parentSpanID, prevTime)
-		return
-	}
-
-	isLeaf := t.doneTime.Equal(t.startTime)
-	if isLeaf {
-		// Printf leaf: no span to emit here; the caller handles attaching it.
-		return
-	}
-
-	// Regular span.
-	spanID := newSpanID()
-	spanIdx := len(tr.Spans)
-	tr.Spans = append(tr.Spans, OTLPSpan{
-		TraceID:      traceID,
-		SpanID:       spanID,
-		ParentSpanID: parentSpanID,
-		Name:         t.message,
-		StartNano:    uint64(t.startTime.UnixNano()),
-		EndNano:      uint64(t.doneTime.UnixNano()),
-	})
-
-	// Process children. Leaf children become events on this span; non-leaf
-	// children recurse and produce their own spans.
-	childPrev := t.startTime
-	for _, child := range t.children {
-		if child.span == nil && child.doneTime.Equal(child.startTime) {
-			// Printf leaf: attach as span event.
-			tr.Spans[spanIdx].Events = append(tr.Spans[spanIdx].Events, OTLPEvent{
-				TimeNano: uint64(child.startTime.UnixNano()),
-				Name:     child.message,
-			})
-			continue
-		}
-		collectSpans(tr, child, traceID, spanID, childPrev)
-		if !child.doneTime.IsZero() && !child.doneTime.Equal(child.startTime) {
-			childPrev = child.doneTime
-		}
-	}
-}
-
-// collectJSONSpans converts a span tree that was deserialized via AddJSON.
-// Timestamps are approximated from prevTime since JSON spans only carry duration_msec.
-func collectJSONSpans(tr *OTLPTrace, s *span, traceID, parentSpanID string, prevTime time.Time) {
-	startNano := uint64(prevTime.UnixNano())
-	durationNano := uint64(s.DurationMsec * float64(time.Millisecond))
-	spanID := newSpanID()
-	tr.Spans = append(tr.Spans, OTLPSpan{
-		TraceID:      traceID,
-		SpanID:       spanID,
-		ParentSpanID: parentSpanID,
-		Name:         s.Message,
-		StartNano:    startNano,
-		EndNano:      startNano + durationNano,
-	})
-	childPrev := prevTime
-	for _, child := range s.Children {
-		collectJSONSpans(tr, child, traceID, spanID, childPrev)
-		childPrev = childPrev.Add(time.Duration(child.DurationMsec * float64(time.Millisecond)))
-	}
-}
-
-func newTraceID() string {
-	var b [16]byte
-	_, _ = rand.Read(b[:])
-	return hex.EncodeToString(b[:])
-}
-
-func newSpanID() string {
-	var b [8]byte
-	_, _ = rand.Read(b[:])
-	return hex.EncodeToString(b[:])
-}

lib/querytracer/push/pb.go

@@ -1,156 +0,0 @@
-package push
-
-// This file contains marshal-only protobuf types for OTLP trace export.
-// Copied and trimmed from VictoriaTraces/lib/protoparser/opentelemetry/pb/
-// Only marshaling is needed since this package only pushes traces outward.
-
-import (
-	"encoding/hex"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// exportTraceServiceRequest is the top-level OTLP protobuf message.
-type exportTraceServiceRequest struct {
-	ResourceSpans []*resourceSpans
-}
-
-func (r *exportTraceServiceRequest) marshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	mm := m.MessageMarshaler()
-	for _, rs := range r.ResourceSpans {
-		rs.marshalProtobuf(mm.AppendMessage(1))
-	}
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-// resourceSpans groups spans from a single resource (e.g. service instance).
-type resourceSpans struct {
-	Resource   resource
-	ScopeSpans []*scopeSpans
-}
-
-func (rs *resourceSpans) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	rs.Resource.marshalProtobuf(mm.AppendMessage(1))
-	for _, ss := range rs.ScopeSpans {
-		ss.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-// resource holds resource-level attributes (service.name, etc.).
-type resource struct {
-	Attributes []*keyValue
-}
-
-func (r *resource) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, a := range r.Attributes {
-		a.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-// scopeSpans groups spans from a single instrumentation scope.
-type scopeSpans struct {
-	Scope instrumentationScope
-	Spans []*span
-}
-
-func (ss *scopeSpans) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	ss.Scope.marshalProtobuf(mm.AppendMessage(1))
-	for _, s := range ss.Spans {
-		s.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-// instrumentationScope identifies the library that produced the spans.
-type instrumentationScope struct {
-	Name    string
-	Version string
-}
-
-func (is *instrumentationScope) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, is.Name)
-	mm.AppendString(2, is.Version)
-}
-
-// span represents a single operation within a trace.
-type span struct {
-	// TraceID is a 32-char lowercase hex string (16 bytes).
-	TraceID string
-	// SpanID is a 16-char lowercase hex string (8 bytes).
-	SpanID string
-	// ParentSpanID is a 16-char lowercase hex string; empty for root spans.
-	ParentSpanID      string
-	Name              string
-	StartTimeUnixNano uint64
-	EndTimeUnixNano   uint64
-	Attributes        []*keyValue
-	Events            []*spanEvent
-}
-
-func (s *span) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	traceID, err := hex.DecodeString(s.TraceID)
-	if err != nil {
-		traceID = []byte(s.TraceID)
-	}
-	mm.AppendBytes(1, traceID)
-
-	spanID, err := hex.DecodeString(s.SpanID)
-	if err != nil {
-		spanID = []byte(s.SpanID)
-	}
-	mm.AppendBytes(2, spanID)
-
-	// field 3: trace_state — omitted
-
-	parentSpanID, err := hex.DecodeString(s.ParentSpanID)
-	if err != nil {
-		parentSpanID = []byte(s.ParentSpanID)
-	}
-	mm.AppendBytes(4, parentSpanID)
-
-	mm.AppendString(5, s.Name)
-	// field 6: kind — omitted (INTERNAL=1 is default)
-	mm.AppendFixed64(7, s.StartTimeUnixNano)
-	mm.AppendFixed64(8, s.EndTimeUnixNano)
-	for _, a := range s.Attributes {
-		a.marshalProtobuf(mm.AppendMessage(9))
-	}
-	for _, e := range s.Events {
-		e.marshalProtobuf(mm.AppendMessage(11))
-	}
-}
-
-// spanEvent is a time-stamped annotation within a span.
-type spanEvent struct {
-	TimeUnixNano uint64
-	Name         string
-}
-
-func (se *spanEvent) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendFixed64(1, se.TimeUnixNano)
-	mm.AppendString(2, se.Name)
-}
-
-// keyValue is an OTLP attribute key-value pair.
-type keyValue struct {
-	Key   string
-	Value anyValue
-}
-
-func (kv *keyValue) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, kv.Key)
-	kv.Value.marshalProtobuf(mm.AppendMessage(2))
-}
-
-// anyValue holds a single string attribute value (sufficient for our use case).
-type anyValue struct {
-	StringValue string
-}
-
-func (av *anyValue) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, av.StringValue)
-}

lib/querytracer/push/push.go

@@ -1,233 +0,0 @@
-// Package push implements background exporting of VictoriaMetrics query traces
-// to VictoriaTraces in OTLP protobuf format over HTTP.
-package push
-
-import (
-	"bytes"
-	"compress/gzip"
-	"flag"
-	"fmt"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	pushURL = flag.String("search.traceExportURL", "", "If set, query traces are exported to this URL in OTLP protobuf format. "+
-		"For example, -search.traceExportURL=http://victoria-traces:4318/insert/opentelemetry/v1/traces . "+
-		"See https://docs.victoriametrics.com/victoriametrics/query-tracing/")
-	minTraceDuration = flag.Duration("search.traceExportMinDuration", 0, "Minimum query duration for exporting traces via -search.traceExportURL. "+
-		"Traces for faster queries are dropped. 0 means all traces are exported.")
-)
-
-var (
-	pushesTotal  = metrics.NewCounter(`vm_trace_export_pushes_total`)
-	errorsTotal  = metrics.NewCounter(`vm_trace_export_errors_total`)
-	droppedTotal = metrics.NewCounter(`vm_trace_export_dropped_total`)
-)
-
-// queueCap is the maximum number of pending traces waiting to be exported.
-const queueCap = 1000
-
-var (
-	queue chan *querytracer.OTLPTrace
-
-	stopCh chan struct{}
-	wg     sync.WaitGroup
-)
-
-// IsEnabled returns true when trace export is configured.
-func IsEnabled() bool {
-	return *pushURL != ""
-}
-
-// MinTraceDuration returns the configured minimum trace duration threshold.
-func MinTraceDuration() time.Duration {
-	return *minTraceDuration
-}
-
-// Init starts the background export goroutine. Must be called after flag.Parse and logger.Init.
-func Init() {
-	if !IsEnabled() {
-		return
-	}
-	queue = make(chan *querytracer.OTLPTrace, queueCap)
-	stopCh = make(chan struct{})
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		runExporter()
-	}()
-	logger.Infof("started query trace exporter to %s", *pushURL)
-}
-
-// Stop drains pending traces and stops the background goroutine.
-// Must be called during graceful shutdown before the process exits.
-func Stop() {
-	if !IsEnabled() {
-		return
-	}
-	close(stopCh)
-	wg.Wait()
-}
-
-// Push enqueues t for async export. t must be a finished tracer.
-// Traces are silently dropped if the queue is full.
-func Push(t *querytracer.Tracer) {
-	if !t.Enabled() {
-		return
-	}
-	tr := t.ToOTLPTrace()
-	if tr == nil {
-		return
-	}
-	select {
-	case queue <- tr:
-	default:
-		droppedTotal.Inc()
-	}
-}
-
-// runExporter reads from queue and sends batches to the configured endpoint.
-func runExporter() {
-	ticker := time.NewTicker(time.Second)
-	defer ticker.Stop()
-
-	var batch []*querytracer.OTLPTrace
-	for {
-		select {
-		case tr := <-queue:
-			batch = append(batch, tr)
-			if len(batch) >= 100 {
-				flushBatch(batch)
-				batch = batch[:0]
-			}
-		case <-ticker.C:
-			if len(batch) > 0 {
-				flushBatch(batch)
-				batch = batch[:0]
-			}
-		case <-stopCh:
-			// Drain remaining items.
-			for {
-				select {
-				case tr := <-queue:
-					batch = append(batch, tr)
-				default:
-					if len(batch) > 0 {
-						flushBatch(batch)
-					}
-					return
-				}
-			}
-		}
-	}
-}
-
-// flushBatch serializes and POSTs a batch of traces.
-func flushBatch(batch []*querytracer.OTLPTrace) {
-	data, err := marshalBatch(batch)
-	if err != nil {
-		logger.Errorf("cannot marshal trace batch: %s", err)
-		errorsTotal.Inc()
-		return
-	}
-	if err := postData(data); err != nil {
-		logger.Warnf("cannot export query traces to %s: %s", *pushURL, err)
-		errorsTotal.Inc()
-		return
-	}
-	pushesTotal.Add(len(batch))
-}
-
-// marshalBatch packs all traces into a single ExportTraceServiceRequest and gzip-compresses it.
-func marshalBatch(batch []*querytracer.OTLPTrace) ([]byte, error) {
-	svcVersion := buildinfo.Version
-
-	// Build resource attributes once (same for all spans in this process).
-	resAttrs := []*keyValue{
-		{Key: "service.name", Value: anyValue{StringValue: "victoriametrics"}},
-		{Key: "service.version", Value: anyValue{StringValue: svcVersion}},
-	}
-
-	var pbSpans []*span
-	for _, tr := range batch {
-		for i := range tr.Spans {
-			s := &tr.Spans[i]
-			pbSpan := &span{
-				TraceID:           s.TraceID,
-				SpanID:            s.SpanID,
-				ParentSpanID:      s.ParentSpanID,
-				Name:              s.Name,
-				StartTimeUnixNano: s.StartNano,
-				EndTimeUnixNano:   s.EndNano,
-			}
-			for _, ev := range s.Events {
-				pbSpan.Events = append(pbSpan.Events, &spanEvent{
-					TimeUnixNano: ev.TimeNano,
-					Name:         ev.Name,
-				})
-			}
-			pbSpans = append(pbSpans, pbSpan)
-		}
-	}
-
-	req := &exportTraceServiceRequest{
-		ResourceSpans: []*resourceSpans{
-			{
-				Resource: resource{Attributes: resAttrs},
-				ScopeSpans: []*scopeSpans{
-					{
-						Scope: instrumentationScope{
-							Name:    "victoriametrics/querytracer",
-							Version: svcVersion,
-						},
-						Spans: pbSpans,
-					},
-				},
-			},
-		},
-	}
-
-	raw := req.marshalProtobuf(nil)
-
-	var buf bytes.Buffer
-	gz := gzip.NewWriter(&buf)
-	if _, err := gz.Write(raw); err != nil {
-		return nil, fmt.Errorf("cannot gzip trace data: %w", err)
-	}
-	if err := gz.Close(); err != nil {
-		return nil, fmt.Errorf("cannot close gzip writer: %w", err)
-	}
-	return buf.Bytes(), nil
-}
-
-var httpClient = &http.Client{
-	Timeout: 10 * time.Second,
-}
-
-// postData sends gzip-compressed protobuf data to the configured endpoint.
-func postData(data []byte) error {
-	req, err := http.NewRequest(http.MethodPost, *pushURL, bytes.NewReader(data))
-	if err != nil {
-		return fmt.Errorf("cannot create HTTP request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/x-protobuf")
-	req.Header.Set("Content-Encoding", "gzip")
-
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("HTTP request failed: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode/100 != 2 {
-		return fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
-	}
-	return nil
-}

lib/writeconcurrencylimiter/concurrencylimiter.go

@@ -10,16 +10,18 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	maxConcurrentInserts = flag.Int("maxConcurrentInserts", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent insert requests. "+
-		"Set higher value when clients send data over slow networks. "+
-		"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. "+
-		"See also -insert.maxQueueDuration")
+	maxConcurrentInserts = flagutil.NewIntWithDynamicDefault("maxConcurrentInserts", 2*cgroup.AvailableCPUs(), "2*cgroup.AvailableCPUs()",
+		"The maximum number of concurrent insert requests. "+
+			"Set higher value when clients send data over slow networks. "+
+			"Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. "+
+			"See also -insert.maxQueueDuration")
 	maxQueueDuration = flag.Duration("insert.maxQueueDuration", time.Minute, "The maximum duration to wait in the queue when -maxConcurrentInserts "+
 		"concurrent insert requests are executed")
 )