lib/memory: expose host memory and cgroup memory limit as separate metrics

Signed-off-by: Zasda Mikail <zmikail@victoriametrics.com>

app/vmselect/netstorage/netstorage.go

@@ -460,8 +460,7 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 			initUnpackWork(upw, br)
 			upw.unpack(tmpBlock)
 			if upw.err != nil {
-				err = upw.err
-				break
+				return dst, upw.err
 			}
 			samples += len(upw.sb.Timestamps)
 			if *maxSamplesPerSeries > 0 && samples > *maxSamplesPerSeries {
@@ -475,11 +474,7 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 		}
 		putTmpStorageBlock(tmpBlock)
 		putUnpackWork(upw)
-		if err != nil {
-			for _, sb := range dst {
-				putSortBlock(sb)
-			}
-		}
+
 		return dst, err
 	}
 
@@ -545,11 +540,6 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 		}
 		putUnpackWork(upw)
 	}
-	if firstErr != nil {
-		for _, sb := range dst {
-			putSortBlock(sb)
-		}
-	}
 
 	return dst, firstErr
 }

app/vmselect/prometheus/prometheus.go

@@ -703,7 +703,7 @@ var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	defer seriesCountDuration.UpdateDuration(startTime)
 
-	deadline := searchutil.GetDeadlineForLabelsAPI(r, startTime)
+	deadline := searchutil.GetDeadlineForStatusRequest(r, startTime)
 	n, err := netstorage.SeriesCount(nil, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %w", err)

app/vmselect/promql/eval.go

@@ -1072,7 +1072,6 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	var samplesScannedTotal atomic.Uint64
 	keepMetricNames := getKeepMetricNames(expr)
 	tsw := getTimeseriesByWorkerID()
-	defer putTimeseriesByWorkerID(tsw)
 	seriesByWorkerID := tsw.byWorkerID
 	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64, workerID uint) ([]float64, []int64) {
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
@@ -1095,6 +1094,7 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	for i := range seriesByWorkerID {
 		tss = append(tss, seriesByWorkerID[i].tss...)
 	}
+	putTimeseriesByWorkerID(tsw)
 
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal.Load()))
 	qt.Printf("rollup %s() over %d series returned by subquery: series=%d, samplesScanned=%d", funcName, len(tssSQ), len(tss), samplesScannedTotal.Load())
@@ -1973,7 +1973,6 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 
 	var samplesScannedTotal atomic.Uint64
 	tsw := getTimeseriesByWorkerID()
-	defer putTimeseriesByWorkerID(tsw)
 	seriesByWorkerID := tsw.byWorkerID
 	seriesLen := rss.Len()
 	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
@@ -2000,6 +1999,7 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 	for i := range seriesByWorkerID {
 		tss = append(tss, seriesByWorkerID[i].tss...)
 	}
+	putTimeseriesByWorkerID(tsw)
 
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal.Load()))
 	qt.Printf("samplesScanned=%d", samplesScannedTotal.Load())

app/vmselect/searchutil/searchutil.go

@@ -56,7 +56,7 @@ func GetDeadlineForExport(r *http.Request, startTime time.Time) Deadline {
 	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxExportDuration")
 }
 
-// GetDeadlineForLabelsAPI returns deadline for the given request to /api/v1/labels, /api/v1/label/.../values, /api/v1/series or /api/v1/series/count
+// GetDeadlineForLabelsAPI returns deadline for the given request to /api/v1/labels, /api/v1/label/.../values or /api/v1/series
 func GetDeadlineForLabelsAPI(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxLabelsAPIDuration.Milliseconds()
 	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxLabelsAPIDuration")

docs/victoriametrics/Articles.md

@@ -117,10 +117,6 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [Claude Code: creating Kubernetes debugging AI Agent for VictoriaMetrics](https://rtfm.co.ua/en/claude-code-creating-kubernetes-debugging-ai-agent-for-victoriametrics/)
 * [OpenTelemetry: OTel Collectors in Kubernetes and VictoriaMetrics Stack integration](https://itnext.io/opentelemetry-otel-collectors-in-kubernetes-and-victoriametrics-stack-integration-d907ed0a15a0)
 * [VictoriaMetrics vs Prometheus: my default, and when I still pick Prometheus](https://jorijn.com/en/blog/victoriametrics-vs-prometheus/)
-* [LiteLLM: Monitoring with VictoriaMetrics – Alerts and Grafana](https://rtfm.co.ua/en/litellm-monitoring-with-victoriametrics-alerts-and-grafana/)
-* [LiteLLM: AI Gateway on Kubernetes and Metrics in VictoriaMetrics](https://rtfm.co.ua/en/litellm-ai-gateway-on-kubernetes-and-metrics-in-victoriametrics/)
-* [LiteLLM: Metrics, Traces, and VictoriaMetrics Stack Integration](https://rtfm.co.ua/en/litellm-metrics-traces-and-victoriametrics-stack-integration/)
-* [llama.cpp: Metrics and Monitoring with VictoriaMetrics](https://rtfm.co.ua/en/llama-cpp-metrics-and-monitoring-with-victoriametrics/)
 
 ## Third-party articles and slides about VictoriaLogs
 

docs/victoriametrics/Quick-Start.md

@@ -60,14 +60,6 @@ tar xzf victoria-metrics-linux-amd64-v1.150.0.tar.gz
 
 The binary is self-contained and requires no installation - it is ready to run as is.
 
-Alternatively, on macOS or Linux with [Homebrew](https://brew.sh/):
-
-```sh
-brew install victoriametrics/tap/victoriametrics
-```
-
-This installs the same release binary as `victoria-metrics` and also provides a `brew services start victoriametrics` service definition.
-
 ### Step 2: Start VictoriaMetrics
 
 Starting VictoriaMetrics is as simple as executing the binary, with no arguments at all.

docs/victoriametrics/README.md

@@ -12,7 +12,7 @@ VictoriaMetrics is a fast, cost-effective, and scalable solution for monitoring
 {width="600"}
 
 - **Case studies**: [Grammarly, Roblox, Wix, Spotify,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
-- **Available**: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), Docker images on [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Homebrew tap](https://github.com/VictoriaMetrics/homebrew-tap) (`brew install victoriametrics/tap/victoriametrics`), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
+- **Available**: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), Docker images on [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
 - **Deployment types**: [Single-node version](https://docs.victoriametrics.com/) and [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) under [Apache License 2.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE).
 - **Getting started:** Read [key concepts](https://docs.victoriametrics.com/victoriametrics/keyconcepts/) and follow the
   [quick start guide](https://docs.victoriametrics.com/victoriametrics/quick-start/).

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,18 +26,16 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): both can now be installed from the official [Homebrew tap](https://github.com/VictoriaMetrics/homebrew-tap) with `brew install victoriametrics/tap/victoriametrics` and `brew install victoriametrics/tap/vmctl`. The formulas install the prebuilt release binaries and include a service definition for `victoria-metrics`.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose the `vm_app_prev_shutdown_unclean` gauge. It is set to `1` when the previous process run didn't shut down cleanly. Added the `UncleanShutdown` [alerting rule](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml), which fires for 10 minutes after an unclean shutdown is detected. See [#8443](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8443).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show the selected time zone UTC offset next to the date/time controls and allow opening time zone settings from it. See [#11332](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11332).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/), and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): show how the default value is calculated for command-line flags which derive it from the number of available CPU cores. For example, `-maxConcurrentInserts` now prints `(default 16 = 2*cgroup.AvailableCPUs())` in `-help` output instead of `(default 16)`. Updated flags: `-search.maxConcurrentRequests`, `-search.maxWorkersPerQuery`, `-fs.maxConcurrency`, `-remoteWrite.concurrency`, `-remoteWrite.queues`. See [#9680](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9680). Thanks to @Vandit1604 for contribution.
+* FEATURE: all VictoriaMetrics components: expose `process_cpu_cores_hardware` and `process_cpu_cores_cgroup_limit` metrics. These metrics report the host CPU count and the configured cgroup CPU quota. The existing `process_cpu_cores_available` metric continues to report the effective CPU limit.
+* FEATURE: all VictoriaMetrics components: expose `process_memory_hardware_bytes` and `process_memory_cgroup_limit_bytes` metrics. These metrics report host RAM and the configured cgroup memory limit. The existing `process_memory_limit_bytes` metric continues to report the effective memory limit used for caches.
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix infinite loop in the OpenTelemetry Firehose ingestion endpoint (`/opentelemetry/api/v1/push`) when receiving a malformed record with an incomplete varint in the `data` field. Previously this caused the goroutine to spin forever, permanently consuming CPU until the process was restarted.
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): reuse connections to `-remoteWrite.url` when writing the results of recording rules and alerts. Previously every series was sent over a new connection, which left a lot of sockets in `TIME_WAIT` state and could exhaust the ephemeral port range. The number of idle connections can be tuned via the new `-remoteWrite.maxIdleConnections` command-line flag. Thanks @evkuzin for contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent process crash in `sort_by_label_numeric()` and `sort_by_label_numeric_desc()` when a label value contains a number with 309 or more digits. See [#11423](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11423).
-* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply default query timeout to `/api/v1/series/count` requests. It used `-search.maxStatusRequestDuration` flag value instead of `-search.maxLabelsAPIDuration`. See [#11422](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11422).
-* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly re-use memory if query aggregation returns error. See [#11426](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11426).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fail the query request directly when there is not enough disk space to store temporary search results. Previously, such queries could lead to vmselect crash. See [#4688](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4688).
-* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): consistently re-use memory during storage blocks unpacking on parsing storage block error. See [#11421](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11421).
 
 ## [v1.150.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.150.0)
 

docs/victoriametrics/vmctl/vmctl.md

@@ -31,7 +31,6 @@ VictoriaMetrics single or cluster version.
 vmctl command-line tool is available as:
 * docker images at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmctl/) and [Quay](https://quay.io/repository/victoriametrics/vmctl?tab=tags)
 * [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) as part of `vmutils` package
-* [Homebrew tap](https://github.com/VictoriaMetrics/homebrew-tap) on macOS and Linux: `brew install victoriametrics/tap/vmctl`. The installed binary is named `vmctl`, not `vmctl-prod`.
 
 Download and unpack vmctl:
 ```sh

lib/cgroup/cpu.go

@@ -21,14 +21,23 @@ func AvailableCPUs() int {
 }
 
 func init() {
+	cpuCoresTotal := float64(runtime.NumCPU())
+	cpuCoresAvailable := cpuCoresTotal
+
 	cpuQuota := getCPUQuota()
 	if cpuQuota > 0 {
 		updateGOMAXPROCSToCPUQuota(cpuQuota)
 	}
-	cpuCoresAvailable := float64(runtime.NumCPU())
 	if cpuQuota > 0 && cpuCoresAvailable > cpuQuota {
 		cpuCoresAvailable = cpuQuota
 	}
+
+	metrics.NewGauge(`process_cpu_cores_hardware`, func() float64 {
+		return cpuCoresTotal
+	})
+	metrics.NewGauge(`process_cpu_cores_cgroup_limit`, func() float64 {
+		return cpuQuota
+	})
 	metrics.NewGauge(`process_cpu_cores_available`, func() float64 {
 		return cpuCoresAvailable
 	})

lib/memory/memory.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -15,9 +17,33 @@ var (
 	allowedBytes   = flagutil.NewBytes("memory.allowedBytes", 0, `Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).`)
 )
 
-var _ = metrics.NewGauge("process_memory_limit_bytes", func() float64 {
-	return float64(memoryLimit)
-})
+func init() {
+	memTotal := float64(physicalMemory())
+	memCgroup := float64(cgroupMemoryLimit(int(memTotal)))
+
+	metrics.NewGauge("process_memory_hardware_bytes", func() float64 {
+		return memTotal
+	})
+	metrics.NewGauge("process_memory_cgroup_limit_bytes", func() float64 {
+		return memCgroup
+	})
+	metrics.NewGauge("process_memory_limit_bytes", func() float64 {
+		return float64(memoryLimit)
+	})
+}
+
+func cgroupMemoryLimit(totalMem int) int64 {
+	mem := cgroup.GetMemoryLimit()
+	if mem <= 0 || int64(int(mem)) != mem || int(mem) > totalMem {
+		// Try reading hierarchical memory limit.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/699
+		mem = cgroup.GetHierarchicalMemoryLimit()
+		if mem <= 0 || int64(int(mem)) != mem || int(mem) > totalMem {
+			return 0
+		}
+	}
+	return mem
+}
 
 var (
 	allowedMemory   int

lib/memory/memory_linux.go

@@ -3,13 +3,12 @@ package memory
 import (
 	"syscall"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 const maxInt = int(^uint(0) >> 1)
 
-func sysTotalMemory() int {
+func physicalMemory() int {
 	var si syscall.Sysinfo_t
 	if err := syscall.Sysinfo(&si); err != nil {
 		logger.Panicf("FATAL: error in syscall.Sysinfo: %s", err)
@@ -18,14 +17,14 @@ func sysTotalMemory() int {
 	if uint64(maxInt)/uint64(si.Totalram) > uint64(si.Unit) {
 		totalMem = int(uint64(si.Totalram) * uint64(si.Unit))
 	}
-	mem := cgroup.GetMemoryLimit()
-	if mem <= 0 || int64(int(mem)) != mem || int(mem) > totalMem {
-		// Try reading hierarchical memory limit.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/699
-		mem = cgroup.GetHierarchicalMemoryLimit()
-		if mem <= 0 || int64(int(mem)) != mem || int(mem) > totalMem {
-			return totalMem
-		}
+	return totalMem
+}
+
+func sysTotalMemory() int {
+	totalMem := physicalMemory()
+	mem := cgroupMemoryLimit(totalMem)
+	if mem <= 0 {
+		return totalMem
 	}
 	return int(mem)
 }

lib/memory/memory_other.go

@@ -0,0 +1,7 @@
+//go:build !linux
+
+package memory
+
+func physicalMemory() int {
+	return sysTotalMemory()
+}

lib/promscrape/scrapework.go

@@ -429,14 +429,12 @@ func (sw *scrapeWork) needStreamParseMode(responseSize int) bool {
 // getTargetResponse() fetches response from sw target in the same way as when scraping the target.
 func (sw *scrapeWork) getTargetResponse() ([]byte, error) {
 	cb := chunkedbuffer.Get()
+	defer chunkedbuffer.Put(cb)
 
 	isGzipped, err := sw.ReadData(cb)
 	if err != nil {
 		return nil, err
 	}
-	// in case of error buffer cannot be returned back to the pool
-	// See https://pkg.go.dev/net/http#RoundTripper
-	defer chunkedbuffer.Put(cb)
 
 	var bb bytesutil.ByteBuffer
 	err = sw.readFromBuffer(&bb, cb, isGzipped)
@@ -468,8 +466,8 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
 	body := leveledbytebufferpool.Get(sw.prevBodyLen)
 	if err == nil {
 		err = sw.readFromBuffer(body, cb, isGzipped)
-		chunkedbuffer.Put(cb)
 	}
+	chunkedbuffer.Put(cb)
 
 	bodyLen := len(body.B)
 	sw.prevBodyLen = bodyLen