app/vmselect: properly return memory to the pool (#11421)

Previously in case of small request processed by vmselect, it may not
reuse sortBlock structure in case of storage block parsing error or
request complexity error.

 This commit addresses these issues and properly puts sort blocks
back to the pool.

Related PR https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11421/
Was discovered as a part of https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11414

app/vmselect/netstorage/netstorage.go

@@ -460,7 +460,8 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 			initUnpackWork(upw, br)
 			upw.unpack(tmpBlock)
 			if upw.err != nil {
-				return dst, upw.err
+				err = upw.err
+				break
 			}
 			samples += len(upw.sb.Timestamps)
 			if *maxSamplesPerSeries > 0 && samples > *maxSamplesPerSeries {
@@ -474,7 +475,11 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 		}
 		putTmpStorageBlock(tmpBlock)
 		putUnpackWork(upw)
-
+		if err != nil {
+			for _, sb := range dst {
+				putSortBlock(sb)
+			}
+		}
 		return dst, err
 	}
 
@@ -540,6 +545,11 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 		}
 		putUnpackWork(upw)
 	}
+	if firstErr != nil {
+		for _, sb := range dst {
+			putSortBlock(sb)
+		}
+	}
 
 	return dst, firstErr
 }

app/vmselect/prometheus/prometheus.go

@@ -703,7 +703,7 @@ var labelsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
 func SeriesCountHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	defer seriesCountDuration.UpdateDuration(startTime)
 
-	deadline := searchutil.GetDeadlineForStatusRequest(r, startTime)
+	deadline := searchutil.GetDeadlineForLabelsAPI(r, startTime)
 	n, err := netstorage.SeriesCount(nil, deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain series count: %w", err)

app/vmselect/promql/eval.go

@@ -1072,6 +1072,7 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	var samplesScannedTotal atomic.Uint64
 	keepMetricNames := getKeepMetricNames(expr)
 	tsw := getTimeseriesByWorkerID()
+	defer putTimeseriesByWorkerID(tsw)
 	seriesByWorkerID := tsw.byWorkerID
 	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64, workerID uint) ([]float64, []int64) {
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
@@ -1094,7 +1095,6 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	for i := range seriesByWorkerID {
 		tss = append(tss, seriesByWorkerID[i].tss...)
 	}
-	putTimeseriesByWorkerID(tsw)
 
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal.Load()))
 	qt.Printf("rollup %s() over %d series returned by subquery: series=%d, samplesScanned=%d", funcName, len(tssSQ), len(tss), samplesScannedTotal.Load())
@@ -1973,6 +1973,7 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 
 	var samplesScannedTotal atomic.Uint64
 	tsw := getTimeseriesByWorkerID()
+	defer putTimeseriesByWorkerID(tsw)
 	seriesByWorkerID := tsw.byWorkerID
 	seriesLen := rss.Len()
 	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
@@ -1999,7 +2000,6 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 	for i := range seriesByWorkerID {
 		tss = append(tss, seriesByWorkerID[i].tss...)
 	}
-	putTimeseriesByWorkerID(tsw)
 
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal.Load()))
 	qt.Printf("samplesScanned=%d", samplesScannedTotal.Load())

app/vmselect/searchutil/searchutil.go

@@ -56,7 +56,7 @@ func GetDeadlineForExport(r *http.Request, startTime time.Time) Deadline {
 	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxExportDuration")
 }
 
-// GetDeadlineForLabelsAPI returns deadline for the given request to /api/v1/labels, /api/v1/label/.../values or /api/v1/series
+// GetDeadlineForLabelsAPI returns deadline for the given request to /api/v1/labels, /api/v1/label/.../values, /api/v1/series or /api/v1/series/count
 func GetDeadlineForLabelsAPI(r *http.Request, startTime time.Time) Deadline {
 	dMax := maxLabelsAPIDuration.Milliseconds()
 	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxLabelsAPIDuration")

docs/victoriametrics/Articles.md

@@ -117,6 +117,10 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [Claude Code: creating Kubernetes debugging AI Agent for VictoriaMetrics](https://rtfm.co.ua/en/claude-code-creating-kubernetes-debugging-ai-agent-for-victoriametrics/)
 * [OpenTelemetry: OTel Collectors in Kubernetes and VictoriaMetrics Stack integration](https://itnext.io/opentelemetry-otel-collectors-in-kubernetes-and-victoriametrics-stack-integration-d907ed0a15a0)
 * [VictoriaMetrics vs Prometheus: my default, and when I still pick Prometheus](https://jorijn.com/en/blog/victoriametrics-vs-prometheus/)
+* [LiteLLM: Monitoring with VictoriaMetrics – Alerts and Grafana](https://rtfm.co.ua/en/litellm-monitoring-with-victoriametrics-alerts-and-grafana/)
+* [LiteLLM: AI Gateway on Kubernetes and Metrics in VictoriaMetrics](https://rtfm.co.ua/en/litellm-ai-gateway-on-kubernetes-and-metrics-in-victoriametrics/)
+* [LiteLLM: Metrics, Traces, and VictoriaMetrics Stack Integration](https://rtfm.co.ua/en/litellm-metrics-traces-and-victoriametrics-stack-integration/)
+* [llama.cpp: Metrics and Monitoring with VictoriaMetrics](https://rtfm.co.ua/en/llama-cpp-metrics-and-monitoring-with-victoriametrics/)
 
 ## Third-party articles and slides about VictoriaLogs
 

docs/victoriametrics/changelog/CHANGELOG.md

@@ -33,7 +33,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix infinite loop in the OpenTelemetry Firehose ingestion endpoint (`/opentelemetry/api/v1/push`) when receiving a malformed record with an incomplete varint in the `data` field. Previously this caused the goroutine to spin forever, permanently consuming CPU until the process was restarted.
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): reuse connections to `-remoteWrite.url` when writing the results of recording rules and alerts. Previously every series was sent over a new connection, which left a lot of sockets in `TIME_WAIT` state and could exhaust the ephemeral port range. The number of idle connections can be tuned via the new `-remoteWrite.maxIdleConnections` command-line flag. Thanks @evkuzin for contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent process crash in `sort_by_label_numeric()` and `sort_by_label_numeric_desc()` when a label value contains a number with 309 or more digits. See [#11423](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11423).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply default query timeout to `/api/v1/series/count` requests. It used `-search.maxStatusRequestDuration` flag value instead of `-search.maxLabelsAPIDuration`. See [#11422](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11422).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly re-use memory if query aggregation returns error. See [#11426](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11426).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fail the query request directly when there is not enough disk space to store temporary search results. Previously, such queries could lead to vmselect crash. See [#4688](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4688).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): consistently re-use memory during storage blocks unpacking on parsing storage block error. See [#11421](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11421).
 
 ## [v1.150.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.150.0)
 

lib/promscrape/scrapework.go

@@ -429,12 +429,14 @@ func (sw *scrapeWork) needStreamParseMode(responseSize int) bool {
 // getTargetResponse() fetches response from sw target in the same way as when scraping the target.
 func (sw *scrapeWork) getTargetResponse() ([]byte, error) {
 	cb := chunkedbuffer.Get()
-	defer chunkedbuffer.Put(cb)
 
 	isGzipped, err := sw.ReadData(cb)
 	if err != nil {
 		return nil, err
 	}
+	// in case of error buffer cannot be returned back to the pool
+	// See https://pkg.go.dev/net/http#RoundTripper
+	defer chunkedbuffer.Put(cb)
 
 	var bb bytesutil.ByteBuffer
 	err = sw.readFromBuffer(&bb, cb, isGzipped)
@@ -466,8 +468,8 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
 	body := leveledbytebufferpool.Get(sw.prevBodyLen)
 	if err == nil {
 		err = sw.readFromBuffer(body, cb, isGzipped)
+		chunkedbuffer.Put(cb)
 	}
-	chunkedbuffer.Put(cb)
 
 	bodyLen := len(body.B)
 	sw.prevBodyLen = bodyLen