From db69fc686ac3ccdf4d3336e4ef7634c34560ba59 Mon Sep 17 00:00:00 2001 From: f41gh7 Date: Thu, 14 May 2026 15:24:25 +0200 Subject: [PATCH] lib/promscape: reduce memory allocations for newCompressedLabels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, creation of compressedLabels could require extra memory due to re-allocation of tmpBuf and clone of 3 extra fields. It could result into extra CPU usage for garbage-collection. This commit adds sync.Pool for labels escape with JSON marshal and allocates dedicated buffer for job, address and ID strings. Optimisations was made based on the following profiles from reported issue: 1) CPU: ``` Showing top 10 nodes out of 172 flat flat% sum% cum cum% 12.17s 17.19% 17.19% 12.25s 17.30% runtime.cgocall 5.87s 8.29% 25.48% 5.87s 8.29% runtime.memmove 3.45s 4.87% 30.35% 6.66s 9.41% runtime.tryDeferToSpanScan ``` 2) memory go tool pprof -alloc_objects heap_profile.txt ``` Showing top 10 nodes out of 94 flat flat% sum% cum cum% 3673568660 26.09% 26.09% 4147984949 29.46% github.com/valyala/quicktemplate.AppendJSONString 1657933055 11.77% 37.86% 1657933055 11.77% internal/stringslite.Clone (inline) 1555166274 11.04% 48.91% 1555166274 11.04% github.com/valyala/gozstd.compress 1254756359 8.91% 57.82% 9433313305 66.99% github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape.newCompressedLabels 1067036870 7.58% 65.39% 1067036870 7.58% github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape.appendExtraLabels ``` results of benchstat: ``` benchstat before after goos: darwin goarch: arm64 pkg: github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape cpu: Apple M1 Pro │ 134/before │ after │ │ sec/op │ sec/op vs base │ NewCompressedLabels-10 981.3n ± 2% 908.6n ± 2% -7.40% (p=0.000 n=10) │ 134/before │ after │ │ B/op │ B/op vs base │ NewCompressedLabels-10 891.5 ± 0% 772.0 ± 0% -13.40% (p=0.000 n=10) │ 134/before │ after │ │ allocs/op │ allocs/op vs base │ NewCompressedLabels-10 10.000 ± 0% 3.000 ± 0% -70.00% (p=0.000 n=10) ``` Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919 --- docs/victoriametrics/changelog/CHANGELOG.md | 1 + lib/promscrape/targetstatus.go | 53 +++++++++++++++++---- lib/promscrape/targetstatus_timing_test.go | 41 ++++++++++++++++ 3 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 lib/promscrape/targetstatus_timing_test.go diff --git a/docs/victoriametrics/changelog/CHANGELOG.md b/docs/victoriametrics/changelog/CHANGELOG.md index ec0a2a4455..46136cbec0 100644 --- a/docs/victoriametrics/changelog/CHANGELOG.md +++ b/docs/victoriametrics/changelog/CHANGELOG.md @@ -28,6 +28,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel * FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935). * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution. +* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919). * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution. * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790). diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go index 05ac009134..26ba31193f 100644 --- a/lib/promscrape/targetstatus.go +++ b/lib/promscrape/targetstatus.go @@ -753,31 +753,64 @@ func newCompressedLabels(src *promutil.Labels) *compressedLabels { bb.Grow(sizeNeeded) // manually craft json in order to reduce memory allocations fmt.Fprintf(bb, `{`) //nolint:errcheck - var tmpBuf []byte + + escapeBB := compressedLabelsEscapePool.Get() + escapeBuf := escapeBB.B + for i, label := range srcLabels { - tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Name, true) - bb.Write(tmpBuf) //nolint:errcheck + escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Name, true) + bb.Write(escapeBuf) //nolint:errcheck bb.Write([]byte(`:`)) //nolint:errcheck - tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Value, true) - bb.Write(tmpBuf) //nolint:errcheck + escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Value, true) + bb.Write(escapeBuf) //nolint:errcheck if i+1 < len(srcLabels) { bb.Write([]byte(`,`)) //nolint:errcheck } } + escapeBB.B = escapeBuf + compressedLabelsEscapePool.Put(escapeBB) + fmt.Fprint(bb, `}`) //nolint:errcheck + dst := zstd.CompressLevel(nil, bb.B, 1) compressedLabelsBufPool.Put(bb) cls := &compressedLabels{ - hashKey: h, - addressLabel: strings.Clone(src.Get("__address__")), - jobLabel: strings.Clone(src.Get("job")), - data: dst, + hashKey: h, + data: dst, } - cls.targetID = fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls))) + addressLabelValue := src.Get("__address__") + jobLabelValue := src.Get("job") + addressLen := len(addressLabelValue) + jobLen := len(jobLabelValue) + // pre-allocate buffer to recuce GC pressure for tracking individual strings + packedBuf := make([]byte, 0, jobLen+addressLen+16) + + packedBuf = append(packedBuf, addressLabelValue...) + cls.addressLabel = bytesutil.ToUnsafeString(packedBuf) + + packedBuf = append(packedBuf, jobLabelValue...) + cls.jobLabel = bytesutil.ToUnsafeString(packedBuf[addressLen:]) + + packedBuf = appendHex16(packedBuf, uint64(uintptr(unsafe.Pointer(cls)))) + cls.targetID = bytesutil.ToUnsafeString(packedBuf[addressLen+jobLen:]) + return cls } +// appendHex16 is an equvialent for fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls))) +// but with zero allocations +func appendHex16(dst []byte, v uint64) []byte { + const hexChars = "0123456789abcdef" + for i := 15; i >= 0; i-- { + dst = append(dst, hexChars[v&0xf]) + v >>= 4 + } + return dst +} + +var compressedLabelsEscapePool = &bytesutil.ByteBufferPool{} + func (cls *compressedLabels) getTargetID() string { if cls == nil { return "" diff --git a/lib/promscrape/targetstatus_timing_test.go b/lib/promscrape/targetstatus_timing_test.go new file mode 100644 index 0000000000..1be34fe1fb --- /dev/null +++ b/lib/promscrape/targetstatus_timing_test.go @@ -0,0 +1,41 @@ +package promscrape + +import ( + "fmt" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil" +) + +func BenchmarkNewCompressedLabels(b *testing.B) { + const numTargets = 1000 + labelSet := func(idx int) *promutil.Labels { + return promutil.NewLabelsFromMap(map[string]string{ + "__address__": fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff), + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_name": fmt.Sprintf("test-%d", idx), + "__meta_kubernetes_pod_uid": fmt.Sprintf("00000000-0000-0000-0000-%012d", idx), + "__meta_kubernetes_pod_ip": fmt.Sprintf("10.0.%d.%d", idx>>8, idx&0xff), + "__meta_kubernetes_pod_node_name": fmt.Sprintf("node-%d", idx%50), + "__meta_kubernetes_pod_label_app": "monitoring", + "__meta_kubernetes_pod_label_release": "prod", + "__meta_kubernetes_pod_annotation_prometheus_io_scrape": "true", + "__meta_kubernetes_pod_annotation_prometheus_io_port": "9100", + "job": "k8spod", + "instance": fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff), + }) + } + + labelss := make([]*promutil.Labels, numTargets) + for i := range labelss { + labelss[i] = labelSet(i) + } + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var i int + for pb.Next() { + _ = newCompressedLabels(labelss[i%numTargets]) + i++ + } + }) +}