mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-05-17 00:26:36 +03:00
lib/promscape: reduce memory allocations for newCompressedLabels
Previously, creation of compressedLabels could require extra memory due
to re-allocation of tmpBuf and clone of 3 extra fields. It could result
into extra CPU usage for garbage-collection.
This commit adds sync.Pool for labels escape with JSON marshal and
allocates dedicated buffer for job, address and ID strings.
Optimisations was made based on the following profiles from reported
issue:
1) CPU:
```
Showing top 10 nodes out of 172
flat flat% sum% cum cum%
12.17s 17.19% 17.19% 12.25s 17.30% runtime.cgocall
5.87s 8.29% 25.48% 5.87s 8.29% runtime.memmove
3.45s 4.87% 30.35% 6.66s 9.41% runtime.tryDeferToSpanScan
```
2) memory go tool pprof -alloc_objects heap_profile.txt
```
Showing top 10 nodes out of 94
flat flat% sum% cum cum%
3673568660 26.09% 26.09% 4147984949 29.46% github.com/valyala/quicktemplate.AppendJSONString
1657933055 11.77% 37.86% 1657933055 11.77% internal/stringslite.Clone (inline)
1555166274 11.04% 48.91% 1555166274 11.04% github.com/valyala/gozstd.compress
1254756359 8.91% 57.82% 9433313305 66.99% github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape.newCompressedLabels
1067036870 7.58% 65.39% 1067036870 7.58% github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape.appendExtraLabels
```
results of benchstat:
```
benchstat before after
goos: darwin
goarch: arm64
pkg: github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape
cpu: Apple M1 Pro
│ 134/before │ after │
│ sec/op │ sec/op vs base │
NewCompressedLabels-10 981.3n ± 2% 908.6n ± 2% -7.40% (p=0.000 n=10)
│ 134/before │ after │
│ B/op │ B/op vs base │
NewCompressedLabels-10 891.5 ± 0% 772.0 ± 0% -13.40% (p=0.000 n=10)
│ 134/before │ after │
│ allocs/op │ allocs/op vs base │
NewCompressedLabels-10 10.000 ± 0% 3.000 ± 0% -70.00% (p=0.000 n=10)
```
Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919
This commit is contained in:
f41gh7
@@ -28,6 +28,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
* FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
|
||||
|
||||
|
||||
@@ -753,31 +753,64 @@ func newCompressedLabels(src *promutil.Labels) *compressedLabels {
|
||||
bb.Grow(sizeNeeded)
|
||||
// manually craft json in order to reduce memory allocations
|
||||
fmt.Fprintf(bb, `{`) //nolint:errcheck
|
||||
var tmpBuf []byte
|
||||
|
||||
escapeBB := compressedLabelsEscapePool.Get()
|
||||
escapeBuf := escapeBB.B
|
||||
|
||||
for i, label := range srcLabels {
|
||||
tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Name, true)
|
||||
bb.Write(tmpBuf) //nolint:errcheck
|
||||
escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Name, true)
|
||||
bb.Write(escapeBuf) //nolint:errcheck
|
||||
bb.Write([]byte(`:`)) //nolint:errcheck
|
||||
tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Value, true)
|
||||
bb.Write(tmpBuf) //nolint:errcheck
|
||||
escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Value, true)
|
||||
bb.Write(escapeBuf) //nolint:errcheck
|
||||
if i+1 < len(srcLabels) {
|
||||
bb.Write([]byte(`,`)) //nolint:errcheck
|
||||
}
|
||||
}
|
||||
escapeBB.B = escapeBuf
|
||||
compressedLabelsEscapePool.Put(escapeBB)
|
||||
|
||||
fmt.Fprint(bb, `}`) //nolint:errcheck
|
||||
|
||||
dst := zstd.CompressLevel(nil, bb.B, 1)
|
||||
|
||||
compressedLabelsBufPool.Put(bb)
|
||||
cls := &compressedLabels{
|
||||
hashKey: h,
|
||||
addressLabel: strings.Clone(src.Get("__address__")),
|
||||
jobLabel: strings.Clone(src.Get("job")),
|
||||
data: dst,
|
||||
hashKey: h,
|
||||
data: dst,
|
||||
}
|
||||
cls.targetID = fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls)))
|
||||
addressLabelValue := src.Get("__address__")
|
||||
jobLabelValue := src.Get("job")
|
||||
addressLen := len(addressLabelValue)
|
||||
jobLen := len(jobLabelValue)
|
||||
// pre-allocate buffer to recuce GC pressure for tracking individual strings
|
||||
packedBuf := make([]byte, 0, jobLen+addressLen+16)
|
||||
|
||||
packedBuf = append(packedBuf, addressLabelValue...)
|
||||
cls.addressLabel = bytesutil.ToUnsafeString(packedBuf)
|
||||
|
||||
packedBuf = append(packedBuf, jobLabelValue...)
|
||||
cls.jobLabel = bytesutil.ToUnsafeString(packedBuf[addressLen:])
|
||||
|
||||
packedBuf = appendHex16(packedBuf, uint64(uintptr(unsafe.Pointer(cls))))
|
||||
cls.targetID = bytesutil.ToUnsafeString(packedBuf[addressLen+jobLen:])
|
||||
|
||||
return cls
|
||||
}
|
||||
|
||||
// appendHex16 is an equvialent for fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls)))
|
||||
// but with zero allocations
|
||||
func appendHex16(dst []byte, v uint64) []byte {
|
||||
const hexChars = "0123456789abcdef"
|
||||
for i := 15; i >= 0; i-- {
|
||||
dst = append(dst, hexChars[v&0xf])
|
||||
v >>= 4
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
var compressedLabelsEscapePool = &bytesutil.ByteBufferPool{}
|
||||
|
||||
func (cls *compressedLabels) getTargetID() string {
|
||||
if cls == nil {
|
||||
return ""
|
||||
|
||||
41
lib/promscrape/targetstatus_timing_test.go
Normal file
41
lib/promscrape/targetstatus_timing_test.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package promscrape
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
)
|
||||
|
||||
func BenchmarkNewCompressedLabels(b *testing.B) {
|
||||
const numTargets = 1000
|
||||
labelSet := func(idx int) *promutil.Labels {
|
||||
return promutil.NewLabelsFromMap(map[string]string{
|
||||
"__address__": fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff),
|
||||
"__meta_kubernetes_namespace": "default",
|
||||
"__meta_kubernetes_pod_name": fmt.Sprintf("test-%d", idx),
|
||||
"__meta_kubernetes_pod_uid": fmt.Sprintf("00000000-0000-0000-0000-%012d", idx),
|
||||
"__meta_kubernetes_pod_ip": fmt.Sprintf("10.0.%d.%d", idx>>8, idx&0xff),
|
||||
"__meta_kubernetes_pod_node_name": fmt.Sprintf("node-%d", idx%50),
|
||||
"__meta_kubernetes_pod_label_app": "monitoring",
|
||||
"__meta_kubernetes_pod_label_release": "prod",
|
||||
"__meta_kubernetes_pod_annotation_prometheus_io_scrape": "true",
|
||||
"__meta_kubernetes_pod_annotation_prometheus_io_port": "9100",
|
||||
"job": "k8spod",
|
||||
"instance": fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff),
|
||||
})
|
||||
}
|
||||
|
||||
labelss := make([]*promutil.Labels, numTargets)
|
||||
for i := range labelss {
|
||||
labelss[i] = labelSet(i)
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
var i int
|
||||
for pb.Next() {
|
||||
_ = newCompressedLabels(labelss[i%numTargets])
|
||||
i++
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user