Update lib/promscrape/targetstatus.go

Co-authored-by: Hui Wang <haley@victoriametrics.com>
Signed-off-by: Nikolay <nik@victoriametrics.com>

docs/victoriametrics/changelog/CHANGELOG.md

@@ -28,6 +28,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 * FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
 

lib/promscrape/targetstatus.go

@@ -752,32 +752,67 @@ func newCompressedLabels(src *promutil.Labels) *compressedLabels {
 	bb := compressedLabelsBufPool.Get()
 	bb.Grow(sizeNeeded)
 	// manually craft json in order to reduce memory allocations
-	fmt.Fprintf(bb, `{`) //nolint:errcheck
-	var tmpBuf []byte
+	bb.B = append(bb.B, '{')
+
+	escapeBB := compressedLabelsEscapePool.Get()
+	escapeBuf := escapeBB.B
+
 	for i, label := range srcLabels {
-		tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Name, true)
-		bb.Write(tmpBuf)      //nolint:errcheck
+		escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Name, true)
+		bb.Write(escapeBuf)   //nolint:errcheck
 		bb.Write([]byte(`:`)) //nolint:errcheck
-		tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Value, true)
-		bb.Write(tmpBuf) //nolint:errcheck
+		escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Value, true)
+		bb.Write(escapeBuf) //nolint:errcheck
 		if i+1 < len(srcLabels) {
 			bb.Write([]byte(`,`)) //nolint:errcheck
 		}
 	}
+	escapeBB.B = escapeBuf
+	compressedLabelsEscapePool.Put(escapeBB)
+
 	fmt.Fprint(bb, `}`) //nolint:errcheck
+
 	dst := zstd.CompressLevel(nil, bb.B, 1)
 
 	compressedLabelsBufPool.Put(bb)
 	cls := &compressedLabels{
-		hashKey:      h,
-		addressLabel: strings.Clone(src.Get("__address__")),
-		jobLabel:     strings.Clone(src.Get("job")),
-		data:         dst,
+		hashKey: h,
+		data:    dst,
 	}
-	cls.targetID = fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls)))
+	addressLabelValue := src.Get("__address__")
+	jobLabelValue := src.Get("job")
+	addressLen := len(addressLabelValue)
+	jobLen := len(jobLabelValue)
+	// pre-allocate buffer to recuce GC pressure for tracking individual strings
+	packedBuf := make([]byte, 0, jobLen+addressLen+16)
+
+	packedBuf = append(packedBuf, addressLabelValue...)
+	cls.addressLabel = bytesutil.ToUnsafeString(packedBuf[:addressLen])
+
+	packedBuf = append(packedBuf, jobLabelValue...)
+	cls.jobLabel = bytesutil.ToUnsafeString(packedBuf[addressLen:])
+
+	packedBuf = appendHex16(packedBuf, uint64(uintptr(unsafe.Pointer(cls))))
+	cls.targetID = bytesutil.ToUnsafeString(packedBuf[addressLen+jobLen:])
+
 	return cls
 }
 
+// appendHex16 is an equvialent for fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls)))
+// but with zero allocations
+func appendHex16(dst []byte, v uint64) []byte {
+	const hexChars = "0123456789abcdef"
+	var buf [16]byte
+	for i := 15; i >= 0; i-- {
+		buf[i] = hexChars[v&0xf]
+		v >>= 4
+	}
+	dst = append(dst, buf[:]...)
+	return dst
+}
+
+var compressedLabelsEscapePool = &bytesutil.ByteBufferPool{}
+
 func (cls *compressedLabels) getTargetID() string {
 	if cls == nil {
 		return ""

lib/promscrape/targetstatus_timing_test.go

@@ -0,0 +1,42 @@
+package promscrape
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
+)
+
+func BenchmarkNewCompressedLabels(b *testing.B) {
+	const numTargets = 1000
+	labelSet := func(idx int) *promutil.Labels {
+		return promutil.NewLabelsFromMap(map[string]string{
+			"__address__":                                           fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff),
+			"__meta_kubernetes_namespace":                           "default",
+			"__meta_kubernetes_pod_name":                            fmt.Sprintf("test-%d", idx),
+			"__meta_kubernetes_pod_uid":                             fmt.Sprintf("00000000-0000-0000-0000-%012d", idx),
+			"__meta_kubernetes_pod_ip":                              fmt.Sprintf("10.0.%d.%d", idx>>8, idx&0xff),
+			"__meta_kubernetes_pod_node_name":                       fmt.Sprintf("node-%d", idx%50),
+			"__meta_kubernetes_pod_label_app":                       "monitoring",
+			"__meta_kubernetes_pod_label_release":                   "prod",
+			"__meta_kubernetes_pod_annotation_prometheus_io_scrape": "true",
+			"__meta_kubernetes_pod_annotation_prometheus_io_port":   "9100",
+			"job":      "k8spod",
+			"instance": fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff),
+		})
+	}
+
+	labelss := make([]*promutil.Labels, numTargets)
+	for i := range labelss {
+		labelss[i] = labelSet(i)
+	}
+	b.ResetTimer()
+	b.ReportAllocs()
+	b.RunParallel(func(pb *testing.PB) {
+		var i int
+		for pb.Next() {
+			_ = newCompressedLabels(labelss[i%numTargets])
+			i++
+		}
+	})
+}