lib/protoparser: add flag to allow OpenTelemetry underscore labels to pass through without being prefixed (#10475)

Add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. The labels starting with `__` are not modified.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663

Signed-off-by: andriibeee <154226341+andriibeee@users.noreply.github.com>
Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
+
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. Thanks to @alexei38 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
 

docs/victoriametrics/integrations/opentelemetry.md

@@ -23,6 +23,8 @@ The following label sanitization options can be enabled:
   For example, `process.cpu.time{service.name="foo"}` is converted to `process_cpu_time_seconds_total{service_name="foo"}`.
 * `-opentelemetry.convertMetricNamesToPrometheus` - converts **only metric names** according to [OTLP Metric points to Prometheus specification](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.33.0/specification/compatibility/prometheus_and_openmetrics.md#otlp-metric-points-to-prometheus) for metrics ingested via OTLP.
   For example, `process.cpu.time{service.name="foo"}` is converted to `process_cpu_time_seconds_total{service.name="foo"}`. See more about this use case [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9830).
+* `-opentelemetry.labelNameUnderscoreSanitization` - controls whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. Reserved labels starting with `__` are not modified.
+  For example, `_mylabel` is converted to `key_mylabel`.
 
 > These flags can be applied on vmagent, vminsert or VictoriaMetrics single-node.
 

lib/protoparser/opentelemetry/stream/sanitize.go

@@ -16,6 +16,9 @@ var (
 		"via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/")
 	convertMetricNamesToPrometheus = flag.Bool("opentelemetry.convertMetricNamesToPrometheus", false, "Whether to convert only metric names into Prometheus-compatible format for the metrics ingested "+
 		"via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/")
+	labelNameUnderscoreSanitization = flag.Bool("opentelemetry.labelNameUnderscoreSanitization", true, "Whether to enable prepending of 'key' to labels starting with '_' "+
+		"when -opentelemetry.usePrometheusNaming is enabled. Reserved labels starting with '__' are not modified. "+
+		"See https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/")
 )
 
 // unitMap is obtained from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/b8655058501bed61a06bb660869051491f46840b/pkg/translator/prometheus/normalize_name.go#L19
@@ -80,8 +83,6 @@ func (sctx *sanitizerContext) reset() {
 	sctx.labelBuf = sctx.labelBuf[:0]
 }
 
-// See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/b8655058501bed61a06bb660869051491f46840b/pkg/translator/prometheus/normalize_label.go#L26
-//
 // The returned string is valid until the next call to sanitizeLabelName.
 func (sctx *sanitizerContext) sanitizeLabelName(labelName string) string {
 	if !*usePrometheusNaming {
@@ -90,6 +91,8 @@ func (sctx *sanitizerContext) sanitizeLabelName(labelName string) string {
 	return sctx.sanitizePrometheusLabelName(labelName)
 }
 
+// sanitizePrometheusLabelName performs conversion and normalization of OpenTelemetry attributes to Prometheus labels.
+// It follows the Prometheus guidelines: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/prometheus#labels
 func (sctx *sanitizerContext) sanitizePrometheusLabelName(labelName string) string {
 	if len(labelName) == 0 {
 		return ""
@@ -97,7 +100,7 @@ func (sctx *sanitizerContext) sanitizePrometheusLabelName(labelName string) stri
 	labelName = promrelabel.SanitizeLabelName(labelName)
 	if labelName[0] >= '0' && labelName[0] <= '9' {
 		return sctx.concatLabel("key_", labelName)
-	} else if strings.HasPrefix(labelName, "_") && !strings.HasPrefix(labelName, "__") {
+	} else if *labelNameUnderscoreSanitization && strings.HasPrefix(labelName, "_") && !strings.HasPrefix(labelName, "__") {
 		return sctx.concatLabel("key", labelName)
 	}
 	return labelName

lib/protoparser/opentelemetry/stream/sanitize_test.go

@@ -24,6 +24,19 @@ func TestSanitizePrometheusLabelName(t *testing.T) {
 	f("1foo", "key_1foo")
 	f("_foo", "key_foo")
 	f("__bar", "__bar")
+
+	prev := *labelNameUnderscoreSanitization
+	*labelNameUnderscoreSanitization = false
+	defer func() {
+		*labelNameUnderscoreSanitization = prev
+	}()
+
+	f("", "")
+	f("foo", "foo")
+	f("foo_bar", "foo_bar")
+	f("1foo", "key_1foo")
+	f("_foo", "_foo")
+	f("__bar", "__bar")
 }
 
 func TestSanitizePrometheusMetricName(t *testing.T) {