lib/protoparser/opentelemetry: support disable scope and resource attributes label promotions

This commit is contained in:
Haley Wang
2026-05-16 00:01:31 +08:00
parent f2ba4bb3b6
commit 50bdcccde5
11 changed files with 331 additions and 11 deletions

View File

@@ -118,6 +118,7 @@ func main() {
remotewrite.InitSecretFlags()
buildinfo.Init()
logger.Init()
opentelemetry.Init()
timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
if promscrape.IsDryRun() {

View File

@@ -11,6 +11,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
@@ -25,6 +26,10 @@ var (
rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="opentelemetry"}`)
)
func Init() {
pb.ValidateFlags()
}
// InsertHandlerForReader processes metrics from given reader.
func InsertHandlerForReader(at *auth.Token, r io.Reader, encoding string) error {
return stream.ParseStream(r, encoding, nil, func(tss []prompb.TimeSeries, mms []prompb.MetricMetadata) error {

View File

@@ -89,6 +89,7 @@ var staticServer = http.FileServer(http.FS(staticFiles))
func Init() {
relabel.Init()
common.InitStreamAggr()
opentelemetry.Init()
protoparserutil.StartUnmarshalWorkers()
if len(*graphiteListenAddr) > 0 {
graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)

View File

@@ -9,6 +9,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/pb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
"github.com/VictoriaMetrics/metrics"
@@ -20,6 +21,10 @@ var (
metadataInserted = metrics.NewCounter(`vm_metadata_rows_inserted_total{type="opentelemetry"}`)
)
func Init() {
pb.ValidateFlags()
}
// InsertHandler processes opentelemetry metrics.
func InsertHandler(req *http.Request) error {
extraLabels, err := protoparserutil.GetExtraLabels(req)

View File

@@ -30,6 +30,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-opentelemetry.promoteAllResourceAttributes` and `-opentelemetry.promoteScopeMetadata` command-line flags to allow managing label promotion for resource attributes and OTel scope metadata. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#10931](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10931).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. Thanks to @alexei38 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).

View File

@@ -28,9 +28,17 @@ The following label sanitization options can be enabled:
> These flags can be applied on vmagent, vminsert or VictoriaMetrics single-node.
## Instrumentation Scope
By default, VictoriaMetrics promotes [OTel scope metadata](https://opentelemetry.io/docs/specs/otel/common/instrumentation-scope/) to metric labels. This behavior can be disabled via `-opentelemetry.promoteScopeMetadata`.
## Resource Attributes
By default, VictoriaMetrics promotes all [OpenTelemetry resource](https://opentelemetry.io/docs/specs/otel/resource/data-model/) attributes to labels and attaches them to all ingested OTLP metrics.
The following attribute promotion options can be configured:
- `opentelemetry.promoteAllResourceAttributes` - promotes all resource attributes to labels, except for the ones configured with `-opentelemetry.ignoreResourceAttributes`.
- `opentelemetry.promoteResourceAttributes` - promotes specific list of resource attributes to labels. It cannot be configured simultaneously with `opentelemetry.promoteAllResourceAttributes`.
- `opentelemetry.ignoreResourceAttributes` - controls which resource attributes to ignore, can only be set when `opentelemetry.promoteAllResourceAttributes` is true.
## Exponential histograms

View File

@@ -217,9 +217,17 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
-opentelemetry.convertMetricNamesToPrometheus
Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
-opentelemetry.ignoreResourceAttributes array
Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
-opentelemetry.maxRequestSize size
The maximum size in bytes of a single OpenTelemetry request
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
-opentelemetry.promoteAllResourceAttributes array
Promote specific list of resource attributes to labels.
-opentelemetry.promoteResourceAttributes
Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
-opentelemetry.promoteScopeMetadata
Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
-opentelemetry.usePrometheusNaming
Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
-opentsdbHTTPListenAddr string

View File

@@ -184,9 +184,17 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
-opentelemetry.convertMetricNamesToPrometheus
Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
-opentelemetry.ignoreResourceAttributes array
Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
-opentelemetry.maxRequestSize size
The maximum size in bytes of a single OpenTelemetry request
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
-opentelemetry.promoteAllResourceAttributes array
Promote specific list of resource attributes to labels.
-opentelemetry.promoteResourceAttributes
Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
-opentelemetry.promoteScopeMetadata
Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
-opentelemetry.usePrometheusNaming
Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
-opentsdbHTTPListenAddr string

View File

@@ -184,9 +184,17 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
-opentelemetry.convertMetricNamesToPrometheus
Whether to convert only metric names into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
-opentelemetry.ignoreResourceAttributes array
Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.
-opentelemetry.maxRequestSize size
The maximum size in bytes of a single OpenTelemetry request
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
-opentelemetry.promoteAllResourceAttributes array
Promote specific list of resource attributes to labels.
-opentelemetry.promoteResourceAttributes
Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.
-opentelemetry.promoteScopeMetadata
Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.
-opentelemetry.usePrometheusNaming
Whether to convert metric names and labels into Prometheus-compatible format for the metrics ingested via OpenTelemetry protocol; see https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/
-opentsdbHTTPListenAddr string

View File

@@ -1,6 +1,7 @@
package pb
import (
"flag"
"fmt"
"math"
"strconv"
@@ -9,11 +10,28 @@ import (
"github.com/VictoriaMetrics/easyproto"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
)
var (
promoteScopeMetadata = flag.Bool("opentelemetry.promoteScopeMetadata", true, "Whether to promote OTel scope metadata (i.e. name, version, schema URL, and attributes) to metric labels.")
promoteAllResourceAttributes = flag.Bool("opentelemetry.promoteAllResourceAttributes", true, "Whether to promote all resource attributes to labels, except for the ones configured with 'opentelemetry.ignoreResourceAttributes'.")
promoteResourceAttributes = flagutil.NewArrayString("opentelemetry.promoteResourceAttributes", "Promote specific list of resource attributes to labels.")
ignoreResourceAttributes = flagutil.NewArrayString("opentelemetry.ignoreResourceAttributes", "Control which resource attributes to ignore, can only be set when 'opentelemetry.promoteAllResourceAttributes' is true.")
)
func ValidateFlags() {
if *promoteAllResourceAttributes && len(*promoteResourceAttributes) > 0 {
logger.Fatalf("cannot set both '-opentelemetry.promoteAllResourceAttributes' and '-opentelemetry.promoteResourceAttributes'")
}
if !*promoteAllResourceAttributes && len(*ignoreResourceAttributes) > 0 {
logger.Fatalf("'-opentelemetry.ignoreResourceAttributes' can only be set when '-opentelemetry.promoteAllResourceAttributes' is true.")
}
}
// MetricPusher must push the parsed samples and metric metadata to the underlying storage.
type MetricPusher interface {
// PushSample must store a sample with the given args.
@@ -132,12 +150,20 @@ func (dctx *decoderContext) decodeResourceMetrics(src []byte) error {
dctx.ls.Reset()
dctx.fb.reset()
attributes := *ignoreResourceAttributes
if !*promoteAllResourceAttributes {
attributes = *promoteResourceAttributes
}
attributeKeys := make(map[string]struct{}, len(attributes))
for _, a := range attributes {
attributeKeys[a] = struct{}{}
}
resourceData, ok, err := easyproto.GetMessageData(src, 1)
if err != nil {
return fmt.Errorf("cannot read Resource data: %w", err)
}
if ok {
if err := dctx.decodeResource(resourceData); err != nil {
if err := dctx.decodeResource(resourceData, *promoteAllResourceAttributes, attributeKeys); err != nil {
return fmt.Errorf("cannot decode Resource: %w", err)
}
}
@@ -180,7 +206,7 @@ func (r *Resource) marshalProtobuf(mm *easyproto.MessageMarshaler) {
}
}
func (dctx *decoderContext) decodeResource(src []byte) (err error) {
func (dctx *decoderContext) decodeResource(src []byte, promoteAllResourceAttributes bool, attributeKeys map[string]struct{}) (err error) {
// See https://github.com/open-telemetry/opentelemetry-proto/blob/049d4332834935792fd4dbd392ecd31904f99ba2/opentelemetry/proto/resource/v1/resource.proto#L28
//
// message Resource {
@@ -199,7 +225,34 @@ func (dctx *decoderContext) decodeResource(src []byte) (err error) {
if !ok {
return fmt.Errorf("cannot read Attributes")
}
if err := decodeKeyValue(data, &dctx.ls, &dctx.fb, ""); err != nil {
keySuffix, ok, err := easyproto.GetString(data, 1)
if err != nil {
return fmt.Errorf("cannot find Key in KeyValue: %w", err)
}
if !ok {
// Key is missing, skip it.
// See https://github.com/VictoriaMetrics/VictoriaLogs/issues/869#issuecomment-3631307996
continue
}
if _, ok := attributeKeys[keySuffix]; ok == promoteAllResourceAttributes {
// Skip the attribute if:
// 1. it is in the list of ignore attributes when promoteAllResourceAttributes is true,
// 2. it isn't in the list of promote attributes when promoteAllResourceAttributes is false
continue
}
key := dctx.fb.formatSubFieldName("", keySuffix)
// Decode value
value, ok, err := easyproto.GetMessageData(data, 2)
if err != nil {
return fmt.Errorf("cannot find Value in KeyValue: %w", err)
}
if !ok {
// Value is null, skip it.
continue
}
if err := decodeAnyValue(value, &dctx.ls, &dctx.fb, key); err != nil {
return fmt.Errorf("cannot unmarshal Attributes: %w", err)
}
}
@@ -257,7 +310,6 @@ func decodeKeyValue(src []byte, ls *promutil.Labels, fb *fmtBuffer, keyPrefix st
if err := decodeAnyValue(valueData, ls, fb, key); err != nil {
return fmt.Errorf("cannot decode AnyValue: %w", err)
}
return nil
}
@@ -460,19 +512,23 @@ func (dctx *decoderContext) decodeScopeMetrics(src []byte) error {
// repeated Metric metrics = 2;
// }
scopeData, ok, err := easyproto.GetMessageData(src, 1)
if err != nil {
return fmt.Errorf("cannot read InstrumentationScope: %w", err)
}
if ok {
if err := dctx.decodeInstrumentationScope(scopeData); err != nil {
return fmt.Errorf("cannot decode InstrumentationScope: %w", err)
if *promoteScopeMetadata {
scopeData, ok, err := easyproto.GetMessageData(src, 1)
if err != nil {
return fmt.Errorf("cannot read InstrumentationScope: %w", err)
}
if ok {
if err := dctx.decodeInstrumentationScope(scopeData); err != nil {
return fmt.Errorf("cannot decode InstrumentationScope: %w", err)
}
}
}
dctxSnapshot := dctx.getSnapshot()
var fc easyproto.FieldContext
var err error
for len(src) > 0 {
src, err = fc.NextField(src)
if err != nil {

View File

@@ -0,0 +1,219 @@
package pb
import (
"fmt"
"reflect"
"sort"
"strings"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
)
type testMetricPusher struct {
samples []testSample
metadata []MetricMetadata
}
type testSample struct {
mm MetricMetadata
suffix string
labels []prompb.Label
ts uint64
value float64
}
func (p *testMetricPusher) PushSample(mm *MetricMetadata, suffix string, ls *promutil.Labels, timestampNsecs uint64, value float64, _ uint32) {
labels := make([]prompb.Label, len(ls.Labels))
copy(labels, ls.Labels)
p.samples = append(p.samples, testSample{
mm: *mm,
suffix: suffix,
labels: labels,
ts: timestampNsecs,
value: value,
})
}
func (p *testMetricPusher) PushMetricMetadata(mm *MetricMetadata) {
p.metadata = append(p.metadata, *mm)
}
func TestDecodeScopeMetrics(t *testing.T) {
buildMetricsData := func() []byte {
scopeName := "my-scope"
scopeVersion := "v1.0"
envVal := "prod"
intVal := int64(1)
md := &MetricsData{
ResourceMetrics: []*ResourceMetrics{
{
Resource: &Resource{
Attributes: []*KeyValue{
{Key: "job", Value: &AnyValue{StringValue: strPtr("vm")}},
{Key: "region", Value: &AnyValue{StringValue: strPtr("us-east-1")}},
},
},
ScopeMetrics: []*ScopeMetrics{
{
Scope: &InstrumentationScope{
Name: &scopeName,
Version: &scopeVersion,
Attributes: []*KeyValue{
{Key: "env", Value: &AnyValue{StringValue: &envVal}},
},
},
Metrics: []*Metric{
{
Name: "my-gauge",
Description: "a test gauge",
Gauge: &Gauge{
DataPoints: []*NumberDataPoint{
{
Attributes: []*KeyValue{{Key: "label1", Value: &AnyValue{StringValue: strPtr("value1")}}},
IntValue: &intVal,
TimeUnixNano: 1000,
},
},
},
},
},
},
},
},
},
}
return md.MarshalProtobuf(nil)
}
decode := func(t *testing.T, data []byte) []prompb.Label {
t.Helper()
mp := &testMetricPusher{}
if err := DecodeMetricsData(data, mp); err != nil {
t.Fatalf("DecodeMetricsData error: %v", err)
}
if len(mp.samples) != 1 {
t.Fatalf("expected 1 sample, got %d", len(mp.samples))
}
return mp.samples[0].labels
}
checkLabels := func(t *testing.T, got []prompb.Label, want map[string]string) {
t.Helper()
gotMap := make(map[string]string, len(got))
for _, l := range got {
gotMap[string([]byte(l.Name))] = string([]byte(l.Value))
}
if !reflect.DeepEqual(gotMap, want) {
t.Errorf("unexpected labels:\n got: %s\n want: %s", fmtLabelMap(gotMap), fmtLabelMap(want))
}
}
// (default) promoteScopeMetadata=true + promoteAllResourceAttributes=true:
// got all scope labels and resource attrs
t.Run("scope_and_all_resource_attrs", func(t *testing.T) {
labels := decode(t, buildMetricsData())
checkLabels(t, labels, map[string]string{
"job": "vm",
"region": "us-east-1",
"scope.name": "my-scope",
"scope.version": "v1.0",
"scope.attributes.env": "prod",
"label1": "value1",
})
})
// promoteScopeMetadata=false + promoteAllResourceAttributes=true:
// got all resource attrs, no scope labels.
t.Run("no_scope_all_resource_attrs", func(t *testing.T) {
prevScope := *promoteScopeMetadata
*promoteScopeMetadata = false
defer func() { *promoteScopeMetadata = prevScope }()
labels := decode(t, buildMetricsData())
checkLabels(t, labels, map[string]string{
"job": "vm",
"region": "us-east-1",
"label1": "value1",
})
})
// promoteScopeMetadata=true + promoteAllResourceAttributes=false + promoteResourceAttributes=[region]:
// got the `region`` attr
t.Run("scope_selected_resource_attrs", func(t *testing.T) {
prevAll := *promoteAllResourceAttributes
*promoteAllResourceAttributes = false
defer func() { *promoteAllResourceAttributes = prevAll }()
prevPromote := *promoteResourceAttributes
*promoteResourceAttributes = flagutil.ArrayString{"region"}
defer func() { *promoteResourceAttributes = prevPromote }()
labels := decode(t, buildMetricsData())
checkLabels(t, labels, map[string]string{
"region": "us-east-1",
"scope.name": "my-scope",
"scope.version": "v1.0",
"scope.attributes.env": "prod",
"label1": "value1",
})
})
// promoteScopeMetadata=true + promoteAllResourceAttributes=true + ignoreResourceAttributes=[region]:
// got all resource attrs except `region`
t.Run("scope_all_resource_attrs_ignore_region", func(t *testing.T) {
prevIgnore := *ignoreResourceAttributes
*ignoreResourceAttributes = flagutil.ArrayString{"region"}
defer func() { *ignoreResourceAttributes = prevIgnore }()
labels := decode(t, buildMetricsData())
checkLabels(t, labels, map[string]string{
"job": "vm",
"scope.name": "my-scope",
"scope.version": "v1.0",
"scope.attributes.env": "prod",
"label1": "value1",
})
})
// promoteScopeMetadata=false + promoteAllResourceAttributes=false + promoteResourceAttributes=[job]:
// got only `job` attr
t.Run("no_scope_selected_resource_attrs", func(t *testing.T) {
prevScope := *promoteScopeMetadata
*promoteScopeMetadata = false
defer func() { *promoteScopeMetadata = prevScope }()
prevAll := *promoteAllResourceAttributes
*promoteAllResourceAttributes = false
defer func() { *promoteAllResourceAttributes = prevAll }()
prevPromote := *promoteResourceAttributes
*promoteResourceAttributes = flagutil.ArrayString{"job"}
defer func() { *promoteResourceAttributes = prevPromote }()
labels := decode(t, buildMetricsData())
checkLabels(t, labels, map[string]string{
"job": "vm",
"label1": "value1",
})
})
}
func strPtr(s string) *string {
return &s
}
func fmtLabelMap(m map[string]string) string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
parts := make([]string, len(keys))
for i, k := range keys {
parts[i] = fmt.Sprintf("%s=%q", k, m[k])
}
return "{" + strings.Join(parts, ", ") + "}"
}