Compare commits

...

16 Commits

Author SHA1 Message Date
Jayice
50f9ca5dd5 polish documentation 2026-07-10 01:50:52 +08:00
Jayice
bd66108d7a polish documentation 2026-07-10 01:43:57 +08:00
Jayice
e20b76a0af polish documentation 2026-07-10 01:43:16 +08:00
Jayice
d1e2df0404 polish documentation 2026-07-10 01:37:25 +08:00
JAYICE
e9fa6f5f76 Merge branch 'master' into issue-10599
Signed-off-by: JAYICE <1185430411@qq.com>
2026-07-10 01:33:20 +08:00
Jayice
baededddc1 improve unit test 2026-07-10 01:28:55 +08:00
JAYICE
1870726d3a Update app/vmagent/remotewrite/obfuscation.go
Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
Signed-off-by: JAYICE <jayice.zhou@qq.com>
2026-07-10 01:26:57 +08:00
Jayice
18a4685440 polish codes 2026-07-10 01:10:49 +08:00
Jayice
27f3c6ba45 use pooled labels array 2026-06-24 16:39:50 +08:00
Jayice
c558291847 update CHANGELOG.md 2026-04-17 14:34:07 +08:00
Jayice
9bd219fdc7 address review 2026-04-17 14:31:52 +08:00
Jayice
ec9d37ce36 improve code style 2026-04-17 13:55:33 +08:00
Jayice
607630b9f5 add unit test 2026-04-17 13:52:07 +08:00
Jayice
f4df18d2db add documentation for obfuscation 2026-04-17 13:03:25 +08:00
Jayice
29bc38871d address review 2026-04-16 15:49:04 +08:00
Jayice
3f35399c24 support obfuscation for rw 2026-04-15 15:09:40 +08:00
5 changed files with 261 additions and 3 deletions

View File

@@ -0,0 +1,88 @@
package remotewrite
import (
"crypto/sha256"
"encoding/hex"
"strings"
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
)
type obfuscationCtx struct {
labels []prompb.Label
cacheObfuscatedResult map[string]string
}
func (ctx *obfuscationCtx) Reset() {
promrelabel.CleanLabels(ctx.labels)
ctx.labels = ctx.labels[:0]
clear(ctx.cacheObfuscatedResult)
}
var obfuscationCtxPool = &sync.Pool{
New: func() any {
return &obfuscationCtx{
cacheObfuscatedResult: make(map[string]string),
}
},
}
func (rwctx *remoteWriteCtx) initObfuscationConfig() {
if len(*obfuscationLabels) == 0 {
return
}
idx := rwctx.idx
rwctx.obfuscationLabels = make(map[string]struct{})
rwObfuscationLabels := obfuscationLabels.GetOptionalArg(idx)
rwObfuscationLabelsList := strings.Split(rwObfuscationLabels, "^^")
for _, label := range rwObfuscationLabelsList {
rwctx.obfuscationLabels[label] = struct{}{}
}
}
func (rwctx *remoteWriteCtx) applyObfuscation(tss []prompb.TimeSeries, ctx *obfuscationCtx) []prompb.TimeSeries {
if len(rwctx.obfuscationLabels) == 0 || len(tss) == 0 {
return tss
}
poolLabels := ctx.labels[:0]
for i := range tss {
ts := &tss[i]
labels := ts.Labels
j := 0
needToObfuscate := false
for ; j < len(labels); j++ {
label := &labels[j]
if _, ok := rwctx.obfuscationLabels[label.Name]; !ok {
continue
}
needToObfuscate = true
break
}
if !needToObfuscate {
continue
}
// Copy the label array to apply obfuscation
poolLabelsLen := len(poolLabels)
poolLabels = append(poolLabels, labels...)
ctx.labels = poolLabels
ts.Labels = poolLabels[poolLabelsLen:]
for ; j < len(ts.Labels); j++ {
label := &ts.Labels[j]
if _, ok := rwctx.obfuscationLabels[label.Name]; !ok {
continue
}
if obfuscatedValue, ok := ctx.cacheObfuscatedResult[label.Value]; ok {
// fast path: the obfuscated result was calculated before
label.Value = obfuscatedValue
} else {
obfuscatedResult := sha256.Sum256([]byte(label.Value))
ctx.cacheObfuscatedResult[label.Value] = hex.EncodeToString(obfuscatedResult[:])
label.Value = ctx.cacheObfuscatedResult[label.Value]
}
}
}
return tss
}

View File

@@ -108,6 +108,8 @@ var (
enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
obfuscationLabels = flagutil.NewArrayString("remoteWrite.obfuscationLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\".")
)
var (
@@ -881,6 +883,8 @@ type remoteWriteCtx struct {
pss []*pendingSeries
pssNextIdx atomic.Uint64
obfuscationLabels map[string]struct{}
rowsPushedAfterRelabel *metrics.Counter
rowsDroppedByRelabel *metrics.Counter
mdxRowsPreserved *metrics.Counter
@@ -995,6 +999,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
}
rwctx.initStreamAggrConfig()
rwctx.initObfuscationConfig()
if enableMdx.GetOptionalArg(argIdx) {
mdxFilter := mdx.NewFilter()
@@ -1198,6 +1203,7 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
var rctx *relabelCtx
var v *[]prompb.TimeSeries
var octx *obfuscationCtx
defer func() {
if rctx == nil {
return
@@ -1216,6 +1222,24 @@ func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries)
rctx.appendExtraLabels(tss, labelsGlobal)
}
if len(rwctx.obfuscationLabels) != 0 {
if rctx == nil {
shadowTss := tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*shadowTss, tss...)
defer func() {
*shadowTss = prompb.ResetTimeSeries(tss)
tssPool.Put(shadowTss)
}()
}
octx = obfuscationCtxPool.Get().(*obfuscationCtx)
defer func() {
octx.Reset()
obfuscationCtxPool.Put(octx)
}()
tss = rwctx.applyObfuscation(tss, octx)
}
pss := rwctx.pss
idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))

View File

@@ -1,6 +1,8 @@
package remotewrite
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"math"
"reflect"
@@ -376,3 +378,126 @@ func TestCalculateHealthyRwctxIdx(t *testing.T) {
f(1, []int{0}, nil)
f(1, []int{}, []int{0})
}
func TestRemoteWriteObfuscation(t *testing.T) {
f := func(obfuscationLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
t.Helper()
rwctx := &remoteWriteCtx{
idx: 0,
}
defer metrics.UnregisterAllMetrics()
originValue := *obfuscationLabels
defer func() {
*obfuscationLabels = originValue
}()
*obfuscationLabels = []string{obfuscationLabelList}
rwctx.initObfuscationConfig()
octx := obfuscationCtx{
cacheObfuscatedResult: make(map[string]string),
}
outputTss := rwctx.applyObfuscation(inputTss, &octx)
if !reflect.DeepEqual(expectedTss, outputTss) {
t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
}
}
sha256Result := func(str string) string {
sha256Result := sha256.Sum256([]byte(str))
return hex.EncodeToString(sha256Result[:])
}
// 1. obfuscation is not set.
f("",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
// 2. obfuscate the value of "ip" label
f("ip",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: sha256Result("123")},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
// 3. obfuscate the values of "ip" and "instance"
f("ip^^instance",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
{
Labels: []prompb.Label{
{Name: "job", Value: "123"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: sha256Result("123")},
{Name: "instance", Value: sha256Result("1234")},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
{
Labels: []prompb.Label{
{Name: "job", Value: "123"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
}

View File

@@ -25,6 +25,7 @@ The sandbox cluster installation runs under the constant load generated by
See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
## tip
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): introduce obfuscation functionality for remote write. By setting `-remoteWrite.obfuscationLabels`, the values of the specific labels will be anonymized before they're sent to corresponding `-remoteWrite.url`. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support `fill` modifiers to allow missing series on either side of a binary operation to be filled with a provided default value. See [#10598](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10598).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
@@ -210,7 +211,6 @@ Released at 2026-04-24
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect evaluation of binary operations caused by an ordering bug (e.g. `10 - (3 + 3 + 4)` being evaluated as `10 - 3 + 3 + 4`). The issue was introduced in v1.140.0, v1.136.4, and v1.122.19. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
Released at 2026-04-10
**Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).

View File

@@ -323,7 +323,8 @@ flowchart TB
H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
H5 --> H6[[push to <b>-remoteWrite.url</b>]]
H5 --> H6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscationLabels</b>]
H6 --> H7[[push to <b>-remoteWrite.url</b>]]
%% Right branch
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
@@ -331,7 +332,8 @@ flowchart TB
R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
R5 --> R6[[push to <b>-remoteWrite.url</b>]]
R5 --> R6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscationLabels</b>]
R6 --> R7[[push to <b>-remoteWrite.url</b>]]
```
Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:
@@ -646,6 +648,25 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
/path/to/vmagent -remoteWrite.url=http://127.0.0.1:8428/api/v1/write?extra_label="env=prod"
```
## Obfuscating label values
Before sending metrics to `-remoteWrite.url`, `vmagent` can obfuscate the values of specific labels by using `-remoteWrite.obfuscationLabels`.
This is useful when one or more `-remoteWrite.url` endpoints point to external services, such as monitoring vendors outside the department or company.
To meet security and compliance requirements, sensitive label values such as `ip`, `host`, `instance`, or `datacenter` can be obfuscated before metrics are sent to these external systems.
Use `-remoteWrite.obfuscationLabels` to specify which labels should have their values obfuscated for the corresponding `-remoteWrite.url`. Multiple label names must be separated with `^^`.
```sh
./vmagent \
-remoteWrite.url=http://<external-service1> \
-remoteWrite.obfuscationLabels='instance^^datacenter' \
-remoteWrite.url=http://<external-service2> \
-remoteWrite.obfuscationLabels='instance' \
-remoteWrite.url=http://<internal-service> \
-remoteWrite.obfuscationLabels=''
```
## Automatically generated metrics
`vmagent` automatically generates the following metrics for each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)