mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-07-15 13:20:36 +03:00
Compare commits
16 Commits
docs-artic
...
issue-1059
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
50f9ca5dd5 | ||
|
|
bd66108d7a | ||
|
|
e20b76a0af | ||
|
|
d1e2df0404 | ||
|
|
e9fa6f5f76 | ||
|
|
baededddc1 | ||
|
|
1870726d3a | ||
|
|
18a4685440 | ||
|
|
27f3c6ba45 | ||
|
|
c558291847 | ||
|
|
9bd219fdc7 | ||
|
|
ec9d37ce36 | ||
|
|
607630b9f5 | ||
|
|
f4df18d2db | ||
|
|
29bc38871d | ||
|
|
3f35399c24 |
88
app/vmagent/remotewrite/obfuscation.go
Normal file
88
app/vmagent/remotewrite/obfuscation.go
Normal file
@@ -0,0 +1,88 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
)
|
||||
|
||||
type obfuscationCtx struct {
|
||||
labels []prompb.Label
|
||||
cacheObfuscatedResult map[string]string
|
||||
}
|
||||
|
||||
func (ctx *obfuscationCtx) Reset() {
|
||||
promrelabel.CleanLabels(ctx.labels)
|
||||
ctx.labels = ctx.labels[:0]
|
||||
clear(ctx.cacheObfuscatedResult)
|
||||
}
|
||||
|
||||
var obfuscationCtxPool = &sync.Pool{
|
||||
New: func() any {
|
||||
return &obfuscationCtx{
|
||||
cacheObfuscatedResult: make(map[string]string),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
func (rwctx *remoteWriteCtx) initObfuscationConfig() {
|
||||
if len(*obfuscationLabels) == 0 {
|
||||
return
|
||||
}
|
||||
idx := rwctx.idx
|
||||
rwctx.obfuscationLabels = make(map[string]struct{})
|
||||
rwObfuscationLabels := obfuscationLabels.GetOptionalArg(idx)
|
||||
rwObfuscationLabelsList := strings.Split(rwObfuscationLabels, "^^")
|
||||
|
||||
for _, label := range rwObfuscationLabelsList {
|
||||
rwctx.obfuscationLabels[label] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (rwctx *remoteWriteCtx) applyObfuscation(tss []prompb.TimeSeries, ctx *obfuscationCtx) []prompb.TimeSeries {
|
||||
if len(rwctx.obfuscationLabels) == 0 || len(tss) == 0 {
|
||||
return tss
|
||||
}
|
||||
poolLabels := ctx.labels[:0]
|
||||
for i := range tss {
|
||||
ts := &tss[i]
|
||||
labels := ts.Labels
|
||||
j := 0
|
||||
needToObfuscate := false
|
||||
for ; j < len(labels); j++ {
|
||||
label := &labels[j]
|
||||
if _, ok := rwctx.obfuscationLabels[label.Name]; !ok {
|
||||
continue
|
||||
}
|
||||
needToObfuscate = true
|
||||
break
|
||||
}
|
||||
if !needToObfuscate {
|
||||
continue
|
||||
}
|
||||
// Copy the label array to apply obfuscation
|
||||
poolLabelsLen := len(poolLabels)
|
||||
poolLabels = append(poolLabels, labels...)
|
||||
ctx.labels = poolLabels
|
||||
ts.Labels = poolLabels[poolLabelsLen:]
|
||||
for ; j < len(ts.Labels); j++ {
|
||||
label := &ts.Labels[j]
|
||||
if _, ok := rwctx.obfuscationLabels[label.Name]; !ok {
|
||||
continue
|
||||
}
|
||||
if obfuscatedValue, ok := ctx.cacheObfuscatedResult[label.Value]; ok {
|
||||
// fast path: the obfuscated result was calculated before
|
||||
label.Value = obfuscatedValue
|
||||
} else {
|
||||
obfuscatedResult := sha256.Sum256([]byte(label.Value))
|
||||
ctx.cacheObfuscatedResult[label.Value] = hex.EncodeToString(obfuscatedResult[:])
|
||||
label.Value = ctx.cacheObfuscatedResult[label.Value]
|
||||
}
|
||||
}
|
||||
}
|
||||
return tss
|
||||
}
|
||||
@@ -108,6 +108,8 @@ var (
|
||||
|
||||
enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
|
||||
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
|
||||
obfuscationLabels = flagutil.NewArrayString("remoteWrite.obfuscationLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
|
||||
"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\".")
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -881,6 +883,8 @@ type remoteWriteCtx struct {
|
||||
pss []*pendingSeries
|
||||
pssNextIdx atomic.Uint64
|
||||
|
||||
obfuscationLabels map[string]struct{}
|
||||
|
||||
rowsPushedAfterRelabel *metrics.Counter
|
||||
rowsDroppedByRelabel *metrics.Counter
|
||||
mdxRowsPreserved *metrics.Counter
|
||||
@@ -995,6 +999,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
|
||||
rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
|
||||
}
|
||||
rwctx.initStreamAggrConfig()
|
||||
rwctx.initObfuscationConfig()
|
||||
|
||||
if enableMdx.GetOptionalArg(argIdx) {
|
||||
mdxFilter := mdx.NewFilter()
|
||||
@@ -1198,6 +1203,7 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
|
||||
func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
|
||||
var rctx *relabelCtx
|
||||
var v *[]prompb.TimeSeries
|
||||
var octx *obfuscationCtx
|
||||
defer func() {
|
||||
if rctx == nil {
|
||||
return
|
||||
@@ -1216,6 +1222,24 @@ func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries)
|
||||
rctx.appendExtraLabels(tss, labelsGlobal)
|
||||
}
|
||||
|
||||
if len(rwctx.obfuscationLabels) != 0 {
|
||||
if rctx == nil {
|
||||
shadowTss := tssPool.Get().(*[]prompb.TimeSeries)
|
||||
tss = append(*shadowTss, tss...)
|
||||
defer func() {
|
||||
*shadowTss = prompb.ResetTimeSeries(tss)
|
||||
tssPool.Put(shadowTss)
|
||||
}()
|
||||
}
|
||||
octx = obfuscationCtxPool.Get().(*obfuscationCtx)
|
||||
defer func() {
|
||||
octx.Reset()
|
||||
obfuscationCtxPool.Put(octx)
|
||||
}()
|
||||
|
||||
tss = rwctx.applyObfuscation(tss, octx)
|
||||
}
|
||||
|
||||
pss := rwctx.pss
|
||||
idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
@@ -376,3 +378,126 @@ func TestCalculateHealthyRwctxIdx(t *testing.T) {
|
||||
f(1, []int{0}, nil)
|
||||
f(1, []int{}, []int{0})
|
||||
}
|
||||
|
||||
func TestRemoteWriteObfuscation(t *testing.T) {
|
||||
f := func(obfuscationLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
|
||||
t.Helper()
|
||||
rwctx := &remoteWriteCtx{
|
||||
idx: 0,
|
||||
}
|
||||
defer metrics.UnregisterAllMetrics()
|
||||
originValue := *obfuscationLabels
|
||||
defer func() {
|
||||
*obfuscationLabels = originValue
|
||||
}()
|
||||
*obfuscationLabels = []string{obfuscationLabelList}
|
||||
rwctx.initObfuscationConfig()
|
||||
octx := obfuscationCtx{
|
||||
cacheObfuscatedResult: make(map[string]string),
|
||||
}
|
||||
outputTss := rwctx.applyObfuscation(inputTss, &octx)
|
||||
|
||||
if !reflect.DeepEqual(expectedTss, outputTss) {
|
||||
t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
|
||||
}
|
||||
}
|
||||
|
||||
sha256Result := func(str string) string {
|
||||
sha256Result := sha256.Sum256([]byte(str))
|
||||
return hex.EncodeToString(sha256Result[:])
|
||||
}
|
||||
|
||||
// 1. obfuscation is not set.
|
||||
f("",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// 2. obfuscate the value of "ip" label
|
||||
f("ip",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: sha256Result("123")},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// 3. obfuscate the values of "ip" and "instance"
|
||||
f("ip^^instance",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "job", Value: "123"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: sha256Result("123")},
|
||||
{Name: "instance", Value: sha256Result("1234")},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "job", Value: "123"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ The sandbox cluster installation runs under the constant load generated by
|
||||
See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
|
||||
|
||||
## tip
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): introduce obfuscation functionality for remote write. By setting `-remoteWrite.obfuscationLabels`, the values of the specific labels will be anonymized before they're sent to corresponding `-remoteWrite.url`. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
|
||||
|
||||
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support `fill` modifiers to allow missing series on either side of a binary operation to be filled with a provided default value. See [#10598](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10598).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
|
||||
@@ -210,7 +211,6 @@ Released at 2026-04-24
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect evaluation of binary operations caused by an ordering bug (e.g. `10 - (3 + 3 + 4)` being evaluated as `10 - 3 + 3 + 4`). The issue was introduced in v1.140.0, v1.136.4, and v1.122.19. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
|
||||
|
||||
## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
|
||||
|
||||
Released at 2026-04-10
|
||||
|
||||
**Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).
|
||||
|
||||
@@ -323,7 +323,8 @@ flowchart TB
|
||||
H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
|
||||
H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
|
||||
H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
|
||||
H5 --> H6[[push to <b>-remoteWrite.url</b>]]
|
||||
H5 --> H6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscationLabels</b>]
|
||||
H6 --> H7[[push to <b>-remoteWrite.url</b>]]
|
||||
|
||||
%% Right branch
|
||||
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
|
||||
@@ -331,7 +332,8 @@ flowchart TB
|
||||
R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
|
||||
R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
|
||||
R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
|
||||
R5 --> R6[[push to <b>-remoteWrite.url</b>]]
|
||||
R5 --> R6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscationLabels</b>]
|
||||
R6 --> R7[[push to <b>-remoteWrite.url</b>]]
|
||||
```
|
||||
|
||||
Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:
|
||||
@@ -646,6 +648,25 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
|
||||
/path/to/vmagent -remoteWrite.url=http://127.0.0.1:8428/api/v1/write?extra_label="env=prod"
|
||||
```
|
||||
|
||||
## Obfuscating label values
|
||||
|
||||
Before sending metrics to `-remoteWrite.url`, `vmagent` can obfuscate the values of specific labels by using `-remoteWrite.obfuscationLabels`.
|
||||
|
||||
This is useful when one or more `-remoteWrite.url` endpoints point to external services, such as monitoring vendors outside the department or company.
|
||||
To meet security and compliance requirements, sensitive label values such as `ip`, `host`, `instance`, or `datacenter` can be obfuscated before metrics are sent to these external systems.
|
||||
|
||||
Use `-remoteWrite.obfuscationLabels` to specify which labels should have their values obfuscated for the corresponding `-remoteWrite.url`. Multiple label names must be separated with `^^`.
|
||||
|
||||
```sh
|
||||
./vmagent \
|
||||
-remoteWrite.url=http://<external-service1> \
|
||||
-remoteWrite.obfuscationLabels='instance^^datacenter' \
|
||||
-remoteWrite.url=http://<external-service2> \
|
||||
-remoteWrite.obfuscationLabels='instance' \
|
||||
-remoteWrite.url=http://<internal-service> \
|
||||
-remoteWrite.obfuscationLabels=''
|
||||
```
|
||||
|
||||
## Automatically generated metrics
|
||||
|
||||
`vmagent` automatically generates the following metrics for each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)
|
||||
|
||||
Reference in New Issue
Block a user