Compare commits
1 Commits
docs-artic
...
query-reso
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2321ea3364 |
@@ -1,108 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
)
|
||||
|
||||
type obfuscateLabelsCtx struct {
|
||||
labels []prompb.Label
|
||||
|
||||
// buf holds allocations for cached results below
|
||||
buf []byte
|
||||
|
||||
// cacheResults maps original label values to their SHA-256 hex digests,
|
||||
// avoiding redundant hashing of repeated values within a single batch.
|
||||
cacheResults map[string]string
|
||||
}
|
||||
|
||||
func (olctx *obfuscateLabelsCtx) reset() {
|
||||
promrelabel.CleanLabels(olctx.labels)
|
||||
olctx.labels = olctx.labels[:0]
|
||||
clear(olctx.cacheResults)
|
||||
olctx.buf = olctx.buf[:0]
|
||||
}
|
||||
|
||||
var obfuscateLabelsCtxPool = &sync.Pool{
|
||||
New: func() any {
|
||||
return &obfuscateLabelsCtx{
|
||||
cacheResults: make(map[string]string),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
func getObfuscateLabelsCtx() *obfuscateLabelsCtx {
|
||||
return obfuscateLabelsCtxPool.Get().(*obfuscateLabelsCtx)
|
||||
}
|
||||
|
||||
func putObfuscateLabelsCtx(ctx *obfuscateLabelsCtx) {
|
||||
ctx.reset()
|
||||
obfuscateLabelsCtxPool.Put(ctx)
|
||||
}
|
||||
|
||||
func (olctx *obfuscateLabelsCtx) obfuscate(tss []prompb.TimeSeries, obfuscateLabels []string) []prompb.TimeSeries {
|
||||
if len(obfuscateLabels) == 0 || len(tss) == 0 {
|
||||
return tss
|
||||
}
|
||||
labels := olctx.labels
|
||||
for i := range tss {
|
||||
ts := &tss[i]
|
||||
labelsLen := len(labels)
|
||||
labels = append(labels, ts.Labels...)
|
||||
found := false
|
||||
for _, labelName := range obfuscateLabels {
|
||||
tmp := promrelabel.GetLabelByName(labels[labelsLen:], labelName)
|
||||
if tmp == nil {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if obfuscatedValue, ok := olctx.cacheResults[tmp.Value]; ok {
|
||||
// fast path: the obfuscated result was calculated before
|
||||
tmp.Value = obfuscatedValue
|
||||
continue
|
||||
}
|
||||
|
||||
digest := sha256.Sum256(bytesutil.ToUnsafeBytes(tmp.Value))
|
||||
buf := olctx.buf
|
||||
bufLen := len(buf)
|
||||
buf = hex.AppendEncode(buf, digest[:])
|
||||
obfuscatedValue := bytesutil.ToUnsafeString(buf[bufLen:])
|
||||
|
||||
olctx.buf = buf
|
||||
olctx.cacheResults[tmp.Value] = obfuscatedValue
|
||||
tmp.Value = obfuscatedValue
|
||||
}
|
||||
if found {
|
||||
ts.Labels = labels[labelsLen:]
|
||||
} else {
|
||||
labels = labels[:labelsLen]
|
||||
}
|
||||
}
|
||||
olctx.labels = labels
|
||||
return tss
|
||||
}
|
||||
|
||||
func (rwctx *remoteWriteCtx) initObfuscateLabels() {
|
||||
if len(*obfuscateLabels) == 0 {
|
||||
return
|
||||
}
|
||||
idx := rwctx.idx
|
||||
rwObfuscateLabels := obfuscateLabels.GetOptionalArg(idx)
|
||||
rwObfuscateLabelsList := strings.Split(rwObfuscateLabels, "^^")
|
||||
|
||||
for _, label := range rwObfuscateLabelsList {
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(rwctx.obfuscateLabels, label) {
|
||||
rwctx.obfuscateLabels = append(rwctx.obfuscateLabels, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
func TestRemoteWriteObfuscateLabels(t *testing.T) {
|
||||
f := func(obfuscateLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
|
||||
t.Helper()
|
||||
rwctx := &remoteWriteCtx{
|
||||
idx: 0,
|
||||
}
|
||||
olctx := &obfuscateLabelsCtx{
|
||||
cacheResults: make(map[string]string),
|
||||
}
|
||||
defer metrics.UnregisterAllMetrics()
|
||||
originValue := *obfuscateLabels
|
||||
defer func() {
|
||||
*obfuscateLabels = originValue
|
||||
}()
|
||||
*obfuscateLabels = []string{obfuscateLabelList}
|
||||
rwctx.initObfuscateLabels()
|
||||
|
||||
outputTss := olctx.obfuscate(inputTss, rwctx.obfuscateLabels)
|
||||
|
||||
if !reflect.DeepEqual(expectedTss, outputTss) {
|
||||
t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
|
||||
}
|
||||
}
|
||||
|
||||
sha256Result := func(str string) string {
|
||||
sha256Result := sha256.Sum256([]byte(str))
|
||||
return hex.EncodeToString(sha256Result[:])
|
||||
}
|
||||
|
||||
// 1. obfuscation is not set.
|
||||
f("",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// 1. obfuscation is set for another rwctx.
|
||||
f(",ip",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// 2. obfuscate the value of "ip" label
|
||||
f("ip",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: sha256Result("123")},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// 3. obfuscate the values of "ip" and "instance"
|
||||
f("ip^^instance",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "1234"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "job", Value: "123"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: sha256Result("123")},
|
||||
{Name: "instance", Value: sha256Result("1234")},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "job", Value: "123"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// 4. duplicate label names in config must produce single SHA-256, not double
|
||||
f("ip^^ip",
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: sha256Result("123")},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package remotewrite
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
func BenchmarkRemoteWriteObfuscateLabels(b *testing.B) {
|
||||
originValue := *obfuscateLabels
|
||||
defer func() {
|
||||
*obfuscateLabels = originValue
|
||||
}()
|
||||
*obfuscateLabels = []string{"ip^^instance"}
|
||||
sha256Result := func(str string) string {
|
||||
sha256Result := sha256.Sum256([]byte(str))
|
||||
return hex.EncodeToString(sha256Result[:])
|
||||
}
|
||||
expected := []prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: sha256Result("123")},
|
||||
{Name: "instance", Value: sha256Result("12345")},
|
||||
{Name: "__name__", Value: "http_requests_total"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: sha256Result("1236")},
|
||||
{Name: "instance", Value: sha256Result("some-long-instante-string")},
|
||||
{Name: "__name__", Value: "concurrent_requests"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
defer metrics.UnregisterAllMetrics()
|
||||
|
||||
inputTss := []prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "123"},
|
||||
{Name: "instance", Value: "12345"},
|
||||
{Name: "__name__", Value: "http_requests_total"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "ip", Value: "1236"},
|
||||
{Name: "instance", Value: "some-long-instante-string"},
|
||||
{Name: "__name__", Value: "concurrent_requests"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Value: 1, Timestamp: 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
rwctx := &remoteWriteCtx{
|
||||
idx: 0,
|
||||
}
|
||||
olctx := &obfuscateLabelsCtx{
|
||||
cacheResults: make(map[string]string),
|
||||
}
|
||||
rwctx.initObfuscateLabels()
|
||||
var localTss []prompb.TimeSeries
|
||||
for pb.Next() {
|
||||
// always make a shallow copy because obfuscate changes input
|
||||
localTss = localTss[:0]
|
||||
localTss = append(localTss, inputTss...)
|
||||
olctx.reset()
|
||||
outputTss := olctx.obfuscate(localTss, rwctx.obfuscateLabels)
|
||||
if !reflect.DeepEqual(expected, outputTss) {
|
||||
b.Fatalf("unexpected output: got: \n%v\n want: \n%v\n", outputTss, expected)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
@@ -107,12 +107,7 @@ var (
|
||||
"By default, metadata sending is controlled by the global -enableMetadata flag")
|
||||
|
||||
enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
|
||||
"Can be combined with -remoteWrite.obfuscateLabels to hide sensitive label values in the forwarded metrics. "+
|
||||
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
|
||||
obfuscateLabels = flagutil.NewArrayString("remoteWrite.obfuscateLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
|
||||
"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
|
||||
"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
|
||||
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -886,8 +881,6 @@ type remoteWriteCtx struct {
|
||||
pss []*pendingSeries
|
||||
pssNextIdx atomic.Uint64
|
||||
|
||||
obfuscateLabels []string
|
||||
|
||||
rowsPushedAfterRelabel *metrics.Counter
|
||||
rowsDroppedByRelabel *metrics.Counter
|
||||
mdxRowsPreserved *metrics.Counter
|
||||
@@ -1002,7 +995,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
|
||||
rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
|
||||
}
|
||||
rwctx.initStreamAggrConfig()
|
||||
rwctx.initObfuscateLabels()
|
||||
|
||||
if enableMdx.GetOptionalArg(argIdx) {
|
||||
mdxFilter := mdx.NewFilter()
|
||||
@@ -1206,41 +1198,24 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
|
||||
func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
|
||||
var rctx *relabelCtx
|
||||
var v *[]prompb.TimeSeries
|
||||
var olctx *obfuscateLabelsCtx
|
||||
defer func() {
|
||||
if v != nil {
|
||||
*v = prompb.ResetTimeSeries(tss)
|
||||
tssPool.Put(v)
|
||||
}
|
||||
if rctx != nil {
|
||||
putRelabelCtx(rctx)
|
||||
}
|
||||
if olctx != nil {
|
||||
putObfuscateLabelsCtx(olctx)
|
||||
if rctx == nil {
|
||||
return
|
||||
}
|
||||
*v = prompb.ResetTimeSeries(tss)
|
||||
tssPool.Put(v)
|
||||
putRelabelCtx(rctx)
|
||||
}()
|
||||
|
||||
copyTimeSeriesIfNeeded := func() {
|
||||
if v == nil {
|
||||
v = tssPool.Get().(*[]prompb.TimeSeries)
|
||||
tss = append(*v, tss...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(labelsGlobal) > 0 {
|
||||
// Make a copy of tss before adding extra labels to prevent
|
||||
// from affecting time series for other remoteWrite.url configs.
|
||||
rctx = getRelabelCtx()
|
||||
copyTimeSeriesIfNeeded()
|
||||
v = tssPool.Get().(*[]prompb.TimeSeries)
|
||||
tss = append(*v, tss...)
|
||||
rctx.appendExtraLabels(tss, labelsGlobal)
|
||||
}
|
||||
|
||||
if len(rwctx.obfuscateLabels) != 0 {
|
||||
copyTimeSeriesIfNeeded()
|
||||
olctx = getObfuscateLabelsCtx()
|
||||
tss = olctx.obfuscate(tss, rwctx.obfuscateLabels)
|
||||
}
|
||||
|
||||
pss := rwctx.pss
|
||||
idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))
|
||||
|
||||
|
||||
@@ -369,10 +369,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
return true
|
||||
case "/tags/delSeries":
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
|
||||
return true
|
||||
}
|
||||
if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
|
||||
return true
|
||||
}
|
||||
@@ -392,10 +388,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
return true
|
||||
case "/api/v1/admin/tsdb/delete_series":
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
|
||||
return true
|
||||
}
|
||||
if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -81,7 +81,6 @@ func AssertSeries(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID st
|
||||
Status: "success",
|
||||
Data: want,
|
||||
},
|
||||
Retries: 1000,
|
||||
FailNow: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
@@ -180,3 +182,123 @@ func TestClusterMaxSeries(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func searchLimitsStartVmsingle(tc *apptest.TestCase, vmstorageFlags, vmselectFlags []string) apptest.PrometheusWriteQuerier {
|
||||
vmsingleFlags := []string{
|
||||
"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
|
||||
"-retentionPeriod=100y",
|
||||
}
|
||||
vmsingleFlags = append(vmsingleFlags, vmstorageFlags...)
|
||||
vmsingleFlags = append(vmsingleFlags, vmselectFlags...)
|
||||
return tc.MustStartVmsingle("vmsingle", vmsingleFlags)
|
||||
}
|
||||
|
||||
func searchLimitsStopVmsingle(tc *apptest.TestCase) {
|
||||
tc.StopApp("vmsingle")
|
||||
}
|
||||
|
||||
func searchLimitsStartVmcluster(tc *apptest.TestCase, vmstorageFlags, vmselectFlags []string) apptest.PrometheusWriteQuerier {
|
||||
vmstorageFlags = append(vmstorageFlags, []string{
|
||||
"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage"),
|
||||
"-retentionPeriod=100y",
|
||||
}...)
|
||||
vmstorage := tc.MustStartVmstorage("vmstorage", vmstorageFlags)
|
||||
vminsert := tc.MustStartVminsert("vminsert", []string{
|
||||
"-storageNode=" + vmstorage.VminsertAddr(),
|
||||
})
|
||||
vmselectFlags = append(vmselectFlags, []string{
|
||||
"-storageNode=" + vmstorage.VmselectAddr(),
|
||||
}...)
|
||||
vmselect := tc.MustStartVmselect("vmselect", vmselectFlags)
|
||||
return &apptest.Vmcluster{vminsert, vmselect, []*apptest.Vmstorage{vmstorage}}
|
||||
}
|
||||
|
||||
func searchLimitsStopVmcluster(tc *apptest.TestCase) {
|
||||
tc.StopApp("vminsert")
|
||||
tc.StopApp("vmselect")
|
||||
tc.StopApp("vmstorage")
|
||||
}
|
||||
|
||||
type searchLimitsOpts struct {
|
||||
startSUT func(tc *apptest.TestCase, vmstorageFlags, vmselectFlags []string) apptest.PrometheusWriteQuerier
|
||||
stopSUT func(tc *apptest.TestCase)
|
||||
}
|
||||
|
||||
func TestSingleSearchLimits_MetricNames(t *testing.T) {
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
testSearchLimits_MetricNames(tc, searchLimitsOpts{
|
||||
startSUT: searchLimitsStartVmsingle,
|
||||
stopSUT: searchLimitsStopVmsingle,
|
||||
})
|
||||
}
|
||||
|
||||
func TestClusterSearchLimits_MetricNames(t *testing.T) {
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
testSearchLimits_MetricNames(tc, searchLimitsOpts{
|
||||
startSUT: searchLimitsStartVmcluster,
|
||||
stopSUT: searchLimitsStopVmcluster,
|
||||
})
|
||||
}
|
||||
|
||||
func testSearchLimits_MetricNames(tc *apptest.TestCase, opts searchLimitsOpts) {
|
||||
const numMetrics = 100
|
||||
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
|
||||
end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
|
||||
data := apptest.GenerateTestData("metric", numMetrics, start, end)
|
||||
|
||||
sut := opts.startSUT(tc, nil, nil)
|
||||
sut.PrometheusAPIV1ImportPrometheus(tc.T(), data.Samples, apptest.QueryOpts{})
|
||||
sut.ForceFlush(tc.T())
|
||||
|
||||
// Check that with default search limits all metrics can be retrieved.
|
||||
apptest.AssertSeries(tc, sut, "metric.*", "", start, end, data.WantSeries)
|
||||
|
||||
// Restart cluster with explicit maxUniqueTimeseries on vmstorage side.
|
||||
opts.stopSUT(tc)
|
||||
sut = opts.startSUT(tc, []string{
|
||||
"-search.maxUniqueTimeseries=10",
|
||||
}, nil)
|
||||
|
||||
// Check that retrieving number of metrics that matches the
|
||||
// maxUniqueTimeseries limit is ok but retrieving more than that causes an
|
||||
// error.
|
||||
apptest.AssertSeries(tc, sut, "metric_000[0-9]{1}", "", start, end, data.WantSeries[0:10])
|
||||
apptest.AssertSeries(tc, sut, "metric.*", "", start, end, data.WantSeries) // Should fail but does not.
|
||||
}
|
||||
|
||||
func _TestSingleSearchLimits(t *testing.T) {
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
const numMetrics = 100
|
||||
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
|
||||
end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
|
||||
data := apptest.GenerateTestData("metric", numMetrics, start, end)
|
||||
|
||||
vmsingle := tc.MustStartVmsingle("vmsingle", []string{
|
||||
"-storageDataPath=" + tc.Dir() + "/vmsingle",
|
||||
"-retentionPeriod=100y",
|
||||
"-search.maxUniqueTimeseries=10",
|
||||
"-search.maxTagKeys=10",
|
||||
"-search.maxTagValues=10",
|
||||
"-search.maxTagValueSuffixesPerSearch=10",
|
||||
"-search.maxFederateSeries=10",
|
||||
"-search.maxExportSeries=10",
|
||||
"-search.maxTSDBStatusSeries=10",
|
||||
"-search.maxSeries=10",
|
||||
"-search.maxDeleteSeries=10",
|
||||
"-search.maxLabelsAPISeries=10",
|
||||
})
|
||||
vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data.Samples, apptest.QueryOpts{})
|
||||
vmsingle.ForceFlush(t)
|
||||
|
||||
apptest.AssertSeries(tc, vmsingle, "metric_00[0-9]{2}", "", start, end, data.WantSeries)
|
||||
return
|
||||
apptest.AssertSeriesCount(tc, vmsingle, "", start, end, numMetrics)
|
||||
apptest.AssertLabels(tc, vmsingle, "metric.*", "", start, end, data.WantLabels)
|
||||
apptest.AssertLabelValues(tc, vmsingle, "metric.*", "label", "", start, end, data.WantLabelValues)
|
||||
apptest.AssertQueryResults(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantQueryResults)
|
||||
apptest.AssertMetadata(tc, vmsingle, "", "", data.WantMetadata)
|
||||
}
|
||||
|
||||
@@ -118,6 +118,27 @@ To visualize and interact with both [self-monitoring metrics](https://docs.victo
|
||||
- {{% available_from "v1.26.0" anomaly %}} For rapid exploration of how different models, their configurations and included domain knowledge impacts the results of anomaly detection, use the built-in [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
|
||||

|
||||
|
||||
## Is vmanomaly stateful?
|
||||
By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
|
||||
|
||||
Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
|
||||
|
||||
## Config hot-reloading
|
||||
|
||||
`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
|
||||
|
||||
## Environment variables
|
||||
|
||||
`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
|
||||
|
||||
## Deploying vmanomaly
|
||||
|
||||
`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
|
||||
|
||||
## Migration
|
||||
|
||||
For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
|
||||
|
||||
## Choosing the right model for vmanomaly
|
||||
|
||||
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect:
|
||||
@@ -134,6 +155,16 @@ Please refer to [respective blogpost on anomaly types and alerting heuristics](h
|
||||
|
||||
Still not 100% sure what to use? We are [here to help](https://docs.victoriametrics.com/anomaly-detection/#get-in-touch).
|
||||
|
||||
## Can AI help configure vmanomaly?
|
||||
|
||||
Yes. The available tools serve different workflows:
|
||||
|
||||
- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
|
||||
- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
|
||||
- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
|
||||
|
||||
Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
|
||||
|
||||
## Incorporating domain knowledge
|
||||
|
||||
Anomaly detection models can significantly improve when incorporating business-specific assumptions about the data and what constitutes an anomaly. `vmanomaly` supports various [business-side configuration parameters](https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args) across all built-in models to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** and **align model behavior with business needs**, for example:
|
||||
@@ -191,15 +222,39 @@ models:
|
||||
provide_series: ['anomaly_score', 'y', 'yhat', 'yhat_lower', 'yhat_upper']
|
||||
```
|
||||
|
||||
## Can AI help configure vmanomaly?
|
||||
## Alert generation in vmanomaly
|
||||
While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
|
||||
|
||||
Yes. The available tools serve different workflows:
|
||||
<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
|
||||
|
||||
- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
|
||||
- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
|
||||
- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
|
||||
Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
|
||||
|
||||
Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
|
||||
```yaml
|
||||
groups:
|
||||
- name: VMAnomalyAlerts
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: HighAnomalyScore
|
||||
expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
|
||||
for: 5m # adjust to your needs based on data frequency and alerting policies
|
||||
labels:
|
||||
severity: warning
|
||||
query_alias: explore
|
||||
model_alias: default
|
||||
scheduler_alias: periodic
|
||||
preset: ui
|
||||
annotations:
|
||||
summary: High anomaly score detected.
|
||||
description: Anomaly score exceeded threshold ({{ $value }}) for more than
|
||||
{{ $for }} for query {{ $labels.for }}.
|
||||
```
|
||||
|
||||
> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
|
||||
|
||||
> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
|
||||
|
||||
## Preventing alert fatigue
|
||||
Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
|
||||
|
||||
## How to backtest particular configuration on historical data?
|
||||
|
||||
@@ -362,61 +417,6 @@ groups:
|
||||
description: "Disk usage is forecasted to exceed 95% in the next 3 days for instance {{ $labels.instance }}. Forecasted value: {{ $value }}."
|
||||
```
|
||||
|
||||
## Alert generation in vmanomaly
|
||||
While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
|
||||
|
||||
<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
|
||||
|
||||
Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: VMAnomalyAlerts
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: HighAnomalyScore
|
||||
expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
|
||||
for: 5m # adjust to your needs based on data frequency and alerting policies
|
||||
labels:
|
||||
severity: warning
|
||||
query_alias: explore
|
||||
model_alias: default
|
||||
scheduler_alias: periodic
|
||||
preset: ui
|
||||
annotations:
|
||||
summary: High anomaly score detected.
|
||||
description: Anomaly score exceeded threshold ({{ $value }}) for more than
|
||||
{{ $for }} for query {{ $labels.for }}.
|
||||
```
|
||||
|
||||
> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
|
||||
|
||||
> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
|
||||
|
||||
## Preventing alert fatigue
|
||||
Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
|
||||
|
||||
## Deploying vmanomaly
|
||||
|
||||
`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
|
||||
|
||||
## Environment variables
|
||||
|
||||
`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
|
||||
|
||||
## Is vmanomaly stateful?
|
||||
By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
|
||||
|
||||
Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
|
||||
|
||||
## Config hot-reloading
|
||||
|
||||
`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
|
||||
|
||||
## Migration
|
||||
|
||||
For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
|
||||
|
||||
## Resource consumption of vmanomaly
|
||||
`vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), and the complexity of the employed [models](https://docs.victoriametrics.com/anomaly-detection/components/models/). Its resource usage is directly related to these factors, making it adaptable to various operational scales. Various optimizations are available to balance between RAM usage, processing speed, and model capacity. These options are described in the sections below.
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ Key functions:
|
||||
|
||||
The diagram below illustrates how `vmanomaly` fits into an observability setup, such as detecting anomalies in metrics collected by `node_exporter`:
|
||||
|
||||
<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
|
||||
<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
|
||||
|
||||
## How does it work?
|
||||
|
||||
@@ -40,7 +40,7 @@ VictoriaMetrics Anomaly Detection **continuously re-fit and apply machine learni
|
||||
- **Confidence intervals** (`[yhat_lower, yhat_upper]`)
|
||||
These outputs integrate seamlessly into downstream applications, making it easier to **visually inspect anomalies**, e.g. in respective [Grafana dashboards](https://docs.victoriametrics.com/anomaly-detection/presets/#grafana-dashboard).
|
||||
|
||||
{{% content "components/vmanomaly-components-diagram.md" %}}
|
||||
<img src="https://docs.victoriametrics.com/anomaly-detection/components/vmanomaly-components.webp" alt="node_exporter_example_diagram" style="width:80%"/>
|
||||
|
||||
## Key benefits
|
||||
|
||||
|
||||
@@ -78,7 +78,9 @@ These [sub-configurations](#sub-configuration) can be assigned to a specific sha
|
||||
|
||||
Additionally, a replication factor `R ≥ 1` ensures [high availability](#high-availability) by enforcing redundancy across shards.
|
||||
|
||||
{{% content "vmanomaly-sharding-ha-diagram.md" %}}
|
||||
<p></p>
|
||||
|
||||

|
||||
|
||||
> Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
|
||||
|
||||
@@ -128,7 +130,9 @@ Similar to other VictoriaMetrics ecosystem components, like [VMAgent](https://do
|
||||
|
||||
When `VMANOMALY_REPLICATION_FACTOR` > 1, each [sub-config](#sub-configuration) `n` from `{0, N-1}` is assigned to exactly `R` nodes. This ensures redundancy, preventing single-node failures from causing data loss.
|
||||
|
||||
{{% content "vmanomaly-sharding-ha-diagram.md" %}}
|
||||
<p></p>
|
||||
|
||||

|
||||
|
||||
> Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ Below, you will find an example illustrating how the components of `vmanomaly` i
|
||||
|
||||
> [Reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [Writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) also support [multitenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy), so you can read/write from/to different locations - see `tenant_id` param description.
|
||||
|
||||
{{% content "vmanomaly-components-diagram.md" %}}
|
||||

|
||||
|
||||
## Example config
|
||||
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 968 420" role="img" aria-labelledby="title description">
|
||||
<title id="title">AutoTunedModel tuning and inference lifecycle</title>
|
||||
<desc id="description">The tuning process tests and scores model candidates across n time-series splits until the trial count or timeout is reached. Each fold fits a candidate on training data and predicts anomalies on its validation segment. The best model is then used on inference data until the next fit call.</desc>
|
||||
<defs>
|
||||
<marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
|
||||
<path d="M0 0 10 5 0 10Z" fill="#202124"/>
|
||||
</marker>
|
||||
<style>
|
||||
text { font-family: Arial, Helvetica, sans-serif; fill: #202124; font-size: 18px; }
|
||||
.line { fill: none; stroke: #202124; stroke-width: 2.5; }
|
||||
.arrow { fill: none; stroke: #202124; stroke-width: 2.5; marker-end: url(#arrow); }
|
||||
.dash { fill: none; stroke: #202124; stroke-width: 2.5; stroke-dasharray: 8 9; }
|
||||
.box { fill: #fff; stroke: #202124; stroke-width: 1.5; }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<rect width="968" height="420" fill="#fff"/>
|
||||
<rect x="112" y="10" width="242" height="83" class="box"/>
|
||||
<text x="233" y="38" text-anchor="middle">
|
||||
<tspan x="233" dy="0">test and score candidates</tspan>
|
||||
<tspan x="233" dy="21">until `n_trials` or `timeout`</tspan>
|
||||
<tspan x="233" dy="21">is reached</tspan>
|
||||
</text>
|
||||
<path d="M354 49 V67" class="arrow"/>
|
||||
<text x="355" y="90" text-anchor="middle">Tuning</text>
|
||||
<text x="355" y="112" text-anchor="middle">process</text>
|
||||
|
||||
<path d="M162 157 V136 H549 V157" class="line"/>
|
||||
<path d="M355 112 V136" class="line"/>
|
||||
<path d="M399 93 H653 V152" class="line"/>
|
||||
|
||||
<path d="M113 161 H75 V350 H113" class="line"/>
|
||||
<path d="M75 236 H39" class="line"/>
|
||||
<text x="28" y="250" transform="rotate(-90 28 250)" text-anchor="middle">n splits</text>
|
||||
|
||||
<text x="102" y="227">Fold 1</text>
|
||||
<text x="102" y="255">Fold 2</text>
|
||||
<text x="102" y="283">Fold 3</text>
|
||||
<text x="120" y="327">...</text>
|
||||
|
||||
<text x="260" y="179" text-anchor="middle">Fit</text>
|
||||
<text x="260" y="202" text-anchor="middle">candidate</text>
|
||||
<text x="413" y="179" text-anchor="middle">Predict</text>
|
||||
<text x="413" y="202" text-anchor="middle">anomalies</text>
|
||||
|
||||
<path d="M173 221 H354 V207" class="line"/>
|
||||
<path d="M232 249 H412 V235" class="line"/>
|
||||
<path d="M293 280 H472 V265" class="line"/>
|
||||
<path d="M354 221 H412" class="dash"/>
|
||||
<path d="M412 249 H474" class="dash"/>
|
||||
<path d="M472 280 H513" class="dash"/>
|
||||
|
||||
<path d="M548 144 V415" class="dash" style="stroke-width:1.5;stroke-dasharray:4 6"/>
|
||||
<rect x="577" y="152" width="136" height="61" rx="12" class="box"/>
|
||||
<text x="645" y="187" text-anchor="middle">best model</text>
|
||||
<path d="M653 213 V348" class="arrow"/>
|
||||
<text x="815" y="168" text-anchor="middle">
|
||||
<tspan x="815" dy="0">used to predict</tspan>
|
||||
<tspan x="815" dy="21">on inference data</tspan>
|
||||
<tspan x="815" dy="21">until the next `fit` call</tspan>
|
||||
</text>
|
||||
|
||||
<path d="M40 350 H895" class="arrow"/>
|
||||
<text x="455" y="402" text-anchor="middle">Training data</text>
|
||||
<text x="623" y="402" text-anchor="middle">Inference data</text>
|
||||
<text x="856" y="402">Time axis, t</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.3 KiB |
BIN
docs/anomaly-detection/components/autotune.webp
Normal file
|
After Width: | Height: | Size: 15 KiB |
@@ -1,139 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
|
||||
<title id="title">Multivariate models lifecycle</title>
|
||||
<desc id="description">MetricsQL queries retrieve an aligned set of series from VictoriaMetrics. Reader data fits one shared multivariate model. The exact same series set produces one anomaly score series with the intersected label set; inference is skipped when the fit and inference series sets differ. Writer stores produced scores in VictoriaMetrics.</desc>
|
||||
<defs>
|
||||
<marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
|
||||
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
<marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
|
||||
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
<style>
|
||||
text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
|
||||
.title { font-size: 50px; font-weight: 400; }
|
||||
.node { font-size: 34px; font-weight: 400; }
|
||||
.label { font-size: 29px; }
|
||||
.blue-label { font-size: 26px; }
|
||||
.small { font-size: 25px; }
|
||||
.box { fill: #fff; stroke: #303038; stroke-width: 3; }
|
||||
.group { fill: #fff; stroke: #303038; stroke-width: 3; }
|
||||
.model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
|
||||
.line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
|
||||
.blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
|
||||
.blue { fill: #1478c9; }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<rect width="1920" height="1295" fill="#fff"/>
|
||||
<text x="20" y="70" class="title">Multivariate Models Lifecycle</text>
|
||||
|
||||
<!-- VictoriaMetrics and query configuration -->
|
||||
<rect x="535" y="175" width="420" height="215" class="box"/>
|
||||
<text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
|
||||
<text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
|
||||
<rect x="270" y="270" width="150" height="155" class="box"/>
|
||||
<text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
|
||||
<text x="285" y="360" class="label">MetricsQL</text>
|
||||
<text x="285" y="395" class="label">queries</text>
|
||||
<path d="M420 346 H535" class="line"/>
|
||||
|
||||
<!-- Reader and datasource exchange -->
|
||||
<rect x="580" y="500" width="310" height="100" class="box"/>
|
||||
<text x="735" y="562" class="node" text-anchor="middle">Reader</text>
|
||||
<path d="M580 545 H455 V365 H535" class="line"/>
|
||||
<text x="465" y="478" class="label">1. Request data</text>
|
||||
<path d="M955 270 H1040 V550 H890" class="line"/>
|
||||
<text x="810" y="478" class="label">2. Get metrics</text>
|
||||
|
||||
<!-- Historical fit data returned by the configured queries -->
|
||||
<g aria-label="Historical fit data">
|
||||
<rect x="1090" y="85" width="430" height="505" class="group"/>
|
||||
<rect x="1120" y="135" width="340" height="170" class="group"/>
|
||||
<text x="1140" y="180" class="node">Query 1</text>
|
||||
<rect x="1135" y="195" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="228" class="label">Metric 1.1</text>
|
||||
<text x="1150" y="262" class="label">...</text>
|
||||
<rect x="1135" y="265" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="298" class="label">Metric 1.M₁</text>
|
||||
<text x="1305" y="355" class="node" text-anchor="middle">...</text>
|
||||
<rect x="1120" y="390" width="340" height="170" class="group"/>
|
||||
<text x="1140" y="435" class="node">Query N</text>
|
||||
<rect x="1135" y="450" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="483" class="label">Metric N.1</text>
|
||||
<text x="1150" y="517" class="label">...</text>
|
||||
<rect x="1135" y="520" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="553" class="label">Metric N.Mₙ</text>
|
||||
</g>
|
||||
|
||||
<!-- One shared model is fitted on the complete aligned set -->
|
||||
<rect x="1215" y="795" width="310" height="115" class="box"/>
|
||||
<text x="1370" y="865" class="node" text-anchor="middle">Model</text>
|
||||
<rect x="1365" y="635" width="155" height="130" class="box"/>
|
||||
<text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
|
||||
<text x="1382" y="723" class="label">Model</text>
|
||||
<text x="1382" y="757" class="label">config</text>
|
||||
<path d="M1305 590 V795" class="line"/>
|
||||
<text x="1055" y="672" class="label">
|
||||
<tspan x="1055" dy="0">3. Fit one model</tspan>
|
||||
<tspan x="1055" dy="38">on all historical series</tspan>
|
||||
</text>
|
||||
<path d="M1443 765 V795" class="line"/>
|
||||
|
||||
<!-- Inference data must contain the same set of series -->
|
||||
<g aria-label="Inference data">
|
||||
<rect x="25" y="615" width="430" height="500" class="group"/>
|
||||
<rect x="55" y="645" width="340" height="170" class="group"/>
|
||||
<text x="75" y="690" class="node">Query 1</text>
|
||||
<rect x="70" y="705" width="300" height="42" class="box"/>
|
||||
<text x="85" y="738" class="label">Metric 1.1</text>
|
||||
<text x="85" y="772" class="label">...</text>
|
||||
<rect x="70" y="775" width="300" height="42" class="box"/>
|
||||
<text x="85" y="808" class="label">Metric 1.M₁</text>
|
||||
<text x="225" y="870" class="node" text-anchor="middle">...</text>
|
||||
<rect x="55" y="900" width="340" height="175" class="group"/>
|
||||
<text x="75" y="945" class="node">Query N</text>
|
||||
<rect x="70" y="960" width="300" height="42" class="box"/>
|
||||
<text x="85" y="993" class="label">Metric N.1</text>
|
||||
<text x="85" y="1027" class="label">...</text>
|
||||
<rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
|
||||
<text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
|
||||
</g>
|
||||
<path d="M580 575 H500 V650 H455" class="line"/>
|
||||
<text x="465" y="680" class="label">4. Provide inference data</text>
|
||||
|
||||
<!-- Single multivariate model registry -->
|
||||
<g aria-label="Multivariate model registry">
|
||||
<rect x="580" y="750" width="420" height="480" class="group"/>
|
||||
<text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
|
||||
<rect x="610" y="835" width="360" height="220" class="model-group"/>
|
||||
<rect x="640" y="885" width="300" height="115" class="box"/>
|
||||
<text x="790" y="955" class="node" text-anchor="middle">Model (single)</text>
|
||||
</g>
|
||||
<path d="M455 1050 H580" class="line"/>
|
||||
<path d="M1215 852 H1000" class="line"/>
|
||||
|
||||
<!-- Joint output and mismatched-series skip path -->
|
||||
<rect x="1600" y="1015" width="285" height="105" class="box"/>
|
||||
<text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
|
||||
<rect x="1740" y="805" width="145" height="145" class="box"/>
|
||||
<text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
|
||||
<text x="1758" y="900" class="label">Writer</text>
|
||||
<text x="1758" y="935" class="label">config</text>
|
||||
<path d="M1813 950 V1015" class="line"/>
|
||||
<path d="M1000 1040 H1600" class="line"/>
|
||||
<text x="1295" y="965" class="label" text-anchor="middle">
|
||||
<tspan x="1295" dy="0">5.a Produce one anomaly-score series</tspan>
|
||||
<tspan x="1295" dy="38">with the intersected label set</tspan>
|
||||
</text>
|
||||
<path d="M1000 1110 H1600" class="blue-line"/>
|
||||
<text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
|
||||
<tspan x="1320" dy="0">5.b Skip inference when the fit and inference</tspan>
|
||||
<tspan x="1320" dy="32">series sets differ;</tspan>
|
||||
<tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
|
||||
</text>
|
||||
|
||||
<!-- Persist the joint anomaly score -->
|
||||
<path d="M1743 1015 V80 H745 V175" class="line"/>
|
||||
<text x="1170" y="55" class="label" text-anchor="middle">6. Write one anomaly-score series (label set = intersection)</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 7.8 KiB |
|
After Width: | Height: | Size: 66 KiB |
@@ -1,148 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
|
||||
<title id="title">Univariate models lifecycle</title>
|
||||
<desc id="description">MetricsQL queries retrieve multiple series from VictoriaMetrics. Reader data fits one model per series in the model registry. Known series produce individual anomaly scores for Writer; an unseen series is skipped until a fitted model exists. Writer stores produced scores in VictoriaMetrics.</desc>
|
||||
<defs>
|
||||
<marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
|
||||
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
<marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
|
||||
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
<style>
|
||||
text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
|
||||
.title { font-size: 50px; font-weight: 400; }
|
||||
.node { font-size: 34px; font-weight: 400; }
|
||||
.label { font-size: 29px; }
|
||||
.blue-label { font-size: 26px; }
|
||||
.small { font-size: 25px; }
|
||||
.box { fill: #fff; stroke: #303038; stroke-width: 3; }
|
||||
.group { fill: #fff; stroke: #303038; stroke-width: 3; }
|
||||
.model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
|
||||
.line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
|
||||
.blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
|
||||
.blue { fill: #1478c9; }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<rect width="1920" height="1295" fill="#fff"/>
|
||||
<text x="20" y="70" class="title">Univariate Models Lifecycle</text>
|
||||
|
||||
<!-- VictoriaMetrics and query configuration -->
|
||||
<rect x="535" y="175" width="420" height="215" class="box"/>
|
||||
<text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
|
||||
<text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
|
||||
<rect x="270" y="270" width="150" height="155" class="box"/>
|
||||
<text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
|
||||
<text x="285" y="360" class="label">MetricsQL</text>
|
||||
<text x="285" y="395" class="label">queries</text>
|
||||
<path d="M420 346 H535" class="line"/>
|
||||
|
||||
<!-- Reader and datasource exchange -->
|
||||
<rect x="580" y="500" width="310" height="100" class="box"/>
|
||||
<text x="735" y="562" class="node" text-anchor="middle">Reader</text>
|
||||
<path d="M580 545 H455 V365 H535" class="line"/>
|
||||
<text x="465" y="478" class="label">1. Request data</text>
|
||||
<path d="M955 270 H1040 V550 H890" class="line"/>
|
||||
<text x="810" y="478" class="label">2. Get metrics</text>
|
||||
|
||||
<!-- Historical fit data returned by the configured queries -->
|
||||
<g aria-label="Historical fit data">
|
||||
<rect x="1090" y="85" width="430" height="505" class="group"/>
|
||||
<rect x="1120" y="135" width="340" height="170" class="group"/>
|
||||
<text x="1140" y="180" class="node">Query 1</text>
|
||||
<rect x="1135" y="195" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="228" class="label">Metric 1.1</text>
|
||||
<text x="1150" y="262" class="label">...</text>
|
||||
<rect x="1135" y="265" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="298" class="label">Metric 1.M₁</text>
|
||||
<text x="1305" y="355" class="node" text-anchor="middle">...</text>
|
||||
<rect x="1120" y="390" width="340" height="170" class="group"/>
|
||||
<text x="1140" y="435" class="node">Query N</text>
|
||||
<rect x="1135" y="450" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="483" class="label">Metric N.1</text>
|
||||
<text x="1150" y="517" class="label">...</text>
|
||||
<rect x="1135" y="520" width="300" height="42" class="box"/>
|
||||
<text x="1150" y="553" class="label">Metric N.Mₙ</text>
|
||||
</g>
|
||||
|
||||
<!-- Model fitting -->
|
||||
<rect x="1215" y="795" width="310" height="115" class="box"/>
|
||||
<text x="1370" y="865" class="node" text-anchor="middle">Model</text>
|
||||
<rect x="1365" y="635" width="155" height="130" class="box"/>
|
||||
<text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
|
||||
<text x="1382" y="723" class="label">Model</text>
|
||||
<text x="1382" y="757" class="label">config</text>
|
||||
<path d="M1305 590 V795" class="line"/>
|
||||
<text x="1055" y="672" class="label">
|
||||
<tspan x="1055" dy="0">3. Fit models</tspan>
|
||||
<tspan x="1055" dy="38">on historical data</tspan>
|
||||
</text>
|
||||
<path d="M1443 765 V795" class="line"/>
|
||||
|
||||
<!-- Inference data, including a new unseen series -->
|
||||
<g aria-label="Inference data">
|
||||
<rect x="25" y="615" width="430" height="500" class="group"/>
|
||||
<rect x="55" y="645" width="340" height="170" class="group"/>
|
||||
<text x="75" y="690" class="node">Query 1 (has fit model)</text>
|
||||
<rect x="70" y="705" width="300" height="42" class="box"/>
|
||||
<text x="85" y="738" class="label">Metric 1.1</text>
|
||||
<text x="85" y="772" class="label">...</text>
|
||||
<rect x="70" y="775" width="300" height="42" class="box"/>
|
||||
<text x="85" y="808" class="label">Metric 1.M₁</text>
|
||||
<text x="225" y="870" class="node" text-anchor="middle">...</text>
|
||||
<rect x="55" y="900" width="340" height="175" class="group"/>
|
||||
<text x="75" y="945" class="node">Query N</text>
|
||||
<rect x="70" y="960" width="300" height="42" class="box"/>
|
||||
<text x="85" y="993" class="label">Metric N.1</text>
|
||||
<text x="85" y="1027" class="label">...</text>
|
||||
<rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
|
||||
<text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
|
||||
</g>
|
||||
<path d="M580 575 H500 V650 H455" class="line"/>
|
||||
<text x="465" y="680" class="label">4. Provide inference data</text>
|
||||
|
||||
<!-- One fitted model per known series -->
|
||||
<g aria-label="Univariate model registry">
|
||||
<rect x="580" y="750" width="420" height="480" class="group"/>
|
||||
<text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
|
||||
<rect x="610" y="830" width="360" height="170" class="model-group"/>
|
||||
<rect x="630" y="865" width="300" height="42" class="box"/>
|
||||
<text x="645" y="898" class="label">Model 1.1</text>
|
||||
<text x="645" y="932" class="label">...</text>
|
||||
<rect x="630" y="935" width="300" height="42" class="box"/>
|
||||
<text x="645" y="968" class="label">Model 1.M₁</text>
|
||||
<text x="790" y="1045" class="node" text-anchor="middle">...</text>
|
||||
<rect x="610" y="1070" width="360" height="135" class="model-group"/>
|
||||
<rect x="630" y="1090" width="300" height="42" class="box"/>
|
||||
<text x="645" y="1123" class="label">Model N.1</text>
|
||||
<rect x="630" y="1145" width="300" height="42" class="box"/>
|
||||
<text x="645" y="1178" class="label">Model N.Mₙ</text>
|
||||
</g>
|
||||
<path d="M455 1050 H580" class="line"/>
|
||||
<path d="M1215 852 H1000" class="line"/>
|
||||
|
||||
<!-- Known-series output and unseen-series skip path -->
|
||||
<rect x="1600" y="1015" width="285" height="105" class="box"/>
|
||||
<text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
|
||||
<rect x="1740" y="805" width="145" height="145" class="box"/>
|
||||
<text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
|
||||
<text x="1758" y="900" class="label">Writer</text>
|
||||
<text x="1758" y="935" class="label">config</text>
|
||||
<path d="M1813 950 V1015" class="line"/>
|
||||
<path d="M1000 1040 H1600" class="line"/>
|
||||
<text x="1275" y="975" class="label" text-anchor="middle">
|
||||
<tspan x="1275" dy="0">5.a Produce anomaly scores</tspan>
|
||||
<tspan x="1275" dy="38">for known series</tspan>
|
||||
</text>
|
||||
<path d="M1000 1110 H1600" class="blue-line"/>
|
||||
<text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
|
||||
<tspan x="1320" dy="0">5.b Skip inference until a fitted model exists</tspan>
|
||||
<tspan x="1320" dy="32">for Metric N.Mₖ;</tspan>
|
||||
<tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
|
||||
</text>
|
||||
|
||||
<!-- Persist anomaly scores -->
|
||||
<path d="M1743 1015 V80 H745 V175" class="line"/>
|
||||
<text x="1130" y="55" class="label" text-anchor="middle">6. Write back produced anomaly scores</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 8.2 KiB |
|
After Width: | Height: | Size: 66 KiB |
@@ -68,10 +68,6 @@ models:
|
||||
|
||||
Common arguments supported by every model were introduced in [v1.10.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1100).
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="Queries" %}}
|
||||
|
||||
### Queries
|
||||
|
||||
The `queries` argument selects the [reader queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) used to fit and run a particular model{{% available_from "v1.10.0" anomaly %}}. Every series returned by a selected query is passed to that model.
|
||||
@@ -97,10 +93,6 @@ models:
|
||||
queries: ['q1', 'q2', 'q3'] # i.e., if your `queries` in `reader` section has exactly q1, q2, q3 aliases
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Schedulers" %}}
|
||||
|
||||
### Schedulers
|
||||
|
||||
The `schedulers` argument selects the [schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) that run a particular model{{% available_from "v1.11.0" anomaly %}}.
|
||||
@@ -126,10 +118,6 @@ models:
|
||||
schedulers: ['s1', 's2', 's3'] # i.e., if your `schedulers` section has exactly s1, s2, s3 aliases
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Provide series" %}}
|
||||
|
||||
### Provide series
|
||||
|
||||
The `provide_series` argument{{% available_from "v1.12.0" anomaly %}} limits the [model output](#vmanomaly-output) sent to the writer. For example, a model may produce `['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']` by default, while the following configuration writes only `anomaly_score` for each input series:
|
||||
@@ -143,10 +131,6 @@ models:
|
||||
|
||||
> If `provide_series` is not specified in model config, the model will produce its default [model-dependent output](#vmanomaly-output). The output can't be less than `['anomaly_score']`. Even if `timestamp` column is omitted, it will be implicitly added to `provide_series` list, as it's required for metrics to be properly written.
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Detection direction" %}}
|
||||
|
||||
### Detection direction
|
||||
The `detection_direction` argument{{% available_from "v1.13.0" anomaly %}} can reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive) when domain knowledge indicates that only values above or below the expected value are anomalous. Available values are `both`, `above_expected`, and `below_expected`.
|
||||
|
||||
@@ -206,10 +190,6 @@ reader:
|
||||
# other components like writer, schedule, monitoring
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Minimal deviation from expected" %}}
|
||||
|
||||
### Minimal deviation from expected
|
||||
|
||||
`min_dev_from_expected`{{% available_from "v1.13.0" anomaly %}} argument is designed to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** in scenarios where deviations between the actual value (`y`) and the expected value (`yhat`) are **relatively** high. Such deviations can cause models to generate high [anomaly scores](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score). However, these deviations may not be significant enough in **absolute values** from a business perspective to be considered anomalies. This parameter ensures that anomaly scores for data points where `|y - yhat| < min_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it is set to `0` to maintain backward compatibility.
|
||||
@@ -258,10 +238,6 @@ models:
|
||||
queries: ['normal_behavior'] # use the default where it's not needed
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Minimal relative deviation from expected" %}}
|
||||
|
||||
### Minimal relative deviation from expected
|
||||
|
||||
{{% available_from "v1.29.1" anomaly %}} `min_rel_dev_from_expected` argument serves a similar purpose to `min_dev_from_expected` (see [section above](#minimal-deviation-from-expected)), but focuses on **relative deviations** rather than absolute ones. It is designed to reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive) in scenarios where the relative deviation between the actual value (`y`) and the expected value (`yhat`) is high, but the absolute deviation is not significant enough to be considered an anomaly from a business perspective. This parameter ensures that anomaly scores for data points where `|y - yhat| / |yhat| < min_rel_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it is set to `0` to maintain backward compatibility.
|
||||
@@ -302,10 +278,6 @@ models:
|
||||
```
|
||||
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Group by" %}}
|
||||
|
||||
### Group by
|
||||
|
||||
> The `groupby` argument works only in combination with [multivariate models](#multivariate-models).
|
||||
@@ -344,10 +316,6 @@ models:
|
||||
groupby: [host]
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Scale" %}}
|
||||
|
||||
### Scale
|
||||
|
||||
Previously available only to [ProphetModel](#prophet) and [OnlineQuantileModel](#online-seasonal-quantile), the `scale` {{% available_from "v1.20.0" anomaly %}} parameter is now applicable to all models that support generating predictions (`yhat`, `yhat_lower`, `yhat_upper`). Also, it is **two-sided** now, represented as a list of two positive float values, allowing separate scaling for the intervals `[yhat, yhat_upper]` and `[yhat_lower, yhat]`. The new margins are calculated as:
|
||||
@@ -378,10 +346,6 @@ models:
|
||||
scale: [1.2, 0.75]
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Clip predictions" %}}
|
||||
|
||||
### Clip predictions
|
||||
|
||||
A post-processing step to **clip model predictions** (`yhat`, `yhat_lower`, and `yhat_upper` series) to the configured [`data_range` values](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) in `VmReader` is available.
|
||||
@@ -436,10 +400,6 @@ models:
|
||||
]
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Score outside data range" %}}
|
||||
|
||||
### Score outside data range
|
||||
|
||||
The `anomaly_score_outside_data_range` {{% available_from "v1.20.0" anomaly %}} parameter allows overriding the default **anomaly score (`1.01`)** assigned when actual values (`y`) fall **outside the defined `data_range` if defined in [reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/)**. This provides greater flexibility for **alerting rule configurations** and enables **clearer visual differentiation** between different types of anomalies:
|
||||
@@ -485,10 +445,6 @@ models:
|
||||
anomaly_score_outside_data_range: 3.0
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Decay" %}}
|
||||
|
||||
### Decay
|
||||
|
||||
> The `decay` argument works only in combination with [online models](#online-models) like [`ZScoreOnlineModel`](#online-z-score) or [`OnlineQuantileModel`](#online-seasonal-quantile).
|
||||
@@ -521,10 +477,6 @@ models:
|
||||
queries: ['q1']
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
## Model types
|
||||
|
||||
@@ -550,7 +502,7 @@ If during an inference, you got a series having **new labelset** (not present in
|
||||
|
||||
**Examples:** [Prophet](#prophet), [Holt-Winters](#holt-winters)
|
||||
|
||||

|
||||

|
||||
|
||||
|
||||
### Multivariate Models
|
||||
@@ -565,34 +517,9 @@ If during an inference, you got a **different amount of series** or some series
|
||||
|
||||
**Implications:** Multivariate models are a go-to default, when your queries returns **fixed** amount of **individual** time series (say, some aggregations), to be used for adding cross-series (and cross-query) context, useful for catching [collective anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#collective-anomalies) or [novelties](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#novelties) (expanded to multi-input scenario). For example, you may set it up for anomaly detection of CPU usage in different modes (`idle`, `user`, `system`, etc.) and use its cross-dependencies to detect **unseen (in fit data)** behavior.
|
||||
|
||||
**Examples:** [Temporal Envelope](#temporal-envelope), [Isolation Forest](#isolation-forest-multivariate)
|
||||
**Examples:** [IsolationForest](#isolation-forest-multivariate)
|
||||
|
||||

|
||||
|
||||
The following configuration applies both models to the same aligned input series. Start with Temporal Envelope when temporal profiles and online adaptation matter; use Isolation Forest as an offline alternative when feature-space outliers are the primary concern.
|
||||
|
||||
```yaml
|
||||
models:
|
||||
service_dependency_envelope:
|
||||
class: temporal_envelope_multivariate
|
||||
queries: [request_rate, error_rate, latency]
|
||||
groupby: [cluster]
|
||||
dependency_rank: 8
|
||||
score_aggregation: l2
|
||||
seasonalities: [hod_smooth, dow_smooth]
|
||||
provide_series: [anomaly_score]
|
||||
|
||||
service_dependency_isolation_forest:
|
||||
class: isolation_forest_multivariate
|
||||
queries: [request_rate, error_rate, latency]
|
||||
groupby: [cluster]
|
||||
contamination: 0.01
|
||||
seasonal_features: [hod, dow]
|
||||
args:
|
||||
n_estimators: 100
|
||||
random_state: 42
|
||||
provide_series: [anomaly_score]
|
||||
```
|
||||

|
||||
|
||||
|
||||
### Online Models
|
||||
@@ -737,7 +664,7 @@ models:
|
||||
|
||||
</div>
|
||||
|
||||

|
||||

|
||||
|
||||
#### Shared asynchronous autotune workflow
|
||||
|
||||
@@ -1417,10 +1344,6 @@ This guide shows how to:
|
||||
|
||||
> The file containing the model must be written in [Python](https://www.python.org/) 3.14 or later. A custom model runs inside the `vmanomaly` Python environment, so keep its dependencies compatible with the target image and keep the module available when restoring serialized model state after a restart.
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="Custom model implementation guide" %}}
|
||||
|
||||
### 1. Custom model
|
||||
|
||||
Create `custom_model.py` with a `CustomModel` class derived from `Model`. A concrete model must implement:
|
||||
@@ -1587,10 +1510,6 @@ The writer emits one `custom_anomaly_score` series for each input series. It ret
|
||||
{__name__="custom_anomaly_score", for="churn_rate", model_alias="custom_model", scheduler_alias="s1", run="test-format"}
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
## Deprecations
|
||||
|
||||
{{% collapse name="Deprecated model types and models" %}}
|
||||
|
||||
@@ -24,10 +24,6 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
|
||||
- Adding `preset` and `scheduler_alias` keys to [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics) and [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics) metrics for consistency in multi-[scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) setups.
|
||||
- Renaming [Counters](https://prometheus.io/docs/concepts/metric_types/#counter) `vmanomaly_reader_response_count` to `vmanomaly_reader_responses` and `vmanomaly_writer_response_count` to `vmanomaly_writer_responses`.
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="Pull model config parameters" %}}
|
||||
|
||||
## Pull Model Config parameters
|
||||
|
||||
<table class="params">
|
||||
@@ -64,10 +60,6 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Push config parameters" %}}
|
||||
|
||||
## Push Config parameters
|
||||
|
||||
By default, metrics are pushed only after the completion of specific stages, e.g., `fit`, `infer`, or `fit_infer` (for each [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) if using a multi-scheduler configuration).
|
||||
@@ -235,10 +227,6 @@ Path to a file with the client certificate key, i.e. `client.key`{{% available_f
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
## Monitoring section config example
|
||||
|
||||
``` yaml
|
||||
@@ -272,10 +260,6 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
|
||||
- [Model metrics](#models-behaviour-metrics)
|
||||
- [Writer metrics](#writer-behaviour-metrics)
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="Startup metrics" %}}
|
||||
|
||||
### Startup metrics
|
||||
|
||||
<table class="params">
|
||||
@@ -401,10 +385,6 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
|
||||
|
||||
[Back to metric sections](#metrics-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Reader behaviour metrics" %}}
|
||||
|
||||
### Reader behaviour metrics
|
||||
Label names [description](#labelnames)
|
||||
|
||||
@@ -529,10 +509,6 @@ Label names [description](#labelnames)
|
||||
|
||||
[Back to metric sections](#metrics-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Models behaviour metrics" %}}
|
||||
|
||||
### Models behaviour metrics
|
||||
Label names [description](#labelnames)
|
||||
|
||||
@@ -659,10 +635,6 @@ Label names [description](#labelnames)
|
||||
|
||||
[Back to metric sections](#metrics-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Writer behaviour metrics" %}}
|
||||
|
||||
### Writer behaviour metrics
|
||||
Label names [description](#labelnames)
|
||||
|
||||
@@ -774,10 +746,6 @@ Label names [description](#labelnames)
|
||||
|
||||
[Back to metric sections](#metrics-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
### Labelnames
|
||||
|
||||
* `stage` - model execution stage: `fit`, `infer`, or `fit_infer` for a combined fit/inference scheduler run. See [model types](https://docs.victoriametrics.com/anomaly-detection/components/models/#model-types).
|
||||
@@ -825,10 +793,6 @@ and the [command-line arguments](https://docs.victoriametrics.com/anomaly-detect
|
||||
- [Query server and task logs](#query-server-and-task-logs)
|
||||
- [AI Copilot logs](#ai-copilot-logs)
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
|
||||
{{% collapse name="Startup logs" %}}
|
||||
|
||||
### Startup logs
|
||||
|
||||
@@ -848,10 +812,6 @@ server addresses, hot-reload state, and active schedulers. The most useful prefi
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Reader logs" %}}
|
||||
|
||||
### Reader logs
|
||||
|
||||
Reader logs cover endpoint checks, request splitting, network failures, response parsing, and coordination between
|
||||
@@ -903,10 +863,6 @@ or parsed. See [reader behaviour metrics](#reader-behaviour-metrics).
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Service logs" %}}
|
||||
|
||||
### Service logs
|
||||
|
||||
The service logs `fit`, `infer`, and combined `fit_infer`/backtesting work for each model alias and scheduler.
|
||||
@@ -935,10 +891,6 @@ an unsuccessful stage. See [models behaviour metrics](#models-behaviour-metrics)
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Writer logs" %}}
|
||||
|
||||
### Writer logs
|
||||
|
||||
Writer logs cover serialization and delivery of produced series such as
|
||||
@@ -966,10 +918,6 @@ and datapoints are recorded only after a successful response. See [writer behavi
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Scheduler supervision logs" %}}
|
||||
|
||||
### Scheduler supervision logs
|
||||
|
||||
Scheduler supervision{{% available_from "v1.30.0" anomaly %}} logs a dead worker, automatic restart, successful
|
||||
@@ -979,10 +927,6 @@ Correlate them with `vmanomaly_scheduler_alive` and `vmanomaly_scheduler_restart
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Hot-reload logs" %}}
|
||||
|
||||
### Hot-reload logs
|
||||
|
||||
Hot reload logs config-change detection, validation, staged service restart, success, and rollback. `Reload aborted
|
||||
@@ -992,10 +936,6 @@ without restarting services`.
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Persisted-state logs" %}}
|
||||
|
||||
### Persisted-state logs
|
||||
|
||||
With `settings.restore_state`, startup logs the stored/runtime version assessment, reusable components, required
|
||||
@@ -1004,10 +944,6 @@ stored artifacts completely` indicates a full reset; missing or unreadable model
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="Query server and task logs" %}}
|
||||
|
||||
### Query server and task logs
|
||||
|
||||
The query server logs its listening address and datasource-proxy timeouts/failures. Background anomaly-detection
|
||||
@@ -1016,10 +952,6 @@ background raw query finishing cleanly.
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="AI Copilot logs" %}}
|
||||
|
||||
### AI Copilot logs
|
||||
|
||||
AI Copilot{{% available_from "v1.30.0" anomaly %}} reports whether it is initialized, disabled, misconfigured, or
|
||||
@@ -1028,7 +960,3 @@ request failed` identifies provider execution failure, and `MCP server unreachab
|
||||
guidance tools.
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
@@ -25,8 +25,6 @@ Use the following playgrounds to develop and test input queries:
|
||||
|
||||
## VM reader
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="Queries format migration (to v1.13.0+)" %}}
|
||||
|
||||
> The backward-compatible `queries` format introduced in v1.13.0 allows [VmReader](#vm-reader) parameters such as `step` to be configured per query. This can reduce the amount of data read from VictoriaMetrics. See [per-query parameters](#per-query-parameters) for details.
|
||||
@@ -63,8 +61,6 @@ reader:
|
||||
```
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="VM reader per-query parameters and example" %}}
|
||||
|
||||
### Per-query parameters
|
||||
|
||||
There is change {{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg format. Now each query alias supports the next (sub)fields, which *override reader-level parameters*, if set:
|
||||
@@ -129,10 +125,6 @@ reader:
|
||||
offset: '-15s' # to override reader-wise `offset` and query data 15 seconds earlier to account for data collection delays
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="VM reader config parameters and example" %}}
|
||||
|
||||
### Config parameters
|
||||
|
||||
<table class="params">
|
||||
@@ -515,16 +507,10 @@ reader:
|
||||
series_processing_batch_size: 8
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
### MetricsQL Playground
|
||||
|
||||
To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMetrics MetricsQL Playground](https://play.victoriametrics.com/), which provides an interactive environment to test and visualize your queries against sample data. You can also access embedded version of the playground below:
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="VictoriaMetrics Playground" %}}
|
||||
|
||||
<div class="position-relative mb-3">
|
||||
@@ -550,8 +536,6 @@ To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMe
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
### mTLS protection
|
||||
|
||||
`vmanomaly` supports [mutual TLS (mTLS)](https://en.wikipedia.org/wiki/Mutual_authentication){{% available_from "v1.16.3" anomaly %}} for secure communication across its components, including [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters). This allows for mutual authentication between the client and server when querying or writing data to [VictoriaMetrics Enterprise, configured for mTLS](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#mtls-protection).
|
||||
@@ -696,8 +680,6 @@ Similarly, [VictoriaTraces LogsQL Playground](https://play-vtraces.victoriametri
|
||||
|
||||
You can also access **embedded version of the playground below** (VictoriaLogs datasource):
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="VictoriaLogs LogsQL Playground" %}}
|
||||
|
||||
<div class="position-relative mb-3">
|
||||
@@ -723,11 +705,6 @@ You can also access **embedded version of the playground below** (VictoriaLogs d
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="VictoriaLogs reader config parameters" %}}
|
||||
|
||||
### Config parameters
|
||||
|
||||
@@ -1012,10 +989,6 @@ Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far last-seen
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="VictoriaLogs reader per-query parameters and example" %}}
|
||||
|
||||
### Per-query parameters
|
||||
|
||||
The names, types and the logic of the per-query parameters subset used in `VLogsReader` are exactly the same as those of [`VmReader`](#vm-reader), please see [per-query parameters](#per-query-parameters) section above for the details. The only difference is that `expr` parameter should contain a valid [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) expression with `stats` [pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe), as described in [query examples](#query-examples) section above.
|
||||
@@ -1063,10 +1036,6 @@ reader:
|
||||
# other config sections, like models, schedulers, writer, ...
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
### mTLS protection
|
||||
|
||||
Please refer to the [mTLS protection](#mtls-protection) section above for details on how to configure mTLS for `VLogsReader`. It uses the same config parameters as `VmReader` for mTLS setup.
|
||||
|
||||
@@ -226,10 +226,6 @@ monitoring:
|
||||
# other monitoring settings
|
||||
```
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="State restoration example" %}}
|
||||
|
||||
### Example
|
||||
|
||||
For a configuration with the following models, queries and schedulers:
|
||||
@@ -309,10 +305,6 @@ This means that the service upon restart:
|
||||
1. Won't restore the state of `zscore_online` model, because its `z_threshold` argument **has changed**, retraining from scratch is needed on the last `fit_window` = 24 hours of data for `q1`, `q2` and `q3` (as model's `queries` arg is not set so it defaults to all queries found in the reader).
|
||||
2. Will **partially** restore the state of `prophet` model, because its class and schedulers are unchanged, but **only instances trained on timeseries returned by `q1` query**. New fit/infer jobs will be set for new query `q3`. The old query `q2` artifacts will be dropped upon restart - all respective models and data for (`prophet`, `q2`) combination will be removed from the database file and from the disk.
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
## Retention
|
||||
|
||||
{{% available_from "v1.28.1" anomaly %}} The `retention` argument sets a [time to live](https://en.wikipedia.org/wiki/Time_to_live) (TTL) for service artifacts such as stored model instances and training data. At each `check_interval`, the service removes artifacts that have not been used for inference or refitting within `ttl`. This bounds stale resource usage in long-running deployments.
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
---
|
||||
build:
|
||||
list: never
|
||||
publishResources: false
|
||||
render: never
|
||||
sitemap:
|
||||
disable: true
|
||||
---
|
||||
|
||||
The required path is `config.yml` → Scheduler → Reader → Model → Writer. The Reader queries the configured VictoriaMetrics, VictoriaLogs, or VictoriaTraces datasource; the Writer stores inferred anomaly scores in VictoriaMetrics. Monitoring is optional and can push metrics or expose them for collection.
|
||||
|
||||
Solid nodes and arrows show the required anomaly-detection path. Dashed nodes and arrows show optional self-monitoring integrations.
|
||||
|
||||

|
||||
{style="display:block; width:80%; min-width:320px; margin:1.5rem auto"}
|
||||
@@ -1,152 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1600" role="img" aria-labelledby="title description">
|
||||
<title id="title">How vmanomaly operates during a scheduled iteration</title>
|
||||
<desc id="description">The required flow starts from config.yml and the Scheduler. The Reader queries either VictoriaMetrics through query_range or VictoriaLogs and VictoriaTraces through stats_query_range, then sends data to a Model. The Model produces anomaly scores, and the Writer stores them in VictoriaMetrics through import. Reader, Model, and Writer can optionally report self-monitoring metrics using push or pull monitoring.</desc>
|
||||
<defs>
|
||||
<marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
|
||||
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
<marker id="arrow-optional" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
|
||||
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#666a73" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
<style>
|
||||
.label { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
|
||||
.title { font-size: 54px; font-weight: 400; }
|
||||
.service-title { font-size: 46px; font-weight: 400; }
|
||||
.node-text { font-size: 34px; font-weight: 600; text-anchor: middle; }
|
||||
.body { font-size: 28px; }
|
||||
.edge-label { font-size: 25px; }
|
||||
.endpoint-text { font-size: 23px; }
|
||||
.source-text { font-size: 21px; }
|
||||
.required-node { fill: #fff; stroke: #303038; stroke-width: 4; }
|
||||
.optional-node { fill: #fff; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
|
||||
.required-line { fill: none; stroke: #303038; stroke-width: 4; marker-end: url(#arrow); }
|
||||
.optional-line { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; marker-end: url(#arrow-optional); }
|
||||
.boundary { fill: none; stroke: #303038; stroke-width: 4; }
|
||||
.optional-boundary { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<rect width="1920" height="1600" fill="#fff"/>
|
||||
<text x="105" y="72" class="label title">How does vmanomaly operate (scheduled iteration example)</text>
|
||||
|
||||
<!-- Required / optional legend -->
|
||||
<g aria-label="Legend" transform="translate(1425 145)">
|
||||
<rect x="0" y="0" width="135" height="62" class="optional-node"/>
|
||||
<rect x="205" y="0" width="135" height="62" class="required-node"/>
|
||||
<path d="M0 100 H135" class="optional-line"/>
|
||||
<path d="M205 100 H340" class="required-line"/>
|
||||
<text x="67" y="162" class="label body" text-anchor="middle">Optional</text>
|
||||
<text x="272" y="162" class="label body" text-anchor="middle">Required</text>
|
||||
</g>
|
||||
|
||||
<!-- Exactly one configured read datasource is used. -->
|
||||
<g aria-label="Configured datasource choice">
|
||||
<rect x="25" y="315" width="300" height="550" class="optional-boundary"/>
|
||||
|
||||
<g transform="translate(34 326)">
|
||||
<path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
|
||||
<ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
|
||||
<rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
|
||||
<text x="144" y="99" class="label endpoint-text" text-anchor="middle">/query_range</text>
|
||||
<text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
|
||||
<text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
|
||||
<text x="144" y="182" class="label source-text" text-anchor="middle">TSDB</text>
|
||||
</g>
|
||||
|
||||
<text x="178" y="600" class="label body" text-anchor="middle">OR</text>
|
||||
|
||||
<g transform="translate(34 632)">
|
||||
<path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
|
||||
<ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
|
||||
<rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
|
||||
<text x="144" y="99" class="label endpoint-text" text-anchor="middle">/stats_query_range</text>
|
||||
<text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
|
||||
<text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaLogs /</text>
|
||||
<text x="144" y="182" class="label source-text" text-anchor="middle">VictoriaTraces</text>
|
||||
</g>
|
||||
</g>
|
||||
|
||||
<!-- vmanomaly service boundary and components -->
|
||||
<g aria-label="vmanomaly service">
|
||||
<rect x="720" y="145" width="610" height="1145" class="boundary"/>
|
||||
<rect x="1005" y="175" width="270" height="100" class="required-node"/>
|
||||
<text x="1140" y="235" class="label node-text">config.yml</text>
|
||||
<text x="1025" y="352" class="label service-title" text-anchor="middle">vmanomaly</text>
|
||||
<text x="1025" y="405" class="label service-title" text-anchor="middle">service</text>
|
||||
|
||||
<rect x="765" y="448" width="445" height="76" rx="16" class="required-node"/>
|
||||
<text x="987" y="500" class="label node-text">Scheduler</text>
|
||||
<rect x="765" y="590" width="445" height="76" rx="16" class="required-node"/>
|
||||
<text x="987" y="642" class="label node-text">Reader</text>
|
||||
<rect x="765" y="732" width="445" height="76" rx="16" class="required-node"/>
|
||||
<text x="987" y="784" class="label node-text">Model</text>
|
||||
<rect x="765" y="874" width="445" height="76" rx="16" class="required-node"/>
|
||||
<text x="987" y="926" class="label node-text">Writer</text>
|
||||
<rect x="778" y="1180" width="420" height="82" rx="16" class="optional-node"/>
|
||||
<text x="988" y="1235" class="label node-text">Monitoring</text>
|
||||
|
||||
<path d="M1005 225 H900 V448" class="required-line"/>
|
||||
<path d="M1210 486 H1290 V628 H1210" class="required-line"/>
|
||||
<path d="M1210 628 H1290 V770 H1210" class="required-line"/>
|
||||
<path d="M1210 770 H1290 V912 H1210" class="required-line"/>
|
||||
|
||||
<path d="M875 666 V1178" class="optional-line"/>
|
||||
<path d="M987 808 V1178" class="optional-line"/>
|
||||
<path d="M1100 950 V1178" class="optional-line"/>
|
||||
</g>
|
||||
|
||||
<!-- Required flow labels -->
|
||||
<text x="1365" y="455" class="label body">
|
||||
<tspan x="1365" dy="0">1. Get the metrics to</tspan>
|
||||
<tspan x="1365" dy="37">a. fit the model, or</tspan>
|
||||
<tspan x="1365" dy="37">b. produce anomaly scores</tspan>
|
||||
</text>
|
||||
<text x="1365" y="620" class="label body">
|
||||
<tspan x="1365" dy="0">3. Model receives data</tspan>
|
||||
<tspan x="1365" dy="37">to train or infer on</tspan>
|
||||
</text>
|
||||
<text x="1365" y="775" class="label body">
|
||||
<tspan x="1365" dy="0">4. Produce anomaly</tspan>
|
||||
<tspan x="1365" dy="37">scores (inference)</tspan>
|
||||
</text>
|
||||
|
||||
<!-- Datasource request and response -->
|
||||
<path d="M765 610 H338" class="required-line"/>
|
||||
<path d="M338 652 H765" class="required-line"/>
|
||||
<text x="370" y="580" class="label edge-label">2.1 Query request</text>
|
||||
<text x="370" y="700" class="label edge-label">2.2 Metrics response</text>
|
||||
|
||||
<!-- Anomaly-score output -->
|
||||
<g aria-label="VictoriaMetrics write endpoint" transform="translate(26 1145)">
|
||||
<path d="M8 32 V184 C8 224 290 224 290 184 V32 C290 70 8 70 8 32Z" class="required-node"/>
|
||||
<ellipse cx="149" cy="32" rx="141" ry="33" class="required-node"/>
|
||||
<rect x="34" y="72" width="230" height="66" rx="16" class="required-node"/>
|
||||
<text x="149" y="99" class="label endpoint-text" text-anchor="middle">/import</text>
|
||||
<text x="149" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
|
||||
<text x="149" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
|
||||
<text x="149" y="182" class="label source-text" text-anchor="middle">TSDB</text>
|
||||
</g>
|
||||
<path d="M765 912 H650 V1095 L316 1197" class="required-line"/>
|
||||
<rect x="338" y="1018" width="292" height="74" fill="#fff"/>
|
||||
<text x="350" y="1048" class="label edge-label">
|
||||
<tspan x="340" dy="0">5. Write anomaly scores</tspan>
|
||||
<tspan x="340" dy="33">to VictoriaMetrics</tspan>
|
||||
</text>
|
||||
|
||||
<!-- Optional monitoring integrations -->
|
||||
<rect x="748" y="1083" width="490" height="48" fill="#fff"/>
|
||||
<text x="993" y="1116" class="label edge-label" text-anchor="middle">6. Produce self-monitoring metrics</text>
|
||||
<rect x="390" y="1460" width="300" height="118" rx="45" class="optional-node"/>
|
||||
<text x="540" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
|
||||
<text x="540" y="1552" class="label body" text-anchor="middle">push approach</text>
|
||||
<rect x="1490" y="1460" width="300" height="118" rx="45" class="optional-node"/>
|
||||
<text x="1640" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
|
||||
<text x="1640" y="1552" class="label body" text-anchor="middle">pull approach</text>
|
||||
<path d="M850 1262 L540 1460" class="optional-line"/>
|
||||
<path d="M1198 1222 L1640 1460" class="optional-line"/>
|
||||
<rect x="590" y="1350" width="180" height="42" fill="#fff"/>
|
||||
<text x="680" y="1380" class="label edge-label" text-anchor="middle">Push metrics</text>
|
||||
<rect x="1230" y="1340" width="415" height="42" fill="#fff"/>
|
||||
<text x="1438" y="1370" class="label edge-label" text-anchor="middle">HTTP GET /metrics or /health</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 9.2 KiB |
BIN
docs/anomaly-detection/components/vmanomaly-components.webp
Normal file
|
After Width: | Height: | Size: 107 KiB |
@@ -17,10 +17,6 @@ Future updates will introduce additional export methods, offering users more fle
|
||||
|
||||
## VM writer
|
||||
|
||||
<div class="collapse-group">
|
||||
|
||||
{{% collapse name="VM writer config parameters and example" %}}
|
||||
|
||||
### Config parameters
|
||||
|
||||
<table class="params">
|
||||
@@ -39,7 +35,8 @@ Future updates will introduce additional export methods, offering users more fle
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
|
||||
<span style="white-space: nowrap;">`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
@@ -66,7 +63,10 @@ Datasource URL address
|
||||
<span style="white-space: nowrap;">`tenant_id`</span>
|
||||
</td>
|
||||
<td>
|
||||
<span>
|
||||
|
||||
`0:0`, `multitenant`{{% available_from "v1.16.2" anomaly %}}
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
@@ -253,8 +253,9 @@ Token is passed in the standard format with header: `Authorization: bearer {toke
|
||||
`path_to_file`
|
||||
</td>
|
||||
<td>
|
||||
<span>
|
||||
Path to a file, which contains token, that is passed in the standard format with header: `Authorization: bearer {token}`{{% available_from "v1.15.9" anomaly %}}
|
||||
</td>
|
||||
</span> </td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
@@ -266,8 +267,9 @@ Path to a file, which contains token, that is passed in the standard format with
|
||||
`1`
|
||||
</td>
|
||||
<td>
|
||||
<span>
|
||||
Number of attempts to retry the connection in case of failure {{% available_from "v1.29.2" anomaly %}}.
|
||||
</td>
|
||||
</span> </td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -291,10 +293,6 @@ writer:
|
||||
connection_retry_attempts: 2 # if not specified, it will be 1 by default
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
</div>
|
||||
|
||||
### Multitenancy support
|
||||
|
||||
> This feature applies to the VictoriaMetrics Cluster version only. Tenants are identified by either `accountID` or `accountID:projectID`. `multitenant` [endpoint](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels){{% available_from "v1.15.9" anomaly %}} is supported for writing data across multiple [tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy). For more details, refer to the VictoriaMetrics Cluster [multitenancy documentation](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).
|
||||
|
||||
@@ -17,7 +17,7 @@ sitemap:
|
||||
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
|
||||
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
|
||||
|
||||

|
||||

|
||||
|
||||
> **Configurations used throughout this guide can be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-integration/)**
|
||||
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1706 1069" role="img" aria-labelledby="title description">
|
||||
<title id="title">Typical vmanomaly observability pipeline</title>
|
||||
<desc id="description">vmagent scrapes node-exporter metrics and writes them to VictoriaMetrics. vmanomaly reads those metrics and writes anomaly scores back. Grafana visualizes the results. vmalert evaluates rules based on anomaly scores and sends alerts to Alertmanager.</desc>
|
||||
<defs>
|
||||
<marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="10" markerHeight="10" orient="auto-start-reverse">
|
||||
<path d="M0 0 10 5 0 10Z" fill="#252525"/>
|
||||
</marker>
|
||||
<style>
|
||||
text { font-family: Arial, Helvetica, sans-serif; fill: #252525; font-size: 40px; }
|
||||
.box { fill: #fff; stroke: #252525; stroke-width: 3; }
|
||||
.arrow { fill: none; stroke: #252525; stroke-width: 4; marker-end: url(#arrow); }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<rect width="1706" height="1069" fill="#fff"/>
|
||||
|
||||
<rect x="20" y="55" width="390" height="100" class="box"/>
|
||||
<text x="215" y="119" text-anchor="middle">node-exporter</text>
|
||||
<rect x="730" y="50" width="395" height="100" class="box"/>
|
||||
<text x="928" y="115" text-anchor="middle">vmagent</text>
|
||||
<path d="M710 100 H435" class="arrow"/>
|
||||
<text x="570" y="56" text-anchor="middle">Scrape metrics</text>
|
||||
|
||||
<rect x="745" y="325" width="400" height="180" class="box"/>
|
||||
<text x="945" y="405" text-anchor="middle">VictoriaMetrics</text>
|
||||
<text x="945" y="457" text-anchor="middle">TSDB</text>
|
||||
<path d="M945 150 V322" class="arrow"/>
|
||||
<text x="1018" y="204">
|
||||
<tspan x="1018" dy="0">Push node</tspan>
|
||||
<tspan x="1018" dy="50">exporter</tspan>
|
||||
<tspan x="1018" dy="50">metrics</tspan>
|
||||
</text>
|
||||
|
||||
<rect x="25" y="340" width="395" height="100" class="box"/>
|
||||
<text x="222" y="405" text-anchor="middle">vmanomaly</text>
|
||||
<path d="M725 365 H440" class="arrow"/>
|
||||
<text x="575" y="319" text-anchor="middle">Read metrics</text>
|
||||
<path d="M440 421 H725" class="arrow"/>
|
||||
<text x="570" y="508" text-anchor="middle">
|
||||
<tspan x="570" dy="0">Write produced</tspan>
|
||||
<tspan x="570" dy="50">anomaly scores</tspan>
|
||||
</text>
|
||||
|
||||
<rect x="1440" y="330" width="245" height="170" class="box"/>
|
||||
<text x="1562" y="430" text-anchor="middle">Grafana</text>
|
||||
<path d="M1160 417 H1418" class="arrow"/>
|
||||
<text x="1285" y="486" text-anchor="middle">
|
||||
<tspan x="1285" dy="0">Visualize the</tspan>
|
||||
<tspan x="1285" dy="50">results</tspan>
|
||||
</text>
|
||||
|
||||
<rect x="750" y="738" width="395" height="100" class="box"/>
|
||||
<text x="948" y="805" text-anchor="middle">vmalert</text>
|
||||
<path d="M946 525 V716" class="arrow"/>
|
||||
<text x="1000" y="644">
|
||||
<tspan x="1000" dy="0">Evaluate rules based</tspan>
|
||||
<tspan x="1000" dy="50">on anomaly scores</tspan>
|
||||
</text>
|
||||
|
||||
<rect x="755" y="948" width="395" height="100" class="box"/>
|
||||
<text x="952" y="1015" text-anchor="middle">alertmanager</text>
|
||||
<path d="M948 858 V928" class="arrow"/>
|
||||
<text x="1004" y="906">Send alerts</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.0 KiB |
|
After Width: | Height: | Size: 29 KiB |
@@ -1,107 +0,0 @@
|
||||
# Render with D2 v0.7.1:
|
||||
# d2 --layout elk --theme 0 --pad 20 vmanomaly-sharding-ha-diagram.d2 vmanomaly-sharding-ha-diagram.svg
|
||||
|
||||
grid-columns: 1
|
||||
grid-gap: 36
|
||||
|
||||
classes: {
|
||||
boundary: {
|
||||
style.fill: "#F7F7F8"
|
||||
style.stroke: "#A7AAB2"
|
||||
style.stroke-width: 2
|
||||
}
|
||||
process: {
|
||||
style.fill: "#FFFFFF"
|
||||
style.stroke: "#303038"
|
||||
style.stroke-width: 2
|
||||
style.border-radius: 8
|
||||
}
|
||||
member: {
|
||||
style.fill: "#F1F3F5"
|
||||
style.stroke: "#59616A"
|
||||
style.stroke-width: 2
|
||||
style.border-radius: 8
|
||||
}
|
||||
}
|
||||
|
||||
flow: "" {
|
||||
grid-columns: 3
|
||||
grid-gap: 36
|
||||
style.fill: transparent
|
||||
style.stroke: transparent
|
||||
|
||||
global: "Global YAML\nconfiguration" {
|
||||
shape: page
|
||||
class: process
|
||||
}
|
||||
|
||||
splitting: Configuration splitting {
|
||||
class: boundary
|
||||
grid-columns: 1
|
||||
grid-gap: 24
|
||||
|
||||
split: "Split by VMANOMALY_SPLIT_BY\ndefault: complete" {
|
||||
class: process
|
||||
}
|
||||
subconfigs: "N valid sub-configurations\nn = 0 ... N-1" {
|
||||
class: process
|
||||
style.multiple: true
|
||||
}
|
||||
|
||||
split -> subconfigs: {
|
||||
style.stroke: "#303038"
|
||||
}
|
||||
}
|
||||
|
||||
placement: Deterministic placement {
|
||||
class: boundary
|
||||
grid-columns: 1
|
||||
grid-gap: 24
|
||||
|
||||
settings: "K members: VMANOMALY_MEMBERS_COUNT\nMember index: VMANOMALY_MEMBER_NUM\nR replicas: VMANOMALY_REPLICATION_FACTOR" {
|
||||
class: process
|
||||
}
|
||||
assign: "Assign every sub-configuration\nto exactly R members" {
|
||||
class: process
|
||||
}
|
||||
|
||||
settings -> assign: {
|
||||
style.stroke: "#303038"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
members: K vmanomaly members {
|
||||
class: boundary
|
||||
grid-columns: 3
|
||||
grid-gap: 30
|
||||
|
||||
member0: "Member 0\nvmanomaly service 1" {
|
||||
class: member
|
||||
}
|
||||
member1: "Member 1\nvmanomaly service 2" {
|
||||
class: member
|
||||
}
|
||||
memberK: "Member K-1\nvmanomaly service K" {
|
||||
class: member
|
||||
}
|
||||
}
|
||||
|
||||
flow.global -> flow.splitting.split: {
|
||||
style.stroke: "#303038"
|
||||
}
|
||||
flow.splitting.subconfigs -> flow.placement.assign: {
|
||||
style.stroke: "#303038"
|
||||
}
|
||||
flow.placement.assign -> members.member0: "assigned subset" {
|
||||
style.stroke: "#59616A"
|
||||
style.stroke-width: 2
|
||||
}
|
||||
flow.placement.assign -> members.member1: "assigned subset" {
|
||||
style.stroke: "#59616A"
|
||||
style.stroke-width: 2
|
||||
}
|
||||
flow.placement.assign -> members.memberK: "assigned subset" {
|
||||
style.stroke: "#59616A"
|
||||
style.stroke-width: 2
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
---
|
||||
build:
|
||||
list: never
|
||||
publishResources: false
|
||||
render: never
|
||||
sitemap:
|
||||
disable: true
|
||||
---
|
||||
|
||||
The global configuration is split into `N` independently valid sub-configurations. Deterministic placement distributes them across `K` members, and each sub-configuration is assigned to exactly `R` members when replication is enabled. Member `k` processes only its assigned subset.
|
||||
|
||||

|
||||
{style="display:block; width:50%; min-width:320px; margin:1.5rem auto"}
|
||||
|
Before Width: | Height: | Size: 33 KiB |
BIN
docs/anomaly-detection/vmanomaly-sharding-ha-diagram.webp
Normal file
|
After Width: | Height: | Size: 87 KiB |
@@ -156,13 +156,6 @@ See [our blog](https://victoriametrics.com/blog) for the latest articles written
|
||||
* [Why irate from Prometheus doesn't capture spikes](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
|
||||
* [VictoriaMetrics: PromQL compliance](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e)
|
||||
* [How do open source solutions for logs work: Elasticsearch, Loki and VictoriaLogs](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f)
|
||||
* [How vmagent Collects and Ships Metrics Fast with Aggregation, Deduplication, and More](https://victoriametrics.com/blog/vmagent-how-it-works/)
|
||||
* [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/)
|
||||
* [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/)
|
||||
* [How vmstorage Processes Data: Retention, Merging, Deduplication...](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/)
|
||||
* [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/)
|
||||
* [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/)
|
||||
* [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/)
|
||||
|
||||
### Tutorials, guides and how-to articles
|
||||
|
||||
@@ -180,12 +173,6 @@ See [our guides](https://docs.victoriametrics.com/guides/) for the up-to-date gu
|
||||
* [Prometheus storage: tech terms for humans](https://valyala.medium.com/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
|
||||
* [Cardinality explorer](https://victoriametrics.com/blog/cardinality-explorer/)
|
||||
* [Rules backfilling via vmalert](https://victoriametrics.com/blog/rules-replay/)
|
||||
* [vmagent: Key Features Explained in Under 15 Minutes](https://victoriametrics.com/blog/vmagent-key-features-explained/)
|
||||
* [Prometheus Metrics Explained: Counters, Gauges, Histograms & Summaries](https://victoriametrics.com/blog/prometheus-monitoring-metrics-counters-gauges-histogram-summaries/)
|
||||
* [Prometheus Monitoring: Instant Queries and Range Queries Explained](https://victoriametrics.com/blog/prometheus-monitoring-instant-range-query/)
|
||||
* [Prometheus Monitoring: Functions, Subqueries, Operators, and Modifiers](https://victoriametrics.com/blog/prometheus-monitoring-function-operator-modifier/)
|
||||
* [Prometheus Alerting 101: Rules, Recording Rules, and Alertmanager](https://victoriametrics.com/blog/alerting-recording-rules-alertmanager/)
|
||||
* [Alerting Best Practices](https://victoriametrics.com/blog/alerting-best-practices/)
|
||||
|
||||
### Other articles
|
||||
|
||||
|
||||
@@ -59,11 +59,6 @@ It increases cluster availability, and simplifies cluster maintenance as well as
|
||||
|
||||

|
||||
|
||||
> Further reading, deep dives into how each service works internally:
|
||||
> - `vmstorage`: [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/), [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/), [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/).
|
||||
> - `vminsert`: [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/).
|
||||
> - `vmselect`: [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/).
|
||||
|
||||
## vmui
|
||||
|
||||
VictoriaMetrics cluster version provides UI for query troubleshooting and exploration. The UI is available at
|
||||
@@ -834,9 +829,9 @@ See also [minimum downtime strategy](#minimum-downtime-strategy).
|
||||
|
||||
## Slowness-based re-routing
|
||||
|
||||
By default{{% available_from "#" %}}, `vminsert` automatically [re-route writes](https://victoriametrics.com/blog/vminsert-how-it-works/#31-rerouting)
|
||||
away from the slowest `vmstorage` node to preserve maximum ingestion throughput. This prevents a single slow `vmstorage`
|
||||
node from throttling the entire cluster.
|
||||
By default{{% available_from "#" %}}, `vminsert` automatically re-routes writes away from the slowest `vmstorage` node
|
||||
to preserve maximum ingestion throughput. This prevents a single slow `vmstorage` node
|
||||
from throttling the entire cluster.
|
||||
|
||||
Re-routing occurs only when all of the following conditions hold:
|
||||
- the storage send buffer is full.
|
||||
@@ -883,7 +878,7 @@ See also [resource usage limits docs](#resource-usage-limits).
|
||||
|
||||
## Rebalancing
|
||||
|
||||
Every `vminsert` node [evenly spreads (shards) incoming data](https://victoriametrics.com/blog/vminsert-how-it-works/#3-sharding-and-buffering) among `vmstorage` nodes specified in the `-storageNode` command-line flag.
|
||||
Every `vminsert` node evenly spreads (shards) incoming data among `vmstorage` nodes specified in the `-storageNode` command-line flag.
|
||||
This guarantees even distribution of the ingested data among `vmstorage` nodes. When new `vmstorage` nodes are added to the `-storageNode`
|
||||
command-line flag at `vminsert`, then only newly ingested data is distributed evenly among old and new `vmstorage` nodes, while
|
||||
historical data remains on the old `vmstorage` nodes. This speeds up data ingestion and querying for the majority of production workloads,
|
||||
@@ -1031,7 +1026,7 @@ By default, VictoriaMetrics offloads replication to the underlying storage point
|
||||
which guarantees data durability. VictoriaMetrics supports application-level replication if replicated durable persistent disks cannot be used for some reason.
|
||||
|
||||
The replication can be enabled by passing `-replicationFactor=N` command-line flag to `vminsert`. This instructs `vminsert` to store `N` copies for every ingested sample
|
||||
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable. See [how `vminsert` replicates each sample to `N` `vmstorage` nodes](https://victoriametrics.com/blog/vminsert-how-it-works/#4-replication-and-sending-data-to-vmstorage) for details.
|
||||
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable.
|
||||
|
||||
Passing `-replicationFactor=N` command-line flag to `vmselect` instructs it to not mark responses as `partial` if less than `-replicationFactor` vmstorage nodes are unavailable during the query.
|
||||
See [cluster availability docs](#cluster-availability) for details.
|
||||
@@ -1066,7 +1061,7 @@ deduplication can't be guaranteed when samples and sample duplicates for the sam
|
||||
- when `vmstorage` node has no enough capacity for processing incoming data stream. Then `vminsert` re-routes new samples to other `vmstorage` nodes.
|
||||
|
||||
It is recommended to set **the same** `-dedup.minScrapeInterval` command-line flag value to both `vmselect` and `vmstorage` nodes
|
||||
to ensure query results consistency, even if [storage layer didn't complete deduplication](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#deduplication) yet.
|
||||
to ensure query results consistency, even if storage layer didn't complete deduplication yet.
|
||||
|
||||
## Metrics Metadata
|
||||
|
||||
|
||||
@@ -1405,9 +1405,6 @@ for fast block lookups, which belong to the given `TSID` and cover the given tim
|
||||
* various background maintenance tasks such as [de-duplication](#deduplication), [downsampling](#downsampling)
|
||||
and [freeing up disk space for the deleted time series](#how-to-delete-time-series) are performed during the merge
|
||||
|
||||
See how `vmstorage` [selects parts for background merging](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#merge-process),
|
||||
including merge limits and monitoring metrics.
|
||||
|
||||
Newly added `parts` either successfully appear in the storage or fail to appear.
|
||||
The newly added `part` is atomically registered in the `parts.json` file under the corresponding partition
|
||||
after it is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html) to the storage.
|
||||
@@ -1535,8 +1532,6 @@ are **eventually deleted** during [background merge](https://medium.com/@valyala
|
||||
The time range covered by data part is **not limited by retention period unit**. One data part can cover hours or days of
|
||||
data. Hence, a data part can be deleted only **when fully outside the configured retention**.
|
||||
See more about partitions and parts in the [Storage section](#storage).
|
||||
See how the [retention and free-disk watchers manage storage](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#retention-free-disk-space-guard-and-downsampling)
|
||||
for implementation details and monitoring metrics.
|
||||
|
||||
The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
|
||||
For example, if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
|
||||
|
||||
@@ -28,16 +28,11 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
**Update Note 1:** `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the default value of `-disableRerouting` flag has changed from `true` to `false`, enabling [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. If you rely on the old behavior, pass `-disableRerouting` command-line flag to `vminsert`. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
|
||||
|
||||
**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints now require `POST` method. Previously, it also accepted `GET` requests. If you use `GET` requests for this endpoint, update your scripts or tooling to use `POST` instead. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
|
||||
|
||||
* SECURITY: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): restrict `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints to `POST` method only to prevent some [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery)-based data deletion attacks. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
|
||||
* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enable [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Previously, `-disableRerouting` defaulted to `true`, which limited ingestion throughput to the slowest `vmstorage` node. Now `-disableRerouting` defaults to `false`, so `vminsert` automatically routes data away from the slowest `vmstorage` node, improving overall ingestion performance. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the selected auto-refresh interval in the URL. See [VictoriaLogs#1310](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1310).
|
||||
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
|
||||
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-remoteWrite.obfuscateLabels` flag for hashing values of the specified labels before sending metrics to the corresponding `-remoteWrite.url`. This allows sharing metrics with external systems while keeping sensitive label values hidden. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
|
||||
|
||||
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
|
||||
@@ -248,6 +243,7 @@ Released at 2026-04-24
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect evaluation of binary operations caused by an ordering bug (e.g. `10 - (3 + 3 + 4)` being evaluated as `10 - 3 + 3 + 4`). The issue was introduced in v1.140.0, v1.136.4, and v1.122.19. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
|
||||
|
||||
## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
|
||||
|
||||
Released at 2026-04-10
|
||||
|
||||
**Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).
|
||||
|
||||
@@ -141,7 +141,6 @@ to other remote storage systems that support Prometheus `remote_write` protocol
|
||||
If a single remote storage instance is temporarily unavailable, the collected data remains available on the other remote storage instances.
|
||||
`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
|
||||
Then it sends the buffered data to the remote storage in order to prevent data gaps.
|
||||
See how `vmagent` [selects shards and places replicas](https://victoriametrics.com/blog/vmagent-how-it-works/#step-4-sharding--replication) for implementation details.
|
||||
|
||||
[VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) already supports replication,
|
||||
so there is no need to specify multiple `-remoteWrite.url` flags when writing data to the same cluster.
|
||||
@@ -151,8 +150,7 @@ See [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victor
|
||||
|
||||
`vmagent` can add, remove, or update labels on the collected data before sending it to the remote storage.
|
||||
It can filter scrape targets or remove unwanted samples via Prometheus-like relabeling.
|
||||
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for configuration examples.
|
||||
For ingestion pipeline internals, see how `vmagent` [applies global relabeling and cardinality limits](https://victoriametrics.com/blog/vmagent-how-it-works/#step-2-global-relabeling-cardinality-reduction).
|
||||
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for details.
|
||||
|
||||
### Sharding among remote storages
|
||||
|
||||
@@ -270,9 +268,7 @@ for the collected samples. Examples:
|
||||
```sh
|
||||
./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -streamAggr.dropInputLabels=replica -streamAggr.dedupInterval=60s
|
||||
```
|
||||
|
||||
See how `vmagent` [orders global deduplication and stream aggregation](https://victoriametrics.com/blog/vmagent-how-it-works/#step-3-global-deduplication--stream-aggregation) in the ingestion pipeline.
|
||||
|
||||
|
||||
### Monitoring Data eXchange
|
||||
|
||||
The Monitoring Data eXchange (MDX){{% available_from "v1.147.0" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
|
||||
@@ -327,8 +323,7 @@ flowchart TB
|
||||
H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
|
||||
H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
|
||||
H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
|
||||
H5 --> H6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
|
||||
H6 --> H7[[push to <b>-remoteWrite.url</b>]]
|
||||
H5 --> H6[[push to <b>-remoteWrite.url</b>]]
|
||||
|
||||
%% Right branch
|
||||
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
|
||||
@@ -336,8 +331,7 @@ flowchart TB
|
||||
R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
|
||||
R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
|
||||
R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
|
||||
R5 --> R6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
|
||||
R6 --> R7[[push to <b>-remoteWrite.url</b>]]
|
||||
R5 --> R6[[push to <b>-remoteWrite.url</b>]]
|
||||
```
|
||||
|
||||
Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:
|
||||
@@ -363,8 +357,6 @@ in addition to the pull-based Prometheus-compatible targets' scraping:
|
||||
* Prometheus exposition format via `http://<vmagent>:8429/api/v1/import/prometheus`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
|
||||
* Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data).
|
||||
|
||||
See how `vmagent` [handles concurrency, decompression, and stream parsing](https://victoriametrics.com/blog/vmagent-how-it-works/#step-1-receiving-data-via-api-or-scrape) during ingestion.
|
||||
|
||||
## How to collect metrics in Prometheus format
|
||||
|
||||
Specify the path to the `prometheus.yml` file via the `-promscrape.config` command-line flag. `vmagent` takes into account the following
|
||||
@@ -691,35 +683,6 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
|
||||
/path/to/vmagent -remoteWrite.url=http://127.0.0.1:8428/api/v1/write?extra_label="env=prod"
|
||||
```
|
||||
|
||||
## Obfuscating label values
|
||||
|
||||
`vmagent` can obfuscate the values of specified labels before sending metrics to `-remoteWrite.url`
|
||||
via `-remoteWrite.obfuscateLabels`{{% available_from "#" %}}.
|
||||
|
||||
This is useful when one or more `-remoteWrite.url` endpoints point to external monitoring services
|
||||
outside the organization, and sensitive label values such as `ip`, `host`, `instance`, or `datacenter`
|
||||
must not be exposed.
|
||||
|
||||
Label values are replaced with their SHA-256 hex digests. No salt is applied, so values with a small
|
||||
or predictable value space can potentially be recovered by brute force.
|
||||
|
||||
This feature pairs well with [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange):
|
||||
use `-remoteWrite.mdx.enable=true` to forward only VictoriaMetrics self-monitoring metrics to an
|
||||
external vendor while hiding sensitive label values with `-remoteWrite.obfuscateLabels`.
|
||||
|
||||
Use `-remoteWrite.obfuscateLabels` to list label names whose values should be obfuscated for the
|
||||
corresponding `-remoteWrite.url`. Separate multiple label names with `^^`.
|
||||
|
||||
```sh
|
||||
./vmagent \
|
||||
-remoteWrite.url=http://<external-service1> \
|
||||
-remoteWrite.obfuscateLabels='instance^^datacenter' \
|
||||
-remoteWrite.url=http://<external-service2> \
|
||||
-remoteWrite.obfuscateLabels='instance' \
|
||||
-remoteWrite.url=http://<internal-service> \
|
||||
-remoteWrite.obfuscateLabels=''
|
||||
```
|
||||
|
||||
## Automatically generated metrics
|
||||
|
||||
`vmagent` automatically generates the following metrics for each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)
|
||||
@@ -1072,7 +1035,6 @@ This behavior can be changed with the `-remoteWrite.inmemoryQueues` {{% availabl
|
||||
When set to a non-zero value, vmagent starts the given number of additional workers,
|
||||
which send only recently ingested data from the in-memory queue, while the workers configured via `-remoteWrite.queues` drain the file-based backlog concurrently.
|
||||
This reduces the delivery lag for fresh samples after remote storage outages or slowdowns. The flag can be set individually per each `-remoteWrite.url`.
|
||||
See how the [in-memory and file-based queues manage blocks](https://victoriametrics.com/blog/vmagent-how-it-works/#in-memory-queue) for implementation details.
|
||||
|
||||
Note that these workers are started in addition to the workers configured via `-remoteWrite.queues`, so the total number of concurrent connections to
|
||||
the remote storage becomes the sum of both flags. Take this into account if the remote storage limits the number of concurrent requests.
|
||||
|
||||
@@ -1024,9 +1024,6 @@ Try the following tips to avoid common issues:
|
||||
In that case, the default step will be used (`-datasource.queryStep`) and may cause unexpected results compared to
|
||||
executing this query in vmui/Grafana, where step is adjusted differently.
|
||||
|
||||
See [practical examples for reducing alert noise](https://victoriametrics.com/blog/alerting-best-practices/#reducing-noise),
|
||||
including aggregating alerts and configuring inhibition.
|
||||
|
||||
### Rule state
|
||||
|
||||
vmalert keeps the last `-rule.updateEntriesLimit` updates (or `update_entries_limit` [per-rule config](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules))
|
||||
@@ -1082,9 +1079,6 @@ Sometimes, it's hard to understand why a specific alert fired or not. Keep in mi
|
||||
If evaluation returns error (i.e. datasource is unavailable), alert state doesn't change.
|
||||
If at least one evaluation returns no data, then alert's `for` state resets.
|
||||
|
||||
See [how to tune the `for` parameter](https://victoriametrics.com/blog/alerting-best-practices/#the-for-param),
|
||||
including its tradeoff with the query lookbehind window.
|
||||
|
||||
> Note: The alert state is tracked separately for each time series returned during evaluation.
|
||||
> For example, if the 1st evaluation returns series A and B, and the 2nd evaluation returns only B – the alert will remain active **only for B**.
|
||||
|
||||
|
||||
@@ -2843,8 +2843,9 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
}
|
||||
var searchTimeRange TimeRange
|
||||
|
||||
// Search time range is the same as globalIndexTimeRange.
|
||||
searchTimeRange = globalIndexTimeRange
|
||||
// Zero search time range is adjusted to globalIndexTimeRange regardless
|
||||
// whether the -disablePerDayIndex flag is set or not.
|
||||
searchTimeRange = TimeRange{}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
@@ -2852,6 +2853,9 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
// The search time range is smaller than a month (and therefore < 40 days)
|
||||
// and is fully included into the partition idb time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange. Otherwise it must remain the same
|
||||
// after the adjustment.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
|
||||
@@ -2862,6 +2866,11 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
|
||||
// The search time range is the same as partition idb time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb: it must remain the same
|
||||
// - For the partition idb: it must be replaced with globalIndexTimeRange.
|
||||
searchTimeRange = partitionIDBTimeRange
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
@@ -2870,6 +2879,11 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
// The search time range is smaller than 40 days and fully includes the
|
||||
// partition idb time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb: it must remain the same
|
||||
// - For the partition idb: it must be replaced with globalIndexTimeRange.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
|
||||
@@ -2881,6 +2895,10 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
// The search time range is 41 days and fully includes the partition idb
|
||||
// time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise it must be replaced with globalIndexTimeRange for both legacy
|
||||
// and partition idbs.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + 41*msecPerDay,
|
||||
@@ -2892,6 +2910,12 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
// The search time range is smaller than 40 days and overlaps with partition
|
||||
// idb time range on the left.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb: it must remain the same
|
||||
// - For the partition idb: the MinTimestamp must be adjusted to match the
|
||||
// partition idb time range MinTimestamp.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
|
||||
@@ -2906,6 +2930,12 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
// The search time range is smaller than 40 days and overlaps with partition
|
||||
// idb time range on the right.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb, it must remain the same
|
||||
// - For the partition idb: its MaxTimestamp must be adjusted to match the
|
||||
// partition idb time range MaxTimestamp.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
|
||||
@@ -2919,7 +2949,7 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
}
|
||||
|
||||
type testStorageSearchWithoutIndexOptions struct {
|
||||
type testStorageSearchWithoutPerDayIndexOptions struct {
|
||||
mrs []MetricRow
|
||||
assertSearchResult func(t *testing.T, s *Storage, tr TimeRange, want any)
|
||||
alwaysPerTimeRange bool // If true, use wantPerTimeRange instead of wantAll
|
||||
@@ -2928,15 +2958,16 @@ type testStorageSearchWithoutIndexOptions struct {
|
||||
wantEmpty any
|
||||
}
|
||||
|
||||
// testStorageSearchWithoutIndex tests how the search behaves when the
|
||||
// testStorageSearchWithoutPerDayIndex tests how the search behaves when the
|
||||
// per-day index is disabled. This function is expected to be called by
|
||||
// functions that test a particular search operation, such as GetTSDBStatus(),
|
||||
// SearchMetricNames(), etc.
|
||||
func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutIndexOptions) {
|
||||
func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWithoutPerDayIndexOptions) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
// The data is inserted and the search is performed when per-day index is enabled.
|
||||
t.Run("Add-Global-PerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
// The data is inserted and the search is performed when the per-day index
|
||||
// is enabled.
|
||||
t.Run("InsertAndSearchWithPerDayIndex", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
@@ -2948,8 +2979,9 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted and the search is performed when per-day index is disabled.
|
||||
t.Run("Add-Global-noPerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
// The data is inserted and the search is performed when the per-day index
|
||||
// is disabled.
|
||||
t.Run("InsertAndSearchWithoutPerDayIndex", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
@@ -2964,9 +2996,9 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when per-day index are enabled.
|
||||
// The search is performed when per-day index is disabled.
|
||||
t.Run("Add-Global-PerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
// The data is inserted when the per-day index is enabled but the search is
|
||||
// performed when the per-day index is disabled.
|
||||
t.Run("InsertWithPerDayIndexSearchWithout", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
@@ -2986,11 +3018,10 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when per-day index is disabled.
|
||||
// The search is performed when per-day index is enabled.
|
||||
// This case also shows that registering metric names recovers the per-day
|
||||
// index.
|
||||
t.Run("Add-Global-noPerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
// The data is inserted when the per-day index is disabled but the search is
|
||||
// performed when the per-day index is enabled. This case also shows that
|
||||
// registering metric names recovers the per-day index.
|
||||
t.Run("InsertWithoutPerDayIndexSearchWith", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
@@ -3017,13 +3048,13 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
})
|
||||
}
|
||||
|
||||
func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
|
||||
func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
wantEmpty: &TSDBStatus{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: &TSDBStatus{TotalSeries: days * rows},
|
||||
@@ -3063,16 +3094,16 @@ func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchMetricNamesWithoutIndex(t *testing.T) {
|
||||
func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3124,16 +3155,16 @@ func TestStorageSearchMetricNamesWithoutIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchLabelNamesWithoutIndex(t *testing.T) {
|
||||
func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3178,17 +3209,17 @@ func TestStorageSearchLabelNamesWithoutIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchLabelValuesWithoutIndex(t *testing.T) {
|
||||
func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
labelName = "job"
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3232,17 +3263,17 @@ func TestStorageSearchLabelValuesWithoutIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchTagValueSuffixesWithoutIndex(t *testing.T) {
|
||||
func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
tagValuePrefix = "metric."
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3282,16 +3313,16 @@ func TestStorageSearchTagValueSuffixesWithoutIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchGraphitePathsWithoutIndex(t *testing.T) {
|
||||
func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3332,16 +3363,16 @@ func TestStorageSearchGraphitePathsWithoutIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageQueryWithoutIndex(t *testing.T) {
|
||||
func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
wantEmpty: []MetricRow(nil),
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
alwaysPerTimeRange: true,
|
||||
@@ -3382,7 +3413,7 @@ func TestStorageQueryWithoutIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
|
||||
@@ -3424,12 +3455,12 @@ func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f(t, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
t.Run("disablePerDayIndex=false", func(t *testing.T) {
|
||||
f(t, false)
|
||||
})
|
||||
t.Run("disablePerDayIndex=true", func(t *testing.T) {
|
||||
f(t, true)
|
||||
})
|
||||
}
|
||||
|
||||
// testSearchMetricIDs returns metricIDs for the given tfss and tr.
|
||||
@@ -3466,13 +3497,13 @@ func testCountAllMetricIDs(s *Storage, tr TimeRange) int {
|
||||
return len(ids)
|
||||
}
|
||||
|
||||
func TestStorageRegisterMetricNamesVariousDataPatterns(t *testing.T) {
|
||||
func TestStorageRegisterMetricNamesForVariousDataPatternsConcurrently(t *testing.T) {
|
||||
testStorageVariousDataPatternsConcurrently(t, true, func(s *Storage, mrs []MetricRow) {
|
||||
s.RegisterMetricNames(nil, mrs)
|
||||
})
|
||||
}
|
||||
|
||||
func TestStorageAddRowsVariousDataPatterns(t *testing.T) {
|
||||
func TestStorageAddRowsForVariousDataPatternsConcurrently(t *testing.T) {
|
||||
testStorageVariousDataPatternsConcurrently(t, false, func(s *Storage, mrs []MetricRow) {
|
||||
s.AddRows(mrs, defaultPrecisionBits)
|
||||
})
|
||||
@@ -3489,18 +3520,27 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
|
||||
|
||||
const concurrency = 4
|
||||
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
prefix := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(prefix+"/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run(prefix+"/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run(prefix+"/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
}
|
||||
disablePerDayIndex := false
|
||||
t.Run("perDayIndexes/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run("perDayIndexes/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run("perDayIndexes/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
|
||||
disablePerDayIndex = true
|
||||
t.Run("noPerDayIndexes/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run("noPerDayIndexes/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run("noPerDayIndexes/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
}
|
||||
|
||||
// testStorageVariousDataPatterns tests the ingestion of different combinations
|
||||
|
||||