Compare commits

..

2 Commits

Author SHA1 Message Date
“Jayice”
c33d0a6c1d add fail case for parsing fractional ts
Signed-off-by: “Jayice” <jzhou@victoriametrics.com>
2026-07-29 19:23:12 +08:00
“Jayice”
c204470b89 add fail case for parsing fractional ts
Signed-off-by: “Jayice” <jzhou@victoriametrics.com>
2026-07-29 19:22:50 +08:00
56 changed files with 291 additions and 2641 deletions

View File

@@ -2,6 +2,7 @@
[![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
[![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
[![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
[![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
[![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
[![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)

View File

@@ -1,108 +0,0 @@
package remotewrite
import (
"crypto/sha256"
"encoding/hex"
"slices"
"strings"
"sync"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
)
type obfuscateLabelsCtx struct {
labels []prompb.Label
// buf holds allocations for cached results below
buf []byte
// cacheResults maps original label values to their SHA-256 hex digests,
// avoiding redundant hashing of repeated values within a single batch.
cacheResults map[string]string
}
func (olctx *obfuscateLabelsCtx) reset() {
promrelabel.CleanLabels(olctx.labels)
olctx.labels = olctx.labels[:0]
clear(olctx.cacheResults)
olctx.buf = olctx.buf[:0]
}
var obfuscateLabelsCtxPool = &sync.Pool{
New: func() any {
return &obfuscateLabelsCtx{
cacheResults: make(map[string]string),
}
},
}
func getObfuscateLabelsCtx() *obfuscateLabelsCtx {
return obfuscateLabelsCtxPool.Get().(*obfuscateLabelsCtx)
}
func putObfuscateLabelsCtx(ctx *obfuscateLabelsCtx) {
ctx.reset()
obfuscateLabelsCtxPool.Put(ctx)
}
func (olctx *obfuscateLabelsCtx) obfuscate(tss []prompb.TimeSeries, obfuscateLabels []string) []prompb.TimeSeries {
if len(obfuscateLabels) == 0 || len(tss) == 0 {
return tss
}
labels := olctx.labels
for i := range tss {
ts := &tss[i]
labelsLen := len(labels)
labels = append(labels, ts.Labels...)
found := false
for _, labelName := range obfuscateLabels {
tmp := promrelabel.GetLabelByName(labels[labelsLen:], labelName)
if tmp == nil {
continue
}
found = true
if obfuscatedValue, ok := olctx.cacheResults[tmp.Value]; ok {
// fast path: the obfuscated result was calculated before
tmp.Value = obfuscatedValue
continue
}
digest := sha256.Sum256(bytesutil.ToUnsafeBytes(tmp.Value))
buf := olctx.buf
bufLen := len(buf)
buf = hex.AppendEncode(buf, digest[:])
obfuscatedValue := bytesutil.ToUnsafeString(buf[bufLen:])
olctx.buf = buf
olctx.cacheResults[tmp.Value] = obfuscatedValue
tmp.Value = obfuscatedValue
}
if found {
ts.Labels = labels[labelsLen:]
} else {
labels = labels[:labelsLen]
}
}
olctx.labels = labels
return tss
}
func (rwctx *remoteWriteCtx) initObfuscateLabels() {
if len(*obfuscateLabels) == 0 {
return
}
idx := rwctx.idx
rwObfuscateLabels := obfuscateLabels.GetOptionalArg(idx)
rwObfuscateLabelsList := strings.Split(rwObfuscateLabels, "^^")
for _, label := range rwObfuscateLabelsList {
if label == "" {
continue
}
if !slices.Contains(rwctx.obfuscateLabels, label) {
rwctx.obfuscateLabels = append(rwctx.obfuscateLabels, label)
}
}
}

View File

@@ -1,186 +0,0 @@
package remotewrite
import (
"crypto/sha256"
"encoding/hex"
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/metrics"
)
func TestRemoteWriteObfuscateLabels(t *testing.T) {
f := func(obfuscateLabelList string, inputTss []prompb.TimeSeries, expectedTss []prompb.TimeSeries) {
t.Helper()
rwctx := &remoteWriteCtx{
idx: 0,
}
olctx := &obfuscateLabelsCtx{
cacheResults: make(map[string]string),
}
defer metrics.UnregisterAllMetrics()
originValue := *obfuscateLabels
defer func() {
*obfuscateLabels = originValue
}()
*obfuscateLabels = []string{obfuscateLabelList}
rwctx.initObfuscateLabels()
outputTss := olctx.obfuscate(inputTss, rwctx.obfuscateLabels)
if !reflect.DeepEqual(expectedTss, outputTss) {
t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", outputTss, expectedTss)
}
}
sha256Result := func(str string) string {
sha256Result := sha256.Sum256([]byte(str))
return hex.EncodeToString(sha256Result[:])
}
// 1. obfuscation is not set.
f("",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
// 1. obfuscation is set for another rwctx.
f(",ip",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
// 2. obfuscate the value of "ip" label
f("ip",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: sha256Result("123")},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
// 3. obfuscate the values of "ip" and "instance"
f("ip^^instance",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "1234"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
{
Labels: []prompb.Label{
{Name: "job", Value: "123"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: sha256Result("123")},
{Name: "instance", Value: sha256Result("1234")},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
{
Labels: []prompb.Label{
{Name: "job", Value: "123"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
// 4. duplicate label names in config must produce single SHA-256, not double
f("ip^^ip",
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: sha256Result("123")},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
},
)
}

View File

@@ -1,94 +0,0 @@
package remotewrite
import (
"crypto/sha256"
"encoding/hex"
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/metrics"
)
func BenchmarkRemoteWriteObfuscateLabels(b *testing.B) {
originValue := *obfuscateLabels
defer func() {
*obfuscateLabels = originValue
}()
*obfuscateLabels = []string{"ip^^instance"}
sha256Result := func(str string) string {
sha256Result := sha256.Sum256([]byte(str))
return hex.EncodeToString(sha256Result[:])
}
expected := []prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: sha256Result("123")},
{Name: "instance", Value: sha256Result("12345")},
{Name: "__name__", Value: "http_requests_total"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
{
Labels: []prompb.Label{
{Name: "ip", Value: sha256Result("1236")},
{Name: "instance", Value: sha256Result("some-long-instante-string")},
{Name: "__name__", Value: "concurrent_requests"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
}
defer metrics.UnregisterAllMetrics()
inputTss := []prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "ip", Value: "123"},
{Name: "instance", Value: "12345"},
{Name: "__name__", Value: "http_requests_total"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
{
Labels: []prompb.Label{
{Name: "ip", Value: "1236"},
{Name: "instance", Value: "some-long-instante-string"},
{Name: "__name__", Value: "concurrent_requests"},
},
Samples: []prompb.Sample{
{Value: 1, Timestamp: 0},
},
},
}
b.ResetTimer()
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
rwctx := &remoteWriteCtx{
idx: 0,
}
olctx := &obfuscateLabelsCtx{
cacheResults: make(map[string]string),
}
rwctx.initObfuscateLabels()
var localTss []prompb.TimeSeries
for pb.Next() {
// always make a shallow copy because obfuscate changes input
localTss = localTss[:0]
localTss = append(localTss, inputTss...)
olctx.reset()
outputTss := olctx.obfuscate(localTss, rwctx.obfuscateLabels)
if !reflect.DeepEqual(expected, outputTss) {
b.Fatalf("unexpected output: got: \n%v\n want: \n%v\n", outputTss, expected)
}
}
})
}

View File

@@ -107,12 +107,7 @@ var (
"By default, metadata sending is controlled by the global -enableMetadata flag")
enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
"Can be combined with -remoteWrite.obfuscateLabels to hide sensitive label values in the forwarded metrics. "+
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
obfuscateLabels = flagutil.NewArrayString("remoteWrite.obfuscateLabels", "List of label names whose values will be obfuscated before being sent to the corresponding -remoteWrite.url. "+
"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
)
var (
@@ -886,8 +881,6 @@ type remoteWriteCtx struct {
pss []*pendingSeries
pssNextIdx atomic.Uint64
obfuscateLabels []string
rowsPushedAfterRelabel *metrics.Counter
rowsDroppedByRelabel *metrics.Counter
mdxRowsPreserved *metrics.Counter
@@ -1002,7 +995,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
}
rwctx.initStreamAggrConfig()
rwctx.initObfuscateLabels()
if enableMdx.GetOptionalArg(argIdx) {
mdxFilter := mdx.NewFilter()
@@ -1206,41 +1198,24 @@ func (rwctx *remoteWriteCtx) tryPushMetadataInternal(mms []prompb.MetricMetadata
func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries) bool {
var rctx *relabelCtx
var v *[]prompb.TimeSeries
var olctx *obfuscateLabelsCtx
defer func() {
if v != nil {
*v = prompb.ResetTimeSeries(tss)
tssPool.Put(v)
}
if rctx != nil {
putRelabelCtx(rctx)
}
if olctx != nil {
putObfuscateLabelsCtx(olctx)
if rctx == nil {
return
}
*v = prompb.ResetTimeSeries(tss)
tssPool.Put(v)
putRelabelCtx(rctx)
}()
copyTimeSeriesIfNeeded := func() {
if v == nil {
v = tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
}
}
if len(labelsGlobal) > 0 {
// Make a copy of tss before adding extra labels to prevent
// from affecting time series for other remoteWrite.url configs.
rctx = getRelabelCtx()
copyTimeSeriesIfNeeded()
v = tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
rctx.appendExtraLabels(tss, labelsGlobal)
}
if len(rwctx.obfuscateLabels) != 0 {
copyTimeSeriesIfNeeded()
olctx = getObfuscateLabelsCtx()
tss = olctx.obfuscate(tss, rwctx.obfuscateLabels)
}
pss := rwctx.pss
idx := rwctx.pssNextIdx.Add(1) % uint64(len(pss))

View File

@@ -11,7 +11,6 @@ import (
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -268,11 +267,7 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
err = &httpserver.ErrorWithStatusCode{
StatusCode: resp.StatusCode,
Err: fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body),
}
return nil, err
return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
}
return resp, nil
}

View File

@@ -31,8 +31,6 @@ type Alert struct {
State AlertState
// Expr contains expression that was executed to generate the Alert
Expr string
// Interval contains the evaluation interval of the Alert's group
Interval time.Duration
// ActiveAt defines the moment of time when Alert has become active
ActiveAt time.Time
// Start defines the moment of time when Alert has become firing
@@ -86,7 +84,6 @@ type AlertTplData struct {
Labels map[string]string
Value float64
Expr string
Interval time.Duration
AlertID uint64
GroupID uint64
ActiveAt time.Time
@@ -99,7 +96,6 @@ var tplHeaders = []string{
"{{ $type := .Type }}",
"{{ $labels := .Labels }}",
"{{ $expr := .Expr }}",
"{{ $interval := .Interval }}",
"{{ $externalLabels := .ExternalLabels }}",
"{{ $externalURL := .ExternalURL }}",
"{{ $alertID := .AlertID }}",
@@ -119,7 +115,6 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
Type: a.Type,
Labels: labels,
Expr: a.Expr,
Interval: a.Interval,
AlertID: a.ID,
GroupID: a.GroupID,
ActiveAt: a.ActiveAt,

View File

@@ -129,17 +129,6 @@ func TestAlertExecTemplate(t *testing.T) {
"exprEscapedHTML": "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
})
// interval-template
f(&Alert{
Interval: 10 * time.Second,
}, map[string]string{
"interval": "{{ .Interval }}",
"intervalVariable": "{{ $interval }}",
}, map[string]string{
"interval": "10s",
"intervalVariable": "10s",
})
// query
f(&Alert{
Expr: `vm_rows{"label"="bar"}>0`,

View File

@@ -25,12 +25,11 @@ var (
replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
"Defines how many retries to make before giving up on rule if request for it returns a retriable error.")
"Defines how many retries to make before giving up on rule if request for it returns an error.")
disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
)
func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
@@ -74,7 +73,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
for _, cfg := range groupsCfg {
ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency, *continueWithExecutionErr)
totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency)
}
logger.Infof("replay evaluation finished, generated %d samples", totalRows)
if err := rw.Close(); err != nil {

View File

@@ -8,10 +8,8 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
"github.com/VictoriaMetrics/metricsql"
)
type fakeReplayQuerier struct {
@@ -34,14 +32,6 @@ func (fc *fakeRWClient) Close() error {
}
func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
_, err = metricsql.Parse(q)
if err != nil {
return res, &httpserver.ErrorWithStatusCode{
StatusCode: 422,
Err: err,
}
}
key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
dps, ok := fr.registry[q]
if !ok {
@@ -285,28 +275,4 @@ func TestReplay(t *testing.T) {
},
},
}, 10)
// rule with wrong expression won't break the other rule with continueWithExecutionErr
continueWithExecutionErrOld := *continueWithExecutionErr
defer func() {
*continueWithExecutionErr = continueWithExecutionErrOld
}()
*continueWithExecutionErr = true
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 1, time.Millisecond, []config.Group{
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
{Rules: []config.Rule{{Record: "bar", Expr: "up ++"}}},
}, &fakeReplayQuerier{
registry: map[string]map[string][]datasource.Metric{
"sum(up)": {
"12:00:00+12:01:00": {
{
Timestamps: []int64{1, 2},
Values: []float64{1, 2},
},
},
"12:01:00+12:02:00": {},
"12:02:00+12:02:30": {},
},
},
}, 2)
}

View File

@@ -530,7 +530,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
ar.logDebugf(ts, a, "INACTIVE => PENDING")
}
a.Value = m.Values[0]
a.Interval = ar.EvalInterval
a.Annotations = annotations
a.KeepFiringSince = time.Time{}
continue
@@ -613,7 +612,6 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
Type: ar.Type.String(),
Labels: ls.origin,
Expr: ar.Expr,
Interval: ar.EvalInterval,
AlertID: hash(ls.processed),
GroupID: ar.GroupID,
ActiveAt: activeAt,
@@ -675,7 +673,6 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
Name: ar.Name,
Type: ar.Type.String(),
Expr: ar.Expr,
Interval: ar.EvalInterval,
For: ar.For,
ActiveAt: start,
Value: m.Values[0],

View File

@@ -662,7 +662,6 @@ func TestAlertingRuleExecRange(t *testing.T) {
GroupID: fakeGroup.GetID(),
Name: "for-pending",
Type: config.NewPrometheusType().String(),
Interval: time.Second,
Labels: map[string]string{"alertname": "for-pending"},
Annotations: map[string]string{},
State: notifier.StatePending,
@@ -683,7 +682,6 @@ func TestAlertingRuleExecRange(t *testing.T) {
GroupID: fakeGroup.GetID(),
Name: "for-firing",
Type: config.NewPrometheusType().String(),
Interval: 3 * time.Second,
Labels: map[string]string{"alertname": "for-firing"},
Annotations: map[string]string{},
State: notifier.StateFiring,
@@ -705,7 +703,6 @@ func TestAlertingRuleExecRange(t *testing.T) {
GroupID: fakeGroup.GetID(),
Name: "for-hold-pending",
Type: config.NewPrometheusType().String(),
Interval: time.Second,
Labels: map[string]string{"alertname": "for-hold-pending"},
Annotations: map[string]string{},
State: notifier.StatePending,
@@ -762,7 +759,6 @@ func TestAlertingRuleExecRange(t *testing.T) {
GroupID: fakeGroup.GetID(),
Name: "multi-series",
Type: config.NewPrometheusType().String(),
Interval: 3 * time.Second,
Labels: map[string]string{"alertname": "multi-series"},
Annotations: map[string]string{},
State: notifier.StateFiring,
@@ -775,7 +771,6 @@ func TestAlertingRuleExecRange(t *testing.T) {
GroupID: fakeGroup.GetID(),
Name: "multi-series",
Type: config.NewPrometheusType().String(),
Interval: 3 * time.Second,
Labels: map[string]string{"alertname": "multi-series", "foo": "bar"},
Annotations: map[string]string{},
State: notifier.StatePending,
@@ -1139,8 +1134,7 @@ func TestAlertingRule_Template(t *testing.T) {
fq.Add(metrics...)
fq.SetPartialResponse(isResponsePartial)
ts := time.Unix(3600, 0)
if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
t.Fatalf("unexpected error: %s", err)
}
for hash, expAlert := range alertsExpected {
@@ -1158,14 +1152,12 @@ func TestAlertingRule_Template(t *testing.T) {
}
f(&AlertingRule{
Name: "common",
EvalInterval: time.Hour,
Name: "common",
Labels: map[string]string{
"region": "east",
},
Annotations: map[string]string{
"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
"dashboard": `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`,
"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
},
alerts: make(map[uint64]*notifier.Alert),
}, []datasource.Metric{
@@ -1174,8 +1166,7 @@ func TestAlertingRule_Template(t *testing.T) {
}, false, map[uint64]*notifier.Alert{
hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
Annotations: map[string]string{
"summary": `common: Too high connection number for "foo"`,
"dashboard": "&from=0&to=3600000",
"summary": `common: Too high connection number for "foo"`,
},
Labels: map[string]string{
alertNameLabel: "common",
@@ -1185,8 +1176,7 @@ func TestAlertingRule_Template(t *testing.T) {
},
hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
Annotations: map[string]string{
"summary": `common: Too high connection number for "bar"`,
"dashboard": "&from=0&to=3600000",
"summary": `common: Too high connection number for "bar"`,
},
Labels: map[string]string{
alertNameLabel: "common",
@@ -1398,7 +1388,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
"alertname": "ConfigurationReloadFailure",
"alertgroup": "vmalert",
"pod": "vmalert-0",
"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
}
expectedProcessedLabels := map[string]string{
@@ -1408,7 +1398,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
"exported_alertname": "ConfigurationReloadFailure",
"group": "vmalert",
"alertgroup": "vmalert",
"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
}
ls, err := ar.toLabels(metric, nil)

View File

@@ -548,7 +548,7 @@ func (g *Group) infof(format string, args ...any) {
}
// Replay performs group replay
func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int) int {
var total int
step := g.Interval * time.Duration(maxDataPoint)
ri := rangeIterator{start: start, end: end, step: step}
@@ -576,7 +576,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
if !disableProgressBar {
bar = pb.StartNew(iterations)
}
total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
if bar != nil {
bar.Finish()
}
@@ -598,7 +598,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
rule := g.Rules[i]
sem <- struct{}{}
wg.Go(func() {
res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
<-sem
})
}
@@ -618,7 +618,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
return total
}
func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
@@ -633,7 +633,7 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
start := ri.s
end := ri.e
wg.Go(func() {
n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts, continueWithExecutionErr)
n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
if err != nil {
logger.Fatalf("rule %q: %s", r, err)
}

View File

@@ -4,14 +4,12 @@ import (
"context"
"errors"
"fmt"
"net/http"
"sync"
"time"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
@@ -120,7 +118,7 @@ func (s *ruleState) add(e StateEntry) {
s.entries[s.cur] = e
}
func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int, continueWithExecutionErr bool) (int, error) {
func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
var err error
var tss []prompb.TimeSeries
for i := range replayRuleRetryAttempts {
@@ -128,21 +126,6 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
if err == nil {
break
}
// retry request if possible to tolerate temporary network or datasource unavailability issues
var esc *httpserver.ErrorWithStatusCode
if errors.As(err, &esc) {
statusCode := esc.StatusCode
// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
if statusCode == http.StatusUnprocessableEntity {
if continueWithExecutionErr {
logger.Errorf("rule %q: %s", r, err)
return 0, nil
} else {
return 0, err
}
}
}
logger.Errorf("attempt %d to execute rule %q failed: %s", i+1, r, err)
time.Sleep(time.Second)
}

View File

@@ -8,14 +8,12 @@ import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/storage/remote"
"github.com/prometheus/prometheus/tsdb/chunkenc"
@@ -236,29 +234,9 @@ func processResponse(body io.ReadCloser, callback StreamCallback) error {
// shouldn't be accounted as an error.
for _, res := range readResp.Results {
for _, ts := range res.Timeseries {
// A series contains either float samples or native histogram samples.
// Both fields are processed independently, since a series may switch
// from float to native histogram representation at some point in time,
// so the requested time range may contain samples of both types.
if len(ts.Samples) > 0 {
vmTs := convertSamples(ts.Samples, ts.Labels)
if err := callback(vmTs); err != nil {
return err
}
}
if len(ts.Histograms) > 0 {
hSamples := make([]histogramSample, 0, len(ts.Histograms))
for _, h := range ts.Histograms {
hSamples = append(hSamples, histogramSample{
timestamp: h.Timestamp,
fh: h.ToFloatHistogram(),
})
}
for _, vmTs := range convertHistograms(hSamples, ts.Labels) {
if err := callback(vmTs); err != nil {
return err
}
}
vmTs := convertSamples(ts.Samples, ts.Labels)
if err := callback(vmTs); err != nil {
return err
}
}
}
@@ -285,45 +263,17 @@ func processStreamResponse(body io.ReadCloser, callback StreamCallback) error {
for _, series := range res.ChunkedSeries {
samples := make([]prompb.Sample, 0)
var hSamples []histogramSample
for _, chunk := range series.Chunks {
switch chunk.Type {
case prompb.Chunk_XOR, prompb.Chunk_UNKNOWN:
// In proto3 the `type` field may be left unset (UNKNOWN) for XOR chunks.
// Prometheus remote.proto: "REQUIREMENT: when using proto3, this field
// MUST be set when using anything else than XOR". Senders before native
// histograms support (Prometheus < 2.40) do not set this field at all,
// so UNKNOWN chunks must be parsed as XOR ones.
s, err := parseSamples(chunk.Data)
if err != nil {
return err
}
samples = append(samples, s...)
case prompb.Chunk_HISTOGRAM, prompb.Chunk_FLOAT_HISTOGRAM:
hs, err := parseHistograms(chunk.Type, chunk.Data)
if err != nil {
return err
}
hSamples = append(hSamples, hs...)
default:
return fmt.Errorf("unsupported chunk encoding %q", chunk.Type)
s, err := parseSamples(chunk.Data)
if err != nil {
return err
}
samples = append(samples, s...)
}
// A series contains either XOR chunks or native histogram chunks.
// Both are processed independently, since a series may switch
// from float to native histogram representation at some point in time,
// so the requested time range may contain chunks of both types.
if len(samples) > 0 {
ts := convertSamples(samples, series.Labels)
if err := callback(ts); err != nil {
return err
}
}
for _, ts := range convertHistograms(hSamples, series.Labels) {
if err := callback(ts); err != nil {
return err
}
ts := convertSamples(samples, series.Labels)
if err := callback(ts); err != nil {
return err
}
}
}
@@ -362,151 +312,6 @@ func parseSamples(chunk []byte) ([]prompb.Sample, error) {
return samples, it.Err()
}
// histogramSample represents a single native histogram sample.
type histogramSample struct {
timestamp int64
fh *histogram.FloatHistogram
}
func parseHistograms(encoding prompb.Chunk_Encoding, chunk []byte) ([]histogramSample, error) {
var enc chunkenc.Encoding
switch encoding {
case prompb.Chunk_HISTOGRAM:
enc = chunkenc.EncHistogram
case prompb.Chunk_FLOAT_HISTOGRAM:
enc = chunkenc.EncFloatHistogram
default:
return nil, fmt.Errorf("unsupported histogram chunk encoding %q", encoding)
}
c, err := chunkenc.FromData(enc, chunk)
if err != nil {
return nil, fmt.Errorf("error read chunk: %w", err)
}
var hSamples []histogramSample
it := c.Iterator(nil)
for {
typ := it.Next()
if typ == chunkenc.ValNone {
break
}
switch typ {
case chunkenc.ValHistogram:
ts, h := it.AtHistogram(nil)
hSamples = append(hSamples, histogramSample{
timestamp: ts,
fh: h.ToFloat(nil),
})
case chunkenc.ValFloatHistogram:
ts, fh := it.AtFloatHistogram(nil)
hSamples = append(hSamples, histogramSample{
timestamp: ts,
fh: fh,
})
default:
// Skip unsupported values
continue
}
}
if err := it.Err(); err != nil {
return nil, fmt.Errorf("error iterate over chunks: %w", err)
}
return hSamples, nil
}
// convertHistograms converts native histogram samples into VictoriaMetrics histogram
// time series in the same way as VictoriaMetrics converts native histograms
// received via Prometheus remote write protocol: every native histogram sample
// is converted into `<name>_count` and `<name>_sum` series plus a set of
// `<name>_bucket` series with `vmrange` labels containing non-cumulative bucket counts.
// The only difference is that for native histograms with custom buckets (NHCB)
// bucket bounds are taken from the custom values, while the remote write protocol
// parser ignores custom values and estimates the bounds with the exponential formula.
// See https://prometheus.io/docs/specs/native_histograms/#data-model
func convertHistograms(hSamples []histogramSample, labels []prompb.Label) []*vm.TimeSeries {
if len(hSamples) == 0 {
return nil
}
labelPairs := make([]vm.LabelPair, 0, len(labels))
nameValue := ""
for _, label := range labels {
if label.Name == "__name__" {
nameValue = label.Value
continue
}
labelPairs = append(labelPairs, vm.LabelPair{Name: label.Name, Value: label.Value})
}
// the metric has no name, skip it in the same way as VictoriaMetrics does
// when it receives a native histogram without the metric name via remote write protocol.
if nameValue == "" {
return nil
}
countSeries := &vm.TimeSeries{
Name: nameValue + "_count",
LabelPairs: labelPairs,
}
sumSeries := &vm.TimeSeries{
Name: nameValue + "_sum",
LabelPairs: labelPairs,
}
bucketSeries := make(map[string]*vm.TimeSeries)
// vmranges preserves the order of bucketSeries creation
// in order to get deterministic results.
var vmranges []string
for _, hs := range hSamples {
fh := hs.fh
countSeries.Timestamps = append(countSeries.Timestamps, hs.timestamp)
countSeries.Values = append(countSeries.Values, fh.Count)
sumSeries.Timestamps = append(sumSeries.Timestamps, hs.timestamp)
sumSeries.Values = append(sumSeries.Values, fh.Sum)
it := fh.AllBucketIterator()
for it.Next() {
b := it.At()
if b.Count <= 0 {
continue
}
vmrange := formatVmrange(b.Lower, b.Upper)
s := bucketSeries[vmrange]
if s == nil {
bucketLabelPairs := make([]vm.LabelPair, len(labelPairs), len(labelPairs)+1)
copy(bucketLabelPairs, labelPairs)
bucketLabelPairs = append(bucketLabelPairs, vm.LabelPair{Name: "vmrange", Value: vmrange})
s = &vm.TimeSeries{
Name: nameValue + "_bucket",
LabelPairs: bucketLabelPairs,
}
bucketSeries[vmrange] = s
vmranges = append(vmranges, vmrange)
}
s.Timestamps = append(s.Timestamps, hs.timestamp)
s.Values = append(s.Values, b.Count)
}
}
tss := make([]*vm.TimeSeries, 0, 2+len(vmranges))
tss = append(tss, countSeries, sumSeries)
for _, vmrange := range vmranges {
tss = append(tss, bucketSeries[vmrange])
}
return tss
}
// formatVmrange formats the given bucket bounds into `vmrange` label value
// in the same way as VictoriaMetrics does for native histograms
// received via Prometheus remote write protocol.
func formatVmrange(lower, upper float64) string {
b := make([]byte, 0, 24)
b = strconv.AppendFloat(b, lower, 'e', 3, 64)
b = append(b, "..."...)
b = strconv.AppendFloat(b, upper, 'e', 3, 64)
return string(b)
}
type keyValue struct {
key string
value string

View File

@@ -1,334 +0,0 @@
package remoteread
import (
"bytes"
"io"
"reflect"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/storage/remote"
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
)
func testHistogram(mul int64) *histogram.Histogram {
return &histogram.Histogram{
Schema: 0,
Count: uint64(10 * mul),
Sum: 25.5 * float64(mul),
ZeroThreshold: 0.001,
ZeroCount: uint64(2 * mul),
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}},
PositiveBuckets: []int64{1 * mul, 2 * mul},
NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}},
NegativeBuckets: []int64{4 * mul},
}
}
func TestConvertHistograms(t *testing.T) {
f := func(hSamples []histogramSample, labels []prompb.Label, expected []*vm.TimeSeries) {
t.Helper()
tss := convertHistograms(hSamples, labels)
if !reflect.DeepEqual(tss, expected) {
t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, expected)
}
}
// series without samples
f(nil, []prompb.Label{{Name: "__name__", Value: "foo"}}, nil)
// series without the metric name must be skipped
f([]histogramSample{
{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
}, []prompb.Label{{Name: "job", Value: "bar"}}, nil)
// native histogram must be converted to _count, _sum and _bucket series
// in the same way as VictoriaMetrics does for Prometheus remote write protocol
labels := []prompb.Label{
{Name: "__name__", Value: "request_duration_seconds"},
{Name: "job", Value: "bar"},
}
jobLabel := []vm.LabelPair{{Name: "job", Value: "bar"}}
bucketLabels := func(vmrange string) []vm.LabelPair {
return []vm.LabelPair{
{Name: "job", Value: "bar"},
{Name: "vmrange", Value: vmrange},
}
}
f([]histogramSample{
{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
{timestamp: 2000, fh: testHistogram(2).ToFloat(nil)},
}, labels, []*vm.TimeSeries{
{
Name: "request_duration_seconds_count",
LabelPairs: jobLabel,
Timestamps: []int64{1000, 2000},
Values: []float64{10, 20},
},
{
Name: "request_duration_seconds_sum",
LabelPairs: jobLabel,
Timestamps: []int64{1000, 2000},
Values: []float64{25.5, 51},
},
{
Name: "request_duration_seconds_bucket",
LabelPairs: bucketLabels("-1.000e+00...-5.000e-01"),
Timestamps: []int64{1000, 2000},
Values: []float64{4, 8},
},
{
Name: "request_duration_seconds_bucket",
LabelPairs: bucketLabels("-1.000e-03...1.000e-03"),
Timestamps: []int64{1000, 2000},
Values: []float64{2, 4},
},
{
Name: "request_duration_seconds_bucket",
LabelPairs: bucketLabels("5.000e-01...1.000e+00"),
Timestamps: []int64{1000, 2000},
Values: []float64{1, 2},
},
{
Name: "request_duration_seconds_bucket",
LabelPairs: bucketLabels("1.000e+00...2.000e+00"),
Timestamps: []int64{1000, 2000},
Values: []float64{3, 6},
},
})
}
func TestParseHistograms(t *testing.T) {
c := chunkenc.NewHistogramChunk()
app, err := c.Appender()
if err != nil {
t.Fatalf("cannot create chunk appender: %s", err)
}
if _, _, _, err := app.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
t.Fatalf("cannot append histogram: %s", err)
}
if _, _, _, err := app.AppendHistogram(nil, 0, 2000, testHistogram(2), true); err != nil {
t.Fatalf("cannot append histogram: %s", err)
}
hSamples, err := parseHistograms(prompb.Chunk_HISTOGRAM, c.Bytes())
if err != nil {
t.Fatalf("cannot parse histogram chunk: %s", err)
}
if len(hSamples) != 2 {
t.Fatalf("unexpected number of histogram samples; got %d; want 2", len(hSamples))
}
for i, expected := range []struct {
timestamp int64
count float64
sum float64
}{
{timestamp: 1000, count: 10, sum: 25.5},
{timestamp: 2000, count: 20, sum: 51},
} {
if hSamples[i].timestamp != expected.timestamp {
t.Fatalf("unexpected timestamp; got %d; want %d", hSamples[i].timestamp, expected.timestamp)
}
if hSamples[i].fh.Count != expected.count {
t.Fatalf("unexpected count; got %f; want %f", hSamples[i].fh.Count, expected.count)
}
if hSamples[i].fh.Sum != expected.sum {
t.Fatalf("unexpected sum; got %f; want %f", hSamples[i].fh.Sum, expected.sum)
}
}
// unsupported chunk encoding must return error
if _, err := parseHistograms(prompb.Chunk_XOR, c.Bytes()); err == nil {
t.Fatalf("expecting non-nil error for unsupported chunk encoding")
}
}
func TestProcessResponse(t *testing.T) {
readResp := &prompb.ReadResponse{
Results: []*prompb.QueryResult{
{
Timeseries: []*prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "cpu_usage"},
{Name: "job", Value: "bar"},
},
Samples: []prompb.Sample{
{Timestamp: 1000, Value: 1.5},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "request_duration_seconds"},
{Name: "job", Value: "bar"},
},
Histograms: []prompb.Histogram{
prompb.FromIntHistogram(1000, testHistogram(1)),
},
},
},
},
},
}
data, err := proto.Marshal(readResp)
if err != nil {
t.Fatalf("cannot marshal ReadResponse: %s", err)
}
compressed := snappy.Encode(nil, data)
var tss []*vm.TimeSeries
err = processResponse(io.NopCloser(bytes.NewReader(compressed)), func(ts *vm.TimeSeries) error {
tss = append(tss, ts)
return nil
})
if err != nil {
t.Fatalf("cannot process response: %s", err)
}
// 1 float series + _count + _sum + 4 buckets
if len(tss) != 7 {
t.Fatalf("unexpected number of time series; got %d; want 7", len(tss))
}
if tss[0].Name != "cpu_usage" || !reflect.DeepEqual(tss[0].Values, []float64{1.5}) {
t.Fatalf("unexpected float series: %v", tss[0])
}
if tss[1].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[1].Values, []float64{10}) {
t.Fatalf("unexpected _count series: %v", tss[1])
}
if tss[2].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[2].Values, []float64{25.5}) {
t.Fatalf("unexpected _sum series: %v", tss[2])
}
for _, ts := range tss[3:] {
if ts.Name != "request_duration_seconds_bucket" {
t.Fatalf("unexpected bucket series name %q", ts.Name)
}
}
}
type nopFlusher struct{}
func (nopFlusher) Flush() {}
func TestProcessStreamResponse(t *testing.T) {
// build a histogram chunk
hc := chunkenc.NewHistogramChunk()
hApp, err := hc.Appender()
if err != nil {
t.Fatalf("cannot create histogram chunk appender: %s", err)
}
if _, _, _, err := hApp.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
t.Fatalf("cannot append histogram: %s", err)
}
// build a float chunk
xc := chunkenc.NewXORChunk()
xApp, err := xc.Appender()
if err != nil {
t.Fatalf("cannot create xor chunk appender: %s", err)
}
xApp.Append(0, 1000, 1.5)
res := &prompb.ChunkedReadResponse{
ChunkedSeries: []*prompb.ChunkedSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "request_duration_seconds"},
{Name: "job", Value: "bar"},
},
Chunks: []prompb.Chunk{
{Type: prompb.Chunk_HISTOGRAM, Data: hc.Bytes()},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "cpu_usage"},
},
Chunks: []prompb.Chunk{
{Type: prompb.Chunk_XOR, Data: xc.Bytes()},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "memory_usage"},
},
Chunks: []prompb.Chunk{
// the `type` field may be unset for XOR chunks,
// such chunks must be parsed as XOR ones
{Type: prompb.Chunk_UNKNOWN, Data: xc.Bytes()},
},
},
},
}
data, err := proto.Marshal(res)
if err != nil {
t.Fatalf("cannot marshal ChunkedReadResponse: %s", err)
}
var buf bytes.Buffer
cw := remote.NewChunkedWriter(&buf, nopFlusher{})
if _, err := cw.Write(data); err != nil {
t.Fatalf("cannot write chunked response: %s", err)
}
var tss []*vm.TimeSeries
err = processStreamResponse(io.NopCloser(&buf), func(ts *vm.TimeSeries) error {
tss = append(tss, ts)
return nil
})
if err != nil {
t.Fatalf("cannot process stream response: %s", err)
}
// _count + _sum + 4 buckets + 1 float series + 1 float series from UNKNOWN chunk
if len(tss) != 8 {
t.Fatalf("unexpected number of time series; got %d; want 8", len(tss))
}
if tss[0].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[0].Values, []float64{10}) {
t.Fatalf("unexpected _count series: %v", tss[0])
}
if tss[1].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[1].Values, []float64{25.5}) {
t.Fatalf("unexpected _sum series: %v", tss[1])
}
for _, ts := range tss[2:6] {
if ts.Name != "request_duration_seconds_bucket" {
t.Fatalf("unexpected bucket series name %q", ts.Name)
}
}
if tss[6].Name != "cpu_usage" || !reflect.DeepEqual(tss[6].Values, []float64{1.5}) {
t.Fatalf("unexpected float series: %v", tss[6])
}
if tss[7].Name != "memory_usage" || !reflect.DeepEqual(tss[7].Values, []float64{1.5}) {
t.Fatalf("unexpected float series from UNKNOWN chunk: %v", tss[7])
}
}
func TestParseFloatHistograms(t *testing.T) {
c := chunkenc.NewFloatHistogramChunk()
app, err := c.Appender()
if err != nil {
t.Fatalf("cannot create chunk appender: %s", err)
}
fh := testHistogram(1).ToFloat(nil)
if _, _, _, err := app.AppendFloatHistogram(nil, 0, 1000, fh, true); err != nil {
t.Fatalf("cannot append float histogram: %s", err)
}
hSamples, err := parseHistograms(prompb.Chunk_FLOAT_HISTOGRAM, c.Bytes())
if err != nil {
t.Fatalf("cannot parse float histogram chunk: %s", err)
}
if len(hSamples) != 1 {
t.Fatalf("unexpected number of histogram samples; got %d; want 1", len(hSamples))
}
if hSamples[0].timestamp != 1000 {
t.Fatalf("unexpected timestamp; got %d; want 1000", hSamples[0].timestamp)
}
if hSamples[0].fh.Count != 10 {
t.Fatalf("unexpected count; got %f; want 10", hSamples[0].fh.Count)
}
}

View File

@@ -369,10 +369,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
}
return true
case "/tags/delSeries":
if r.Method != "POST" {
http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
return true
}
if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
return true
}
@@ -392,10 +388,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
}
return true
case "/api/v1/admin/tsdb/delete_series":
if r.Method != "POST" {
http.Error(w, fmt.Sprintf("Only POST method is allowed. Got %s.", r.Method), http.StatusMethodNotAllowed)
return true
}
if !httpserver.CheckAuthFlag(w, r, deleteAuthKey) {
return true
}

View File

@@ -59,7 +59,7 @@ func (ms *PrometheusMockStorage) Read(_ context.Context, query *prompb.Query, so
}
if !notMatch {
q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples, Histograms: s.Histograms})
q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples})
}
}

View File

@@ -12,7 +12,6 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/prompb"
"github.com/prometheus/prometheus/storage/remote"
@@ -87,17 +86,10 @@ func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
samples = append(samples, sample)
}
}
var histograms []prompb.Histogram
for _, h := range s.Histograms {
if h.Timestamp >= startTs && h.Timestamp < endTs {
histograms = append(histograms, h)
}
}
var series prompb.TimeSeries
if len(samples) > 0 || len(histograms) > 0 {
if len(samples) > 0 {
series.Labels = s.Labels
series.Samples = samples
series.Histograms = histograms
}
ts[i] = &series
}
@@ -325,37 +317,6 @@ func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64)
return samples
}
// GenerateRemoteReadHistogramSeries generates a remote read series
// with native histogram samples within the given time range.
func GenerateRemoteReadHistogramSeries(start, end, numOfSamples int64) []*prompb.TimeSeries {
timeSeries := &prompb.TimeSeries{
Labels: []prompb.Label{
{Name: labels.MetricName, Value: "vm_histogram_metric"},
{Name: "job", Value: "0"},
},
}
delta := (end - start) / numOfSamples
mul := int64(0)
for t := start; t != end; t += delta {
mul++
h := &histogram.Histogram{
Schema: 0,
Count: uint64(10 * mul),
Sum: 25.5 * float64(mul),
ZeroThreshold: 0.001,
ZeroCount: uint64(2 * mul),
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}},
PositiveBuckets: []int64{mul, 2 * mul},
NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}},
NegativeBuckets: []int64{4 * mul},
}
timeSeries.Histograms = append(timeSeries.Histograms, prompb.FromIntHistogram(t*1000, h))
}
return []*prompb.TimeSeries{timeSeries}
}
func labelsToLabelsProto(ls labels.Labels) []prompb.Label {
result := make([]prompb.Label, 0, ls.Len())
ls.Range(func(l labels.Label) {

View File

@@ -75,118 +75,6 @@ func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
}
func TestSingleVmctlRemoteReadNativeHistograms(t *testing.T) {
fs.MustRemoveDir(t.Name())
tc := apptest.NewTestCase(t)
defer tc.Stop()
vmsingleDst := tc.MustStartDefaultVmsingle()
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
vmctlFlags := []string{
`remote-read`,
`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
`--remote-read-step-interval=minute`,
`--vm-addr=` + vmAddr,
`--disable-progress-bar=true`,
}
testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadServer, vmctlFlags)
}
func TestSingleVmctlRemoteReadStreamNativeHistograms(t *testing.T) {
fs.MustRemoveDir(t.Name())
tc := apptest.NewTestCase(t)
defer tc.Stop()
vmsingleDst := tc.MustStartDefaultVmsingle()
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
vmctlFlags := []string{
`remote-read`,
`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
`--remote-read-step-interval=minute`,
`--vm-addr=` + vmAddr,
`--remote-read-use-stream=true`,
`--disable-progress-bar=true`,
}
testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadStreamServer, vmctlFlags)
}
// testRemoteReadNativeHistograms verifies that native histograms are migrated
// as _count, _sum and _bucket series with vmrange labels in the same way
// as VictoriaMetrics converts native histograms received via Prometheus remote write protocol.
func testRemoteReadNativeHistograms(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T, series []*prompb.TimeSeries) *RemoteReadServer, vmctlFlags []string) {
t := tc.T()
t.Helper()
series := GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2)
rrs := newRemoteReadServer(t, series)
defer rrs.Close()
vmctlFlags = append(vmctlFlags, `--remote-read-src-addr=`+rrs.HTTPAddr())
tc.MustStartVmctl("vmctl", vmctlFlags)
sut.ForceFlush(t)
tc.Assert(&apptest.AssertOptions{
Retries: 300,
Msg: `unexpected native histogram metrics stored on vmsingle via the prometheus protocol`,
Got: func() any {
got := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
Start: "2025-06-11T15:31:10Z",
End: "2025-06-11T15:32:20Z",
})
got.Sort()
return got.Data.Result
},
Want: expectedNativeHistogramQueryResult(),
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
// expectedNativeHistogramQueryResult returns the series expected to be stored in VictoriaMetrics
// after migrating the series generated by GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2).
func expectedNativeHistogramQueryResult() []*apptest.QueryResult {
metric := func(name, vmrange string) map[string]string {
m := map[string]string{
"__name__": name,
"job": "0",
}
if vmrange != "" {
m["vmrange"] = vmrange
}
return m
}
samples := func(v1, v2 float64) []*apptest.Sample {
return []*apptest.Sample{
{Timestamp: 1749655870000, Value: v1},
{Timestamp: 1749655875000, Value: v2},
}
}
resp := &apptest.PrometheusAPIV1QueryResponse{
Data: &apptest.QueryData{
Result: []*apptest.QueryResult{
{Metric: metric("vm_histogram_metric_count", ""), Samples: samples(10, 20)},
{Metric: metric("vm_histogram_metric_sum", ""), Samples: samples(25.5, 51)},
{Metric: metric("vm_histogram_metric_bucket", "-1.000e+00...-5.000e-01"), Samples: samples(4, 8)},
{Metric: metric("vm_histogram_metric_bucket", "-1.000e-03...1.000e-03"), Samples: samples(2, 4)},
{Metric: metric("vm_histogram_metric_bucket", "5.000e-01...1.000e+00"), Samples: samples(1, 2)},
{Metric: metric("vm_histogram_metric_bucket", "1.000e+00...2.000e+00"), Samples: samples(3, 6)},
},
},
}
// sort in the same way as the exported result
resp.Sort()
return resp.Data.Result
}
func testRemoteReadProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
t := tc.T()
t.Helper()

View File

@@ -118,6 +118,27 @@ To visualize and interact with both [self-monitoring metrics](https://docs.victo
- {{% available_from "v1.26.0" anomaly %}} For rapid exploration of how different models, their configurations and included domain knowledge impacts the results of anomaly detection, use the built-in [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
## Is vmanomaly stateful?
By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
## Config hot-reloading
`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
## Environment variables
`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
## Deploying vmanomaly
`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
## Migration
For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
## Choosing the right model for vmanomaly
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect:
@@ -134,6 +155,16 @@ Please refer to [respective blogpost on anomaly types and alerting heuristics](h
Still not 100% sure what to use? We are [here to help](https://docs.victoriametrics.com/anomaly-detection/#get-in-touch).
## Can AI help configure vmanomaly?
Yes. The available tools serve different workflows:
- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
## Incorporating domain knowledge
Anomaly detection models can significantly improve when incorporating business-specific assumptions about the data and what constitutes an anomaly. `vmanomaly` supports various [business-side configuration parameters](https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args) across all built-in models to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** and **align model behavior with business needs**, for example:
@@ -191,15 +222,39 @@ models:
provide_series: ['anomaly_score', 'y', 'yhat', 'yhat_lower', 'yhat_upper']
```
## Can AI help configure vmanomaly?
## Alert generation in vmanomaly
While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
Yes. The available tools serve different workflows:
<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
```yaml
groups:
- name: VMAnomalyAlerts
interval: 60s
rules:
- alert: HighAnomalyScore
expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
for: 5m # adjust to your needs based on data frequency and alerting policies
labels:
severity: warning
query_alias: explore
model_alias: default
scheduler_alias: periodic
preset: ui
annotations:
summary: High anomaly score detected.
description: Anomaly score exceeded threshold ({{ $value }}) for more than
{{ $for }} for query {{ $labels.for }}.
```
> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
## Preventing alert fatigue
Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
## How to backtest particular configuration on historical data?
@@ -362,61 +417,6 @@ groups:
description: "Disk usage is forecasted to exceed 95% in the next 3 days for instance {{ $labels.instance }}. Forecasted value: {{ $value }}."
```
## Alert generation in vmanomaly
While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where respective alerting tool, like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/), can be used to create alerts based on these scores for integrating it with your alerting management system. See an example diagram of how `vmanomaly` integrates into observability pipeline for anomaly detection on `node_exporter` metrics:
<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
Once anomaly scores are written back to VictoriaMetrics, you can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions in `vmalert` to define alerting rules based on these scores. Reasonable defaults are based around default threshold of `anomaly_score > 1`:
```yaml
groups:
- name: VMAnomalyAlerts
interval: 60s
rules:
- alert: HighAnomalyScore
expr: min(anomaly_score) without (model_alias, scheduler_alias) >= 1
for: 5m # adjust to your needs based on data frequency and alerting policies
labels:
severity: warning
query_alias: explore
model_alias: default
scheduler_alias: periodic
preset: ui
annotations:
summary: High anomaly score detected.
description: Anomaly score exceeded threshold ({{ $value }}) for more than
{{ $for }} for query {{ $labels.for }}.
```
> {{% available_from "v1.27.0" anomaly %}} You can also use the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to generate alerting rules automatically based on your model configurations and selected thresholds.
> {{% available_from "v1.28.3" anomaly %}} Check out our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on setting up alerting rules based on produced anomaly scores. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
## Preventing alert fatigue
Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) can use [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
## Deploying vmanomaly
`vmanomaly` can be deployed in various environments, including Docker, Kubernetes, and VM Operator. For detailed deployment instructions, refer to the [QuickStart section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
## Environment variables
`vmanomaly` supports {{% available_from "v1.25.0" anomaly %}} an option to reference environment variables in [configuration files](https://docs.victoriametrics.com/anomaly-detection/components/) using scalar string placeholders `%{ENV_NAME}`. This feature is particularly useful for managing sensitive information like API keys or database credentials while still making it accessible to the service. Please refer to the [environment variables section](https://docs.victoriametrics.com/anomaly-detection/components/#environment-variables) for more details and examples.
## Is vmanomaly stateful?
By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state. Also, [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) works well with state restoration, allowing for on-the-fly configuration changes without losing the current state of the models and reusing unchanged models/data/scheduler combinations.
Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) for more details on how it works and how to configure it.
## Config hot-reloading
`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
## Migration
For information on migrating between different versions of `vmanomaly`, please refer to the [Migration section](https://docs.victoriametrics.com/anomaly-detection/migration/) for compatibility considerations and steps for a smooth transition.
## Resource consumption of vmanomaly
`vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), and the complexity of the employed [models](https://docs.victoriametrics.com/anomaly-detection/components/models/). Its resource usage is directly related to these factors, making it adaptable to various operational scales. Various optimizations are available to balance between RAM usage, processing speed, and model capacity. These options are described in the sections below.

View File

@@ -27,7 +27,7 @@ Key functions:
The diagram below illustrates how `vmanomaly` fits into an observability setup, such as detecting anomalies in metrics collected by `node_exporter`:
<img src="/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.svg" alt="Typical vmanomaly observability pipeline using node-exporter, vmagent, VictoriaMetrics, Grafana, vmalert, and Alertmanager" style="width:60%"/>
<img src="https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="node_exporter_example_diagram" style="width:60%"/>
## How does it work?
@@ -40,7 +40,7 @@ VictoriaMetrics Anomaly Detection **continuously re-fit and apply machine learni
- **Confidence intervals** (`[yhat_lower, yhat_upper]`)
These outputs integrate seamlessly into downstream applications, making it easier to **visually inspect anomalies**, e.g. in respective [Grafana dashboards](https://docs.victoriametrics.com/anomaly-detection/presets/#grafana-dashboard).
{{% content "components/vmanomaly-components-diagram.md" %}}
<img src="https://docs.victoriametrics.com/anomaly-detection/components/vmanomaly-components.webp" alt="node_exporter_example_diagram" style="width:80%"/>
## Key benefits

View File

@@ -78,7 +78,9 @@ These [sub-configurations](#sub-configuration) can be assigned to a specific sha
Additionally, a replication factor `R ≥ 1` ensures [high availability](#high-availability) by enforcing redundancy across shards.
{{% content "vmanomaly-sharding-ha-diagram.md" %}}
<p></p>
![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
> Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).
@@ -128,7 +130,9 @@ Similar to other VictoriaMetrics ecosystem components, like [VMAgent](https://do
When `VMANOMALY_REPLICATION_FACTOR` > 1, each [sub-config](#sub-configuration) `n` from `{0, N-1}` is assigned to exactly `R` nodes. This ensures redundancy, preventing single-node failures from causing data loss.
{{% content "vmanomaly-sharding-ha-diagram.md" %}}
<p></p>
![vmanomaly-sharding-ha-diagram](vmanomaly-sharding-ha-diagram.webp)
> Please [refer to deployment options section](#deployment-options) for the examples (Docker, Docker Compose, Helm). To avoid duplicate metrics being reported from each vmanomaly service used in sharded mode, make sure that [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) is configured on vmsingle or vmselect and vmstorage for the VictoriaMetrics instance used in the [writer section of the configuration](https://docs.victoriametrics.com/anomaly-detection/components/writer/).

View File

@@ -28,7 +28,7 @@ Below, you will find an example illustrating how the components of `vmanomaly` i
> [Reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [Writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer) also support [multitenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy), so you can read/write from/to different locations - see `tenant_id` param description.
{{% content "vmanomaly-components-diagram.md" %}}
![vmanomaly-components](vmanomaly-components.webp)
## Example config

View File

@@ -1,68 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 968 420" role="img" aria-labelledby="title description">
<title id="title">AutoTunedModel tuning and inference lifecycle</title>
<desc id="description">The tuning process tests and scores model candidates across n time-series splits until the trial count or timeout is reached. Each fold fits a candidate on training data and predicts anomalies on its validation segment. The best model is then used on inference data until the next fit call.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
<path d="M0 0 10 5 0 10Z" fill="#202124"/>
</marker>
<style>
text { font-family: Arial, Helvetica, sans-serif; fill: #202124; font-size: 18px; }
.line { fill: none; stroke: #202124; stroke-width: 2.5; }
.arrow { fill: none; stroke: #202124; stroke-width: 2.5; marker-end: url(#arrow); }
.dash { fill: none; stroke: #202124; stroke-width: 2.5; stroke-dasharray: 8 9; }
.box { fill: #fff; stroke: #202124; stroke-width: 1.5; }
</style>
</defs>
<rect width="968" height="420" fill="#fff"/>
<rect x="112" y="10" width="242" height="83" class="box"/>
<text x="233" y="38" text-anchor="middle">
<tspan x="233" dy="0">test and score candidates</tspan>
<tspan x="233" dy="21">until `n_trials` or `timeout`</tspan>
<tspan x="233" dy="21">is reached</tspan>
</text>
<path d="M354 49 V67" class="arrow"/>
<text x="355" y="90" text-anchor="middle">Tuning</text>
<text x="355" y="112" text-anchor="middle">process</text>
<path d="M162 157 V136 H549 V157" class="line"/>
<path d="M355 112 V136" class="line"/>
<path d="M399 93 H653 V152" class="line"/>
<path d="M113 161 H75 V350 H113" class="line"/>
<path d="M75 236 H39" class="line"/>
<text x="28" y="250" transform="rotate(-90 28 250)" text-anchor="middle">n splits</text>
<text x="102" y="227">Fold 1</text>
<text x="102" y="255">Fold 2</text>
<text x="102" y="283">Fold 3</text>
<text x="120" y="327">...</text>
<text x="260" y="179" text-anchor="middle">Fit</text>
<text x="260" y="202" text-anchor="middle">candidate</text>
<text x="413" y="179" text-anchor="middle">Predict</text>
<text x="413" y="202" text-anchor="middle">anomalies</text>
<path d="M173 221 H354 V207" class="line"/>
<path d="M232 249 H412 V235" class="line"/>
<path d="M293 280 H472 V265" class="line"/>
<path d="M354 221 H412" class="dash"/>
<path d="M412 249 H474" class="dash"/>
<path d="M472 280 H513" class="dash"/>
<path d="M548 144 V415" class="dash" style="stroke-width:1.5;stroke-dasharray:4 6"/>
<rect x="577" y="152" width="136" height="61" rx="12" class="box"/>
<text x="645" y="187" text-anchor="middle">best model</text>
<path d="M653 213 V348" class="arrow"/>
<text x="815" y="168" text-anchor="middle">
<tspan x="815" dy="0">used to predict</tspan>
<tspan x="815" dy="21">on inference data</tspan>
<tspan x="815" dy="21">until the next `fit` call</tspan>
</text>
<path d="M40 350 H895" class="arrow"/>
<text x="455" y="402" text-anchor="middle">Training data</text>
<text x="623" y="402" text-anchor="middle">Inference data</text>
<text x="856" y="402">Time axis, t</text>
</svg>

Before

Width:  |  Height:  |  Size: 3.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

View File

@@ -1,139 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
<title id="title">Multivariate models lifecycle</title>
<desc id="description">MetricsQL queries retrieve an aligned set of series from VictoriaMetrics. Reader data fits one shared multivariate model. The exact same series set produces one anomaly score series with the intersected label set; inference is skipped when the fit and inference series sets differ. Writer stores produced scores in VictoriaMetrics.</desc>
<defs>
<marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
</marker>
<marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
</marker>
<style>
text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
.title { font-size: 50px; font-weight: 400; }
.node { font-size: 34px; font-weight: 400; }
.label { font-size: 29px; }
.blue-label { font-size: 26px; }
.small { font-size: 25px; }
.box { fill: #fff; stroke: #303038; stroke-width: 3; }
.group { fill: #fff; stroke: #303038; stroke-width: 3; }
.model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
.line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
.blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
.blue { fill: #1478c9; }
</style>
</defs>
<rect width="1920" height="1295" fill="#fff"/>
<text x="20" y="70" class="title">Multivariate Models Lifecycle</text>
<!-- VictoriaMetrics and query configuration -->
<rect x="535" y="175" width="420" height="215" class="box"/>
<text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
<text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
<rect x="270" y="270" width="150" height="155" class="box"/>
<text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
<text x="285" y="360" class="label">MetricsQL</text>
<text x="285" y="395" class="label">queries</text>
<path d="M420 346 H535" class="line"/>
<!-- Reader and datasource exchange -->
<rect x="580" y="500" width="310" height="100" class="box"/>
<text x="735" y="562" class="node" text-anchor="middle">Reader</text>
<path d="M580 545 H455 V365 H535" class="line"/>
<text x="465" y="478" class="label">1. Request data</text>
<path d="M955 270 H1040 V550 H890" class="line"/>
<text x="810" y="478" class="label">2. Get metrics</text>
<!-- Historical fit data returned by the configured queries -->
<g aria-label="Historical fit data">
<rect x="1090" y="85" width="430" height="505" class="group"/>
<rect x="1120" y="135" width="340" height="170" class="group"/>
<text x="1140" y="180" class="node">Query 1</text>
<rect x="1135" y="195" width="300" height="42" class="box"/>
<text x="1150" y="228" class="label">Metric 1.1</text>
<text x="1150" y="262" class="label">...</text>
<rect x="1135" y="265" width="300" height="42" class="box"/>
<text x="1150" y="298" class="label">Metric 1.M₁</text>
<text x="1305" y="355" class="node" text-anchor="middle">...</text>
<rect x="1120" y="390" width="340" height="170" class="group"/>
<text x="1140" y="435" class="node">Query N</text>
<rect x="1135" y="450" width="300" height="42" class="box"/>
<text x="1150" y="483" class="label">Metric N.1</text>
<text x="1150" y="517" class="label">...</text>
<rect x="1135" y="520" width="300" height="42" class="box"/>
<text x="1150" y="553" class="label">Metric N.Mₙ</text>
</g>
<!-- One shared model is fitted on the complete aligned set -->
<rect x="1215" y="795" width="310" height="115" class="box"/>
<text x="1370" y="865" class="node" text-anchor="middle">Model</text>
<rect x="1365" y="635" width="155" height="130" class="box"/>
<text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
<text x="1382" y="723" class="label">Model</text>
<text x="1382" y="757" class="label">config</text>
<path d="M1305 590 V795" class="line"/>
<text x="1055" y="672" class="label">
<tspan x="1055" dy="0">3. Fit one model</tspan>
<tspan x="1055" dy="38">on all historical series</tspan>
</text>
<path d="M1443 765 V795" class="line"/>
<!-- Inference data must contain the same set of series -->
<g aria-label="Inference data">
<rect x="25" y="615" width="430" height="500" class="group"/>
<rect x="55" y="645" width="340" height="170" class="group"/>
<text x="75" y="690" class="node">Query 1</text>
<rect x="70" y="705" width="300" height="42" class="box"/>
<text x="85" y="738" class="label">Metric 1.1</text>
<text x="85" y="772" class="label">...</text>
<rect x="70" y="775" width="300" height="42" class="box"/>
<text x="85" y="808" class="label">Metric 1.M₁</text>
<text x="225" y="870" class="node" text-anchor="middle">...</text>
<rect x="55" y="900" width="340" height="175" class="group"/>
<text x="75" y="945" class="node">Query N</text>
<rect x="70" y="960" width="300" height="42" class="box"/>
<text x="85" y="993" class="label">Metric N.1</text>
<text x="85" y="1027" class="label">...</text>
<rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
<text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
</g>
<path d="M580 575 H500 V650 H455" class="line"/>
<text x="465" y="680" class="label">4. Provide inference data</text>
<!-- Single multivariate model registry -->
<g aria-label="Multivariate model registry">
<rect x="580" y="750" width="420" height="480" class="group"/>
<text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
<rect x="610" y="835" width="360" height="220" class="model-group"/>
<rect x="640" y="885" width="300" height="115" class="box"/>
<text x="790" y="955" class="node" text-anchor="middle">Model (single)</text>
</g>
<path d="M455 1050 H580" class="line"/>
<path d="M1215 852 H1000" class="line"/>
<!-- Joint output and mismatched-series skip path -->
<rect x="1600" y="1015" width="285" height="105" class="box"/>
<text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
<rect x="1740" y="805" width="145" height="145" class="box"/>
<text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
<text x="1758" y="900" class="label">Writer</text>
<text x="1758" y="935" class="label">config</text>
<path d="M1813 950 V1015" class="line"/>
<path d="M1000 1040 H1600" class="line"/>
<text x="1295" y="965" class="label" text-anchor="middle">
<tspan x="1295" dy="0">5.a Produce one anomaly-score series</tspan>
<tspan x="1295" dy="38">with the intersected label set</tspan>
</text>
<path d="M1000 1110 H1600" class="blue-line"/>
<text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
<tspan x="1320" dy="0">5.b Skip inference when the fit and inference</tspan>
<tspan x="1320" dy="32">series sets differ;</tspan>
<tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
</text>
<!-- Persist the joint anomaly score -->
<path d="M1743 1015 V80 H745 V175" class="line"/>
<text x="1170" y="55" class="label" text-anchor="middle">6. Write one anomaly-score series (label set = intersection)</text>
</svg>

Before

Width:  |  Height:  |  Size: 7.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

View File

@@ -1,148 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1295" role="img" aria-labelledby="title description">
<title id="title">Univariate models lifecycle</title>
<desc id="description">MetricsQL queries retrieve multiple series from VictoriaMetrics. Reader data fits one model per series in the model registry. Known series produce individual anomaly scores for Writer; an unseen series is skipped until a fitted model exists. Writer stores produced scores in VictoriaMetrics.</desc>
<defs>
<marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
</marker>
<marker id="arrow-blue" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#1478c9" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
</marker>
<style>
text { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
.title { font-size: 50px; font-weight: 400; }
.node { font-size: 34px; font-weight: 400; }
.label { font-size: 29px; }
.blue-label { font-size: 26px; }
.small { font-size: 25px; }
.box { fill: #fff; stroke: #303038; stroke-width: 3; }
.group { fill: #fff; stroke: #303038; stroke-width: 3; }
.model-group { fill: #fff; stroke: #303038; stroke-width: 3; stroke-dasharray: 14 12; }
.line { fill: none; stroke: #303038; stroke-width: 3; marker-end: url(#arrow); }
.blue-line { fill: none; stroke: #1478c9; stroke-width: 4; marker-end: url(#arrow-blue); }
.blue { fill: #1478c9; }
</style>
</defs>
<rect width="1920" height="1295" fill="#fff"/>
<text x="20" y="70" class="title">Univariate Models Lifecycle</text>
<!-- VictoriaMetrics and query configuration -->
<rect x="535" y="175" width="420" height="215" class="box"/>
<text x="745" y="265" class="node" text-anchor="middle">VictoriaMetrics TSDB</text>
<text x="745" y="310" class="node" text-anchor="middle">(Single-Node or Cluster)</text>
<rect x="270" y="270" width="150" height="155" class="box"/>
<text x="345" y="305" class="small" text-anchor="middle">Config.yml</text>
<text x="285" y="360" class="label">MetricsQL</text>
<text x="285" y="395" class="label">queries</text>
<path d="M420 346 H535" class="line"/>
<!-- Reader and datasource exchange -->
<rect x="580" y="500" width="310" height="100" class="box"/>
<text x="735" y="562" class="node" text-anchor="middle">Reader</text>
<path d="M580 545 H455 V365 H535" class="line"/>
<text x="465" y="478" class="label">1. Request data</text>
<path d="M955 270 H1040 V550 H890" class="line"/>
<text x="810" y="478" class="label">2. Get metrics</text>
<!-- Historical fit data returned by the configured queries -->
<g aria-label="Historical fit data">
<rect x="1090" y="85" width="430" height="505" class="group"/>
<rect x="1120" y="135" width="340" height="170" class="group"/>
<text x="1140" y="180" class="node">Query 1</text>
<rect x="1135" y="195" width="300" height="42" class="box"/>
<text x="1150" y="228" class="label">Metric 1.1</text>
<text x="1150" y="262" class="label">...</text>
<rect x="1135" y="265" width="300" height="42" class="box"/>
<text x="1150" y="298" class="label">Metric 1.M₁</text>
<text x="1305" y="355" class="node" text-anchor="middle">...</text>
<rect x="1120" y="390" width="340" height="170" class="group"/>
<text x="1140" y="435" class="node">Query N</text>
<rect x="1135" y="450" width="300" height="42" class="box"/>
<text x="1150" y="483" class="label">Metric N.1</text>
<text x="1150" y="517" class="label">...</text>
<rect x="1135" y="520" width="300" height="42" class="box"/>
<text x="1150" y="553" class="label">Metric N.Mₙ</text>
</g>
<!-- Model fitting -->
<rect x="1215" y="795" width="310" height="115" class="box"/>
<text x="1370" y="865" class="node" text-anchor="middle">Model</text>
<rect x="1365" y="635" width="155" height="130" class="box"/>
<text x="1443" y="675" class="small" text-anchor="middle">Config.yml</text>
<text x="1382" y="723" class="label">Model</text>
<text x="1382" y="757" class="label">config</text>
<path d="M1305 590 V795" class="line"/>
<text x="1055" y="672" class="label">
<tspan x="1055" dy="0">3. Fit models</tspan>
<tspan x="1055" dy="38">on historical data</tspan>
</text>
<path d="M1443 765 V795" class="line"/>
<!-- Inference data, including a new unseen series -->
<g aria-label="Inference data">
<rect x="25" y="615" width="430" height="500" class="group"/>
<rect x="55" y="645" width="340" height="170" class="group"/>
<text x="75" y="690" class="node">Query 1 (has fit model)</text>
<rect x="70" y="705" width="300" height="42" class="box"/>
<text x="85" y="738" class="label">Metric 1.1</text>
<text x="85" y="772" class="label">...</text>
<rect x="70" y="775" width="300" height="42" class="box"/>
<text x="85" y="808" class="label">Metric 1.M₁</text>
<text x="225" y="870" class="node" text-anchor="middle">...</text>
<rect x="55" y="900" width="340" height="175" class="group"/>
<text x="75" y="945" class="node">Query N</text>
<rect x="70" y="960" width="300" height="42" class="box"/>
<text x="85" y="993" class="label">Metric N.1</text>
<text x="85" y="1027" class="label">...</text>
<rect x="70" y="1030" width="300" height="42" fill="#fff" stroke="#1478c9" stroke-width="4"/>
<text x="85" y="1063" class="label blue">Metric N.Mₖ</text>
</g>
<path d="M580 575 H500 V650 H455" class="line"/>
<text x="465" y="680" class="label">4. Provide inference data</text>
<!-- One fitted model per known series -->
<g aria-label="Univariate model registry">
<rect x="580" y="750" width="420" height="480" class="group"/>
<text x="790" y="800" class="node" text-anchor="middle">Model registry</text>
<rect x="610" y="830" width="360" height="170" class="model-group"/>
<rect x="630" y="865" width="300" height="42" class="box"/>
<text x="645" y="898" class="label">Model 1.1</text>
<text x="645" y="932" class="label">...</text>
<rect x="630" y="935" width="300" height="42" class="box"/>
<text x="645" y="968" class="label">Model 1.M₁</text>
<text x="790" y="1045" class="node" text-anchor="middle">...</text>
<rect x="610" y="1070" width="360" height="135" class="model-group"/>
<rect x="630" y="1090" width="300" height="42" class="box"/>
<text x="645" y="1123" class="label">Model N.1</text>
<rect x="630" y="1145" width="300" height="42" class="box"/>
<text x="645" y="1178" class="label">Model N.Mₙ</text>
</g>
<path d="M455 1050 H580" class="line"/>
<path d="M1215 852 H1000" class="line"/>
<!-- Known-series output and unseen-series skip path -->
<rect x="1600" y="1015" width="285" height="105" class="box"/>
<text x="1743" y="1080" class="node" text-anchor="middle">Writer</text>
<rect x="1740" y="805" width="145" height="145" class="box"/>
<text x="1813" y="845" class="small" text-anchor="middle">Config.yml</text>
<text x="1758" y="900" class="label">Writer</text>
<text x="1758" y="935" class="label">config</text>
<path d="M1813 950 V1015" class="line"/>
<path d="M1000 1040 H1600" class="line"/>
<text x="1275" y="975" class="label" text-anchor="middle">
<tspan x="1275" dy="0">5.a Produce anomaly scores</tspan>
<tspan x="1275" dy="38">for known series</tspan>
</text>
<path d="M1000 1110 H1600" class="blue-line"/>
<text x="1320" y="1145" class="blue-label blue" text-anchor="middle">
<tspan x="1320" dy="0">5.b Skip inference until a fitted model exists</tspan>
<tspan x="1320" dy="32">for Metric N.Mₖ;</tspan>
<tspan x="1320" dy="32">update the model_runs_skipped counter</tspan>
</text>
<!-- Persist anomaly scores -->
<path d="M1743 1015 V80 H745 V175" class="line"/>
<text x="1130" y="55" class="label" text-anchor="middle">6. Write back produced anomaly scores</text>
</svg>

Before

Width:  |  Height:  |  Size: 8.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

View File

@@ -68,10 +68,6 @@ models:
Common arguments supported by every model were introduced in [v1.10.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1100).
<div class="collapse-group">
{{% collapse name="Queries" %}}
### Queries
The `queries` argument selects the [reader queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) used to fit and run a particular model{{% available_from "v1.10.0" anomaly %}}. Every series returned by a selected query is passed to that model.
@@ -97,10 +93,6 @@ models:
queries: ['q1', 'q2', 'q3'] # i.e., if your `queries` in `reader` section has exactly q1, q2, q3 aliases
```
{{% /collapse %}}
{{% collapse name="Schedulers" %}}
### Schedulers
The `schedulers` argument selects the [schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) that run a particular model{{% available_from "v1.11.0" anomaly %}}.
@@ -126,10 +118,6 @@ models:
schedulers: ['s1', 's2', 's3'] # i.e., if your `schedulers` section has exactly s1, s2, s3 aliases
```
{{% /collapse %}}
{{% collapse name="Provide series" %}}
### Provide series
The `provide_series` argument{{% available_from "v1.12.0" anomaly %}} limits the [model output](#vmanomaly-output) sent to the writer. For example, a model may produce `['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']` by default, while the following configuration writes only `anomaly_score` for each input series:
@@ -143,10 +131,6 @@ models:
> If `provide_series` is not specified in model config, the model will produce its default [model-dependent output](#vmanomaly-output). The output can't be less than `['anomaly_score']`. Even if `timestamp` column is omitted, it will be implicitly added to `provide_series` list, as it's required for metrics to be properly written.
{{% /collapse %}}
{{% collapse name="Detection direction" %}}
### Detection direction
The `detection_direction` argument{{% available_from "v1.13.0" anomaly %}} can reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive) when domain knowledge indicates that only values above or below the expected value are anomalous. Available values are `both`, `above_expected`, and `below_expected`.
@@ -206,10 +190,6 @@ reader:
# other components like writer, schedule, monitoring
```
{{% /collapse %}}
{{% collapse name="Minimal deviation from expected" %}}
### Minimal deviation from expected
`min_dev_from_expected`{{% available_from "v1.13.0" anomaly %}} argument is designed to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** in scenarios where deviations between the actual value (`y`) and the expected value (`yhat`) are **relatively** high. Such deviations can cause models to generate high [anomaly scores](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score). However, these deviations may not be significant enough in **absolute values** from a business perspective to be considered anomalies. This parameter ensures that anomaly scores for data points where `|y - yhat| < min_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it is set to `0` to maintain backward compatibility.
@@ -258,10 +238,6 @@ models:
queries: ['normal_behavior'] # use the default where it's not needed
```
{{% /collapse %}}
{{% collapse name="Minimal relative deviation from expected" %}}
### Minimal relative deviation from expected
{{% available_from "v1.29.1" anomaly %}} `min_rel_dev_from_expected` argument serves a similar purpose to `min_dev_from_expected` (see [section above](#minimal-deviation-from-expected)), but focuses on **relative deviations** rather than absolute ones. It is designed to reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive) in scenarios where the relative deviation between the actual value (`y`) and the expected value (`yhat`) is high, but the absolute deviation is not significant enough to be considered an anomaly from a business perspective. This parameter ensures that anomaly scores for data points where `|y - yhat| / |yhat| < min_rel_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it is set to `0` to maintain backward compatibility.
@@ -302,10 +278,6 @@ models:
```
{{% /collapse %}}
{{% collapse name="Group by" %}}
### Group by
> The `groupby` argument works only in combination with [multivariate models](#multivariate-models).
@@ -344,10 +316,6 @@ models:
groupby: [host]
```
{{% /collapse %}}
{{% collapse name="Scale" %}}
### Scale
Previously available only to [ProphetModel](#prophet) and [OnlineQuantileModel](#online-seasonal-quantile), the `scale` {{% available_from "v1.20.0" anomaly %}} parameter is now applicable to all models that support generating predictions (`yhat`, `yhat_lower`, `yhat_upper`). Also, it is **two-sided** now, represented as a list of two positive float values, allowing separate scaling for the intervals `[yhat, yhat_upper]` and `[yhat_lower, yhat]`. The new margins are calculated as:
@@ -378,10 +346,6 @@ models:
scale: [1.2, 0.75]
```
{{% /collapse %}}
{{% collapse name="Clip predictions" %}}
### Clip predictions
A post-processing step to **clip model predictions** (`yhat`, `yhat_lower`, and `yhat_upper` series) to the configured [`data_range` values](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) in `VmReader` is available.
@@ -436,10 +400,6 @@ models:
]
```
{{% /collapse %}}
{{% collapse name="Score outside data range" %}}
### Score outside data range
The `anomaly_score_outside_data_range` {{% available_from "v1.20.0" anomaly %}} parameter allows overriding the default **anomaly score (`1.01`)** assigned when actual values (`y`) fall **outside the defined `data_range` if defined in [reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/)**. This provides greater flexibility for **alerting rule configurations** and enables **clearer visual differentiation** between different types of anomalies:
@@ -485,10 +445,6 @@ models:
anomaly_score_outside_data_range: 3.0
```
{{% /collapse %}}
{{% collapse name="Decay" %}}
### Decay
> The `decay` argument works only in combination with [online models](#online-models) like [`ZScoreOnlineModel`](#online-z-score) or [`OnlineQuantileModel`](#online-seasonal-quantile).
@@ -521,10 +477,6 @@ models:
queries: ['q1']
```
{{% /collapse %}}
</div>
## Model types
@@ -550,7 +502,7 @@ If during an inference, you got a series having **new labelset** (not present in
**Examples:** [Prophet](#prophet), [Holt-Winters](#holt-winters)
![Univariate model lifecycle](model-lifecycle-univariate.svg)
![vmanomaly-model-type-univariate](model-lifecycle-univariate.webp)
### Multivariate Models
@@ -565,34 +517,9 @@ If during an inference, you got a **different amount of series** or some series
**Implications:** Multivariate models are a go-to default, when your queries returns **fixed** amount of **individual** time series (say, some aggregations), to be used for adding cross-series (and cross-query) context, useful for catching [collective anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#collective-anomalies) or [novelties](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#novelties) (expanded to multi-input scenario). For example, you may set it up for anomaly detection of CPU usage in different modes (`idle`, `user`, `system`, etc.) and use its cross-dependencies to detect **unseen (in fit data)** behavior.
**Examples:** [Temporal Envelope](#temporal-envelope), [Isolation Forest](#isolation-forest-multivariate)
**Examples:** [IsolationForest](#isolation-forest-multivariate)
![Multivariate model lifecycle](model-lifecycle-multivariate.svg)
The following configuration applies both models to the same aligned input series. Start with Temporal Envelope when temporal profiles and online adaptation matter; use Isolation Forest as an offline alternative when feature-space outliers are the primary concern.
```yaml
models:
service_dependency_envelope:
class: temporal_envelope_multivariate
queries: [request_rate, error_rate, latency]
groupby: [cluster]
dependency_rank: 8
score_aggregation: l2
seasonalities: [hod_smooth, dow_smooth]
provide_series: [anomaly_score]
service_dependency_isolation_forest:
class: isolation_forest_multivariate
queries: [request_rate, error_rate, latency]
groupby: [cluster]
contamination: 0.01
seasonal_features: [hod, dow]
args:
n_estimators: 100
random_state: 42
provide_series: [anomaly_score]
```
![vmanomaly-model-type-multivariate](model-lifecycle-multivariate.webp)
### Online Models
@@ -737,7 +664,7 @@ models:
</div>
![AutoTunedModel tuning and inference lifecycle](autotune.svg)
![vmanomaly-autotune-schema](autotune.webp)
#### Shared asynchronous autotune workflow
@@ -1417,10 +1344,6 @@ This guide shows how to:
> The file containing the model must be written in [Python](https://www.python.org/) 3.14 or later. A custom model runs inside the `vmanomaly` Python environment, so keep its dependencies compatible with the target image and keep the module available when restoring serialized model state after a restart.
<div class="collapse-group">
{{% collapse name="Custom model implementation guide" %}}
### 1. Custom model
Create `custom_model.py` with a `CustomModel` class derived from `Model`. A concrete model must implement:
@@ -1587,10 +1510,6 @@ The writer emits one `custom_anomaly_score` series for each input series. It ret
{__name__="custom_anomaly_score", for="churn_rate", model_alias="custom_model", scheduler_alias="s1", run="test-format"}
```
{{% /collapse %}}
</div>
## Deprecations
{{% collapse name="Deprecated model types and models" %}}

View File

@@ -24,10 +24,6 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
- Adding `preset` and `scheduler_alias` keys to [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics) and [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics) metrics for consistency in multi-[scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) setups.
- Renaming [Counters](https://prometheus.io/docs/concepts/metric_types/#counter) `vmanomaly_reader_response_count` to `vmanomaly_reader_responses` and `vmanomaly_writer_response_count` to `vmanomaly_writer_responses`.
<div class="collapse-group">
{{% collapse name="Pull model config parameters" %}}
## Pull Model Config parameters
<table class="params">
@@ -64,10 +60,6 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
</tbody>
</table>
{{% /collapse %}}
{{% collapse name="Push config parameters" %}}
## Push Config parameters
By default, metrics are pushed only after the completion of specific stages, e.g., `fit`, `infer`, or `fit_infer` (for each [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) if using a multi-scheduler configuration).
@@ -235,10 +227,6 @@ Path to a file with the client certificate key, i.e. `client.key`{{% available_f
</tbody>
</table>
{{% /collapse %}}
</div>
## Monitoring section config example
``` yaml
@@ -272,10 +260,6 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
- [Model metrics](#models-behaviour-metrics)
- [Writer metrics](#writer-behaviour-metrics)
<div class="collapse-group">
{{% collapse name="Startup metrics" %}}
### Startup metrics
<table class="params">
@@ -401,10 +385,6 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
[Back to metric sections](#metrics-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Reader behaviour metrics" %}}
### Reader behaviour metrics
Label names [description](#labelnames)
@@ -529,10 +509,6 @@ Label names [description](#labelnames)
[Back to metric sections](#metrics-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Models behaviour metrics" %}}
### Models behaviour metrics
Label names [description](#labelnames)
@@ -659,10 +635,6 @@ Label names [description](#labelnames)
[Back to metric sections](#metrics-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Writer behaviour metrics" %}}
### Writer behaviour metrics
Label names [description](#labelnames)
@@ -774,10 +746,6 @@ Label names [description](#labelnames)
[Back to metric sections](#metrics-generated-by-vmanomaly)
{{% /collapse %}}
</div>
### Labelnames
* `stage` - model execution stage: `fit`, `infer`, or `fit_infer` for a combined fit/inference scheduler run. See [model types](https://docs.victoriametrics.com/anomaly-detection/components/models/#model-types).
@@ -825,10 +793,6 @@ and the [command-line arguments](https://docs.victoriametrics.com/anomaly-detect
- [Query server and task logs](#query-server-and-task-logs)
- [AI Copilot logs](#ai-copilot-logs)
<div class="collapse-group">
{{% collapse name="Startup logs" %}}
### Startup logs
@@ -848,10 +812,6 @@ server addresses, hot-reload state, and active schedulers. The most useful prefi
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Reader logs" %}}
### Reader logs
Reader logs cover endpoint checks, request splitting, network failures, response parsing, and coordination between
@@ -903,10 +863,6 @@ or parsed. See [reader behaviour metrics](#reader-behaviour-metrics).
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Service logs" %}}
### Service logs
The service logs `fit`, `infer`, and combined `fit_infer`/backtesting work for each model alias and scheduler.
@@ -935,10 +891,6 @@ an unsuccessful stage. See [models behaviour metrics](#models-behaviour-metrics)
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Writer logs" %}}
### Writer logs
Writer logs cover serialization and delivery of produced series such as
@@ -966,10 +918,6 @@ and datapoints are recorded only after a successful response. See [writer behavi
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Scheduler supervision logs" %}}
### Scheduler supervision logs
Scheduler supervision{{% available_from "v1.30.0" anomaly %}} logs a dead worker, automatic restart, successful
@@ -979,10 +927,6 @@ Correlate them with `vmanomaly_scheduler_alive` and `vmanomaly_scheduler_restart
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Hot-reload logs" %}}
### Hot-reload logs
Hot reload logs config-change detection, validation, staged service restart, success, and rollback. `Reload aborted
@@ -992,10 +936,6 @@ without restarting services`.
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Persisted-state logs" %}}
### Persisted-state logs
With `settings.restore_state`, startup logs the stored/runtime version assessment, reusable components, required
@@ -1004,10 +944,6 @@ stored artifacts completely` indicates a full reset; missing or unreadable model
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="Query server and task logs" %}}
### Query server and task logs
The query server logs its listening address and datasource-proxy timeouts/failures. Background anomaly-detection
@@ -1016,10 +952,6 @@ background raw query finishing cleanly.
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
{{% collapse name="AI Copilot logs" %}}
### AI Copilot logs
AI Copilot{{% available_from "v1.30.0" anomaly %}} reports whether it is initialized, disabled, misconfigured, or
@@ -1028,7 +960,3 @@ request failed` identifies provider execution failure, and `MCP server unreachab
guidance tools.
[Back to logging sections](#logs-generated-by-vmanomaly)
{{% /collapse %}}
</div>

View File

@@ -25,8 +25,6 @@ Use the following playgrounds to develop and test input queries:
## VM reader
<div class="collapse-group">
{{% collapse name="Queries format migration (to v1.13.0+)" %}}
> The backward-compatible `queries` format introduced in v1.13.0 allows [VmReader](#vm-reader) parameters such as `step` to be configured per query. This can reduce the amount of data read from VictoriaMetrics. See [per-query parameters](#per-query-parameters) for details.
@@ -63,8 +61,6 @@ reader:
```
{{% /collapse %}}
{{% collapse name="VM reader per-query parameters and example" %}}
### Per-query parameters
There is change {{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg format. Now each query alias supports the next (sub)fields, which *override reader-level parameters*, if set:
@@ -129,10 +125,6 @@ reader:
offset: '-15s' # to override reader-wise `offset` and query data 15 seconds earlier to account for data collection delays
```
{{% /collapse %}}
{{% collapse name="VM reader config parameters and example" %}}
### Config parameters
<table class="params">
@@ -515,16 +507,10 @@ reader:
series_processing_batch_size: 8
```
{{% /collapse %}}
</div>
### MetricsQL Playground
To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMetrics MetricsQL Playground](https://play.victoriametrics.com/), which provides an interactive environment to test and visualize your queries against sample data. You can also access embedded version of the playground below:
<div class="collapse-group">
{{% collapse name="VictoriaMetrics Playground" %}}
<div class="position-relative mb-3">
@@ -550,8 +536,6 @@ To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMe
{{% /collapse %}}
</div>
### mTLS protection
`vmanomaly` supports [mutual TLS (mTLS)](https://en.wikipedia.org/wiki/Mutual_authentication){{% available_from "v1.16.3" anomaly %}} for secure communication across its components, including [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters). This allows for mutual authentication between the client and server when querying or writing data to [VictoriaMetrics Enterprise, configured for mTLS](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#mtls-protection).
@@ -696,8 +680,6 @@ Similarly, [VictoriaTraces LogsQL Playground](https://play-vtraces.victoriametri
You can also access **embedded version of the playground below** (VictoriaLogs datasource):
<div class="collapse-group">
{{% collapse name="VictoriaLogs LogsQL Playground" %}}
<div class="position-relative mb-3">
@@ -723,11 +705,6 @@ You can also access **embedded version of the playground below** (VictoriaLogs d
{{% /collapse %}}
</div>
<div class="collapse-group">
{{% collapse name="VictoriaLogs reader config parameters" %}}
### Config parameters
@@ -1012,10 +989,6 @@ Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far last-seen
</tbody>
</table>
{{% /collapse %}}
{{% collapse name="VictoriaLogs reader per-query parameters and example" %}}
### Per-query parameters
The names, types and the logic of the per-query parameters subset used in `VLogsReader` are exactly the same as those of [`VmReader`](#vm-reader), please see [per-query parameters](#per-query-parameters) section above for the details. The only difference is that `expr` parameter should contain a valid [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) expression with `stats` [pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe), as described in [query examples](#query-examples) section above.
@@ -1063,10 +1036,6 @@ reader:
# other config sections, like models, schedulers, writer, ...
```
{{% /collapse %}}
</div>
### mTLS protection
Please refer to the [mTLS protection](#mtls-protection) section above for details on how to configure mTLS for `VLogsReader`. It uses the same config parameters as `VmReader` for mTLS setup.

View File

@@ -226,10 +226,6 @@ monitoring:
# other monitoring settings
```
<div class="collapse-group">
{{% collapse name="State restoration example" %}}
### Example
For a configuration with the following models, queries and schedulers:
@@ -309,10 +305,6 @@ This means that the service upon restart:
1. Won't restore the state of `zscore_online` model, because its `z_threshold` argument **has changed**, retraining from scratch is needed on the last `fit_window` = 24 hours of data for `q1`, `q2` and `q3` (as model's `queries` arg is not set so it defaults to all queries found in the reader).
2. Will **partially** restore the state of `prophet` model, because its class and schedulers are unchanged, but **only instances trained on timeseries returned by `q1` query**. New fit/infer jobs will be set for new query `q3`. The old query `q2` artifacts will be dropped upon restart - all respective models and data for (`prophet`, `q2`) combination will be removed from the database file and from the disk.
{{% /collapse %}}
</div>
## Retention
{{% available_from "v1.28.1" anomaly %}} The `retention` argument sets a [time to live](https://en.wikipedia.org/wiki/Time_to_live) (TTL) for service artifacts such as stored model instances and training data. At each `check_interval`, the service removes artifacts that have not been used for inference or refitting within `ttl`. This bounds stale resource usage in long-running deployments.

View File

@@ -1,15 +0,0 @@
---
build:
list: never
publishResources: false
render: never
sitemap:
disable: true
---
The required path is `config.yml` → Scheduler → Reader → Model → Writer. The Reader queries the configured VictoriaMetrics, VictoriaLogs, or VictoriaTraces datasource; the Writer stores inferred anomaly scores in VictoriaMetrics. Monitoring is optional and can push metrics or expose them for collection.
Solid nodes and arrows show the required anomaly-detection path. Dashed nodes and arrows show optional self-monitoring integrations.
![vmanomaly component interaction: the Scheduler starts Reader, Model, and Writer work against a configured datasource; Monitoring is optional](/anomaly-detection/components/vmanomaly-components.svg)
{style="display:block; width:80%; min-width:320px; margin:1.5rem auto"}

View File

@@ -1,152 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1920 1600" role="img" aria-labelledby="title description">
<title id="title">How vmanomaly operates during a scheduled iteration</title>
<desc id="description">The required flow starts from config.yml and the Scheduler. The Reader queries either VictoriaMetrics through query_range or VictoriaLogs and VictoriaTraces through stats_query_range, then sends data to a Model. The Model produces anomaly scores, and the Writer stores them in VictoriaMetrics through import. Reader, Model, and Writer can optionally report self-monitoring metrics using push or pull monitoring.</desc>
<defs>
<marker id="arrow" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#303038" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
</marker>
<marker id="arrow-optional" viewBox="0 0 12 12" refX="10.5" refY="6" markerWidth="16" markerHeight="16" markerUnits="userSpaceOnUse" orient="auto-start-reverse">
<path d="M1.5 1.5 10.5 6 1.5 10.5" fill="none" stroke="#666a73" stroke-width="2.25" stroke-linecap="round" stroke-linejoin="round"/>
</marker>
<style>
.label { font-family: Arial, Helvetica, sans-serif; fill: #303038; }
.title { font-size: 54px; font-weight: 400; }
.service-title { font-size: 46px; font-weight: 400; }
.node-text { font-size: 34px; font-weight: 600; text-anchor: middle; }
.body { font-size: 28px; }
.edge-label { font-size: 25px; }
.endpoint-text { font-size: 23px; }
.source-text { font-size: 21px; }
.required-node { fill: #fff; stroke: #303038; stroke-width: 4; }
.optional-node { fill: #fff; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
.required-line { fill: none; stroke: #303038; stroke-width: 4; marker-end: url(#arrow); }
.optional-line { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; marker-end: url(#arrow-optional); }
.boundary { fill: none; stroke: #303038; stroke-width: 4; }
.optional-boundary { fill: none; stroke: #666a73; stroke-width: 4; stroke-dasharray: 14 12; }
</style>
</defs>
<rect width="1920" height="1600" fill="#fff"/>
<text x="105" y="72" class="label title">How does vmanomaly operate (scheduled iteration example)</text>
<!-- Required / optional legend -->
<g aria-label="Legend" transform="translate(1425 145)">
<rect x="0" y="0" width="135" height="62" class="optional-node"/>
<rect x="205" y="0" width="135" height="62" class="required-node"/>
<path d="M0 100 H135" class="optional-line"/>
<path d="M205 100 H340" class="required-line"/>
<text x="67" y="162" class="label body" text-anchor="middle">Optional</text>
<text x="272" y="162" class="label body" text-anchor="middle">Required</text>
</g>
<!-- Exactly one configured read datasource is used. -->
<g aria-label="Configured datasource choice">
<rect x="25" y="315" width="300" height="550" class="optional-boundary"/>
<g transform="translate(34 326)">
<path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
<ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
<rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
<text x="144" y="99" class="label endpoint-text" text-anchor="middle">/query_range</text>
<text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
<text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
<text x="144" y="182" class="label source-text" text-anchor="middle">TSDB</text>
</g>
<text x="178" y="600" class="label body" text-anchor="middle">OR</text>
<g transform="translate(34 632)">
<path d="M8 32 V184 C8 224 280 224 280 184 V32 C280 70 8 70 8 32Z" class="required-node"/>
<ellipse cx="144" cy="32" rx="136" ry="33" class="required-node"/>
<rect x="30" y="72" width="228" height="66" rx="16" class="required-node"/>
<text x="144" y="99" class="label endpoint-text" text-anchor="middle">/stats_query_range</text>
<text x="144" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
<text x="144" y="158" class="label source-text" text-anchor="middle">VictoriaLogs /</text>
<text x="144" y="182" class="label source-text" text-anchor="middle">VictoriaTraces</text>
</g>
</g>
<!-- vmanomaly service boundary and components -->
<g aria-label="vmanomaly service">
<rect x="720" y="145" width="610" height="1145" class="boundary"/>
<rect x="1005" y="175" width="270" height="100" class="required-node"/>
<text x="1140" y="235" class="label node-text">config.yml</text>
<text x="1025" y="352" class="label service-title" text-anchor="middle">vmanomaly</text>
<text x="1025" y="405" class="label service-title" text-anchor="middle">service</text>
<rect x="765" y="448" width="445" height="76" rx="16" class="required-node"/>
<text x="987" y="500" class="label node-text">Scheduler</text>
<rect x="765" y="590" width="445" height="76" rx="16" class="required-node"/>
<text x="987" y="642" class="label node-text">Reader</text>
<rect x="765" y="732" width="445" height="76" rx="16" class="required-node"/>
<text x="987" y="784" class="label node-text">Model</text>
<rect x="765" y="874" width="445" height="76" rx="16" class="required-node"/>
<text x="987" y="926" class="label node-text">Writer</text>
<rect x="778" y="1180" width="420" height="82" rx="16" class="optional-node"/>
<text x="988" y="1235" class="label node-text">Monitoring</text>
<path d="M1005 225 H900 V448" class="required-line"/>
<path d="M1210 486 H1290 V628 H1210" class="required-line"/>
<path d="M1210 628 H1290 V770 H1210" class="required-line"/>
<path d="M1210 770 H1290 V912 H1210" class="required-line"/>
<path d="M875 666 V1178" class="optional-line"/>
<path d="M987 808 V1178" class="optional-line"/>
<path d="M1100 950 V1178" class="optional-line"/>
</g>
<!-- Required flow labels -->
<text x="1365" y="455" class="label body">
<tspan x="1365" dy="0">1. Get the metrics to</tspan>
<tspan x="1365" dy="37">a. fit the model, or</tspan>
<tspan x="1365" dy="37">b. produce anomaly scores</tspan>
</text>
<text x="1365" y="620" class="label body">
<tspan x="1365" dy="0">3. Model receives data</tspan>
<tspan x="1365" dy="37">to train or infer on</tspan>
</text>
<text x="1365" y="775" class="label body">
<tspan x="1365" dy="0">4. Produce anomaly</tspan>
<tspan x="1365" dy="37">scores (inference)</tspan>
</text>
<!-- Datasource request and response -->
<path d="M765 610 H338" class="required-line"/>
<path d="M338 652 H765" class="required-line"/>
<text x="370" y="580" class="label edge-label">2.1 Query request</text>
<text x="370" y="700" class="label edge-label">2.2 Metrics response</text>
<!-- Anomaly-score output -->
<g aria-label="VictoriaMetrics write endpoint" transform="translate(26 1145)">
<path d="M8 32 V184 C8 224 290 224 290 184 V32 C290 70 8 70 8 32Z" class="required-node"/>
<ellipse cx="149" cy="32" rx="141" ry="33" class="required-node"/>
<rect x="34" y="72" width="230" height="66" rx="16" class="required-node"/>
<text x="149" y="99" class="label endpoint-text" text-anchor="middle">/import</text>
<text x="149" y="126" class="label endpoint-text" text-anchor="middle">endpoint</text>
<text x="149" y="158" class="label source-text" text-anchor="middle">VictoriaMetrics</text>
<text x="149" y="182" class="label source-text" text-anchor="middle">TSDB</text>
</g>
<path d="M765 912 H650 V1095 L316 1197" class="required-line"/>
<rect x="338" y="1018" width="292" height="74" fill="#fff"/>
<text x="350" y="1048" class="label edge-label">
<tspan x="340" dy="0">5. Write anomaly scores</tspan>
<tspan x="340" dy="33">to VictoriaMetrics</tspan>
</text>
<!-- Optional monitoring integrations -->
<rect x="748" y="1083" width="490" height="48" fill="#fff"/>
<text x="993" y="1116" class="label edge-label" text-anchor="middle">6. Produce self-monitoring metrics</text>
<rect x="390" y="1460" width="300" height="118" rx="45" class="optional-node"/>
<text x="540" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
<text x="540" y="1552" class="label body" text-anchor="middle">push approach</text>
<rect x="1490" y="1460" width="300" height="118" rx="45" class="optional-node"/>
<text x="1640" y="1515" class="label body" text-anchor="middle">Monitoring system</text>
<text x="1640" y="1552" class="label body" text-anchor="middle">pull approach</text>
<path d="M850 1262 L540 1460" class="optional-line"/>
<path d="M1198 1222 L1640 1460" class="optional-line"/>
<rect x="590" y="1350" width="180" height="42" fill="#fff"/>
<text x="680" y="1380" class="label edge-label" text-anchor="middle">Push metrics</text>
<rect x="1230" y="1340" width="415" height="42" fill="#fff"/>
<text x="1438" y="1370" class="label edge-label" text-anchor="middle">HTTP GET /metrics or /health</text>
</svg>

Before

Width:  |  Height:  |  Size: 9.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 107 KiB

View File

@@ -17,10 +17,6 @@ Future updates will introduce additional export methods, offering users more fle
## VM writer
<div class="collapse-group">
{{% collapse name="VM writer config parameters and example" %}}
### Config parameters
<table class="params">
@@ -39,7 +35,8 @@ Future updates will introduce additional export methods, offering users more fle
</td>
<td>
`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
<span style="white-space: nowrap;">`writer.vm.VmWriter` or `vm`{{% available_from "v1.13.0" anomaly %}}
</span>
</td>
<td>
@@ -66,7 +63,10 @@ Datasource URL address
<span style="white-space: nowrap;">`tenant_id`</span>
</td>
<td>
<span>
`0:0`, `multitenant`{{% available_from "v1.16.2" anomaly %}}
</span>
</td>
<td>
@@ -253,8 +253,9 @@ Token is passed in the standard format with header: `Authorization: bearer {toke
`path_to_file`
</td>
<td>
<span>
Path to a file, which contains token, that is passed in the standard format with header: `Authorization: bearer {token}`{{% available_from "v1.15.9" anomaly %}}
</td>
</span> </td>
</tr>
<tr>
<td>
@@ -266,8 +267,9 @@ Path to a file, which contains token, that is passed in the standard format with
`1`
</td>
<td>
<span>
Number of attempts to retry the connection in case of failure {{% available_from "v1.29.2" anomaly %}}.
</td>
</span> </td>
</tr>
</tbody>
</table>
@@ -291,10 +293,6 @@ writer:
connection_retry_attempts: 2 # if not specified, it will be 1 by default
```
{{% /collapse %}}
</div>
### Multitenancy support
> This feature applies to the VictoriaMetrics Cluster version only. Tenants are identified by either `accountID` or `accountID:projectID`. `multitenant` [endpoint](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels){{% available_from "v1.15.9" anomaly %}} is supported for writing data across multiple [tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy). For more details, refer to the VictoriaMetrics Cluster [multitenancy documentation](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).

View File

@@ -17,7 +17,7 @@ sitemap:
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
![Typical vmanomaly observability pipeline](guide-vmanomaly-vmalert_overview.svg)
![typical setup diagram](guide-vmanomaly-vmalert_overview.webp)
> **Configurations used throughout this guide can be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-integration/)**

View File

@@ -1,65 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1706 1069" role="img" aria-labelledby="title description">
<title id="title">Typical vmanomaly observability pipeline</title>
<desc id="description">vmagent scrapes node-exporter metrics and writes them to VictoriaMetrics. vmanomaly reads those metrics and writes anomaly scores back. Grafana visualizes the results. vmalert evaluates rules based on anomaly scores and sends alerts to Alertmanager.</desc>
<defs>
<marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="10" markerHeight="10" orient="auto-start-reverse">
<path d="M0 0 10 5 0 10Z" fill="#252525"/>
</marker>
<style>
text { font-family: Arial, Helvetica, sans-serif; fill: #252525; font-size: 40px; }
.box { fill: #fff; stroke: #252525; stroke-width: 3; }
.arrow { fill: none; stroke: #252525; stroke-width: 4; marker-end: url(#arrow); }
</style>
</defs>
<rect width="1706" height="1069" fill="#fff"/>
<rect x="20" y="55" width="390" height="100" class="box"/>
<text x="215" y="119" text-anchor="middle">node-exporter</text>
<rect x="730" y="50" width="395" height="100" class="box"/>
<text x="928" y="115" text-anchor="middle">vmagent</text>
<path d="M710 100 H435" class="arrow"/>
<text x="570" y="56" text-anchor="middle">Scrape metrics</text>
<rect x="745" y="325" width="400" height="180" class="box"/>
<text x="945" y="405" text-anchor="middle">VictoriaMetrics</text>
<text x="945" y="457" text-anchor="middle">TSDB</text>
<path d="M945 150 V322" class="arrow"/>
<text x="1018" y="204">
<tspan x="1018" dy="0">Push node</tspan>
<tspan x="1018" dy="50">exporter</tspan>
<tspan x="1018" dy="50">metrics</tspan>
</text>
<rect x="25" y="340" width="395" height="100" class="box"/>
<text x="222" y="405" text-anchor="middle">vmanomaly</text>
<path d="M725 365 H440" class="arrow"/>
<text x="575" y="319" text-anchor="middle">Read metrics</text>
<path d="M440 421 H725" class="arrow"/>
<text x="570" y="508" text-anchor="middle">
<tspan x="570" dy="0">Write produced</tspan>
<tspan x="570" dy="50">anomaly scores</tspan>
</text>
<rect x="1440" y="330" width="245" height="170" class="box"/>
<text x="1562" y="430" text-anchor="middle">Grafana</text>
<path d="M1160 417 H1418" class="arrow"/>
<text x="1285" y="486" text-anchor="middle">
<tspan x="1285" dy="0">Visualize the</tspan>
<tspan x="1285" dy="50">results</tspan>
</text>
<rect x="750" y="738" width="395" height="100" class="box"/>
<text x="948" y="805" text-anchor="middle">vmalert</text>
<path d="M946 525 V716" class="arrow"/>
<text x="1000" y="644">
<tspan x="1000" dy="0">Evaluate rules based</tspan>
<tspan x="1000" dy="50">on anomaly scores</tspan>
</text>
<rect x="755" y="948" width="395" height="100" class="box"/>
<text x="952" y="1015" text-anchor="middle">alertmanager</text>
<path d="M948 858 V928" class="arrow"/>
<text x="1004" y="906">Send alerts</text>
</svg>

Before

Width:  |  Height:  |  Size: 3.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

View File

@@ -1,107 +0,0 @@
# Render with D2 v0.7.1:
# d2 --layout elk --theme 0 --pad 20 vmanomaly-sharding-ha-diagram.d2 vmanomaly-sharding-ha-diagram.svg
grid-columns: 1
grid-gap: 36
classes: {
boundary: {
style.fill: "#F7F7F8"
style.stroke: "#A7AAB2"
style.stroke-width: 2
}
process: {
style.fill: "#FFFFFF"
style.stroke: "#303038"
style.stroke-width: 2
style.border-radius: 8
}
member: {
style.fill: "#F1F3F5"
style.stroke: "#59616A"
style.stroke-width: 2
style.border-radius: 8
}
}
flow: "" {
grid-columns: 3
grid-gap: 36
style.fill: transparent
style.stroke: transparent
global: "Global YAML\nconfiguration" {
shape: page
class: process
}
splitting: Configuration splitting {
class: boundary
grid-columns: 1
grid-gap: 24
split: "Split by VMANOMALY_SPLIT_BY\ndefault: complete" {
class: process
}
subconfigs: "N valid sub-configurations\nn = 0 ... N-1" {
class: process
style.multiple: true
}
split -> subconfigs: {
style.stroke: "#303038"
}
}
placement: Deterministic placement {
class: boundary
grid-columns: 1
grid-gap: 24
settings: "K members: VMANOMALY_MEMBERS_COUNT\nMember index: VMANOMALY_MEMBER_NUM\nR replicas: VMANOMALY_REPLICATION_FACTOR" {
class: process
}
assign: "Assign every sub-configuration\nto exactly R members" {
class: process
}
settings -> assign: {
style.stroke: "#303038"
}
}
}
members: K vmanomaly members {
class: boundary
grid-columns: 3
grid-gap: 30
member0: "Member 0\nvmanomaly service 1" {
class: member
}
member1: "Member 1\nvmanomaly service 2" {
class: member
}
memberK: "Member K-1\nvmanomaly service K" {
class: member
}
}
flow.global -> flow.splitting.split: {
style.stroke: "#303038"
}
flow.splitting.subconfigs -> flow.placement.assign: {
style.stroke: "#303038"
}
flow.placement.assign -> members.member0: "assigned subset" {
style.stroke: "#59616A"
style.stroke-width: 2
}
flow.placement.assign -> members.member1: "assigned subset" {
style.stroke: "#59616A"
style.stroke-width: 2
}
flow.placement.assign -> members.memberK: "assigned subset" {
style.stroke: "#59616A"
style.stroke-width: 2
}

View File

@@ -1,13 +0,0 @@
---
build:
list: never
publishResources: false
render: never
sitemap:
disable: true
---
The global configuration is split into `N` independently valid sub-configurations. Deterministic placement distributes them across `K` members, and each sub-configuration is assigned to exactly `R` members when replication is enabled. Member `k` processes only its assigned subset.
![vmanomaly sharding and high availability: a global configuration is split into N sub-configurations and replicated across K members](/anomaly-detection/vmanomaly-sharding-ha-diagram.svg)
{style="display:block; width:50%; min-width:320px; margin:1.5rem auto"}

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 87 KiB

View File

@@ -39,8 +39,6 @@ VictoriaMetrics has the following prominent features:
* Easy and fast backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
can be done with [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) / [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/) tools.
See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
* It supports storage and retrieval of samples with timestamps that fall within the `[1970-01-02T00:00:00.000Z, 2262-03-31T23:59:59.999Z]` time range with millisecond precision.
See [Retention](#retention) for details.
* It implements a PromQL-like query language - [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/), which provides improved functionality on top of PromQL.
* It provides a global query view. Multiple Prometheus instances or any other data sources may ingest data into VictoriaMetrics. Later this data may be queried via a single query.
* It provides high performance and good vertical and horizontal scalability for both
@@ -1542,9 +1540,6 @@ It is safe to extend `-retentionPeriod` on existing data. If `-retentionPeriod`
value than before, then data outside the configured period will be eventually deleted.
VictoriaMetrics does not support indefinite retention, but you can specify an arbitrarily high duration, e.g. `-retentionPeriod=100y`.
Just keep in mind that VictoriaMetrics does not support samples with negative timestamps. Timestamps from `1970-01-01` are also not
supported because this date has a special meaning internally. It therefore will reject samples with timestamps before
`1970-01-02T00:00:00.000Z`.
By default, VictoriaMetrics doesn't accept samples with timestamps bigger than `now+2d`, e.g. 2 days in the future.
If you need accepting samples with bigger timestamps, then specify the desired "future retention" via `-futureRetention` command-line flag.
@@ -1556,9 +1551,6 @@ For example, the following command starts VictoriaMetrics, which accepts samples
/path/to/victoria-metrics -futureRetention=1y
```
VictoriaMetrics does not support stamples after `2262-03-31T23:59:59.999Z`. If the future retention includes dates after this timestamp,
the samples for those dates will be rejected.
By default, VictoriaMetrics accepts samples with timestamps as old as the configured `-retentionPeriod` allows, e.g. it accepts backfilled
historical data as long as it fits into the retention. If you need rejecting samples with historical timestamps older than the specified
duration, then specify the desired duration via the `-maxBackfillAge` command-line flag. This can be useful for limiting ingestion of

View File

@@ -28,23 +28,13 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
**Update Note 1:** `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the default value of `-disableRerouting` flag has changed from `true` to `false`, enabling [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. If you rely on the old behavior, pass `-disableRerouting` command-line flag to `vminsert`. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): the `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints now require `POST` method. Previously, it also accepted `GET` requests. If you use `GET` requests for this endpoint, update your scripts or tooling to use `POST` instead. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
* SECURITY: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): restrict `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints to `POST` method only to prevent some [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery)-based data deletion attacks. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-remoteWrite.obfuscateLabels` flag for hashing values of the specified labels before sending metrics to the corresponding `-remoteWrite.url`. This allows sharing metrics with external systems while keeping sensitive label values hidden. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-replay.continueWithExecutionErr` flag to allow continuing to replay other rules when a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit. See [11313](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11313).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add template variable `$interval` to expose the alerting rule group's evaluation interval. This allows generating dashboard links with a lookback window relative to the rule's interval, for example `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`. See this issue [#11232](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11232) for more details. Thanks to @1solomonwakhungu for contribution.
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`,
`NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support [Prometheus native histograms](https://prometheus.io/docs/specs/native_histograms/) migration in [remote read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/). Native histograms are converted into `_count`, `_sum` and `_bucket` series with `vmrange` labels in the same way as VictoriaMetrics converts native histograms received via Prometheus remote write protocol, except that for native histograms with custom buckets the original bucket bounds are preserved instead of being estimated with the exponential formula. Previously native histograms were silently ignored in `SAMPLES` mode, while in stream mode the migration failed with `EOF` error. See [#11292](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11292). Thanks to @liuxu623 for contribution.
* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enable [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Previously, `-disableRerouting` defaulted to `true`, which limited ingestion throughput to the slowest `vmstorage` node. Now `-disableRerouting` defaults to `false`, so `vminsert` automatically routes data away from the slowest `vmstorage` node, improving overall ingestion performance. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the selected auto-refresh interval in the URL. See [VictoriaLogs#1310](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1310).
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): ignore HTTP proxy environment variables when scraping targets over Unix domain sockets. See [#11318](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11318). Thanks to @lwmacct for contribution.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
@@ -253,6 +243,7 @@ Released at 2026-04-24
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect evaluation of binary operations caused by an ordering bug (e.g. `10 - (3 + 3 + 4)` being evaluated as `10 - 3 + 3 + 4`). The issue was introduced in v1.140.0, v1.136.4, and v1.122.19. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
## [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0)
Released at 2026-04-10
**Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): [CSV export](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-csv-data) (`/api/v1/export/csv`) now adds a header row as the first line of the response, so existing CSV-processing scripts may need to skip this header. See [#10666](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10666).

View File

@@ -323,8 +323,7 @@ flowchart TB
H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
H5 --> H6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
H6 --> H7[[push to <b>-remoteWrite.url</b>]]
H5 --> H6[[push to <b>-remoteWrite.url</b>]]
%% Right branch
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
@@ -332,8 +331,7 @@ flowchart TB
R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
R5 --> R6[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values">obfuscate labels</a><br><b>-remoteWrite.obfuscateLabels</b>]
R6 --> R7[[push to <b>-remoteWrite.url</b>]]
R5 --> R6[[push to <b>-remoteWrite.url</b>]]
```
Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:
@@ -685,35 +683,6 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
/path/to/vmagent -remoteWrite.url=http://127.0.0.1:8428/api/v1/write?extra_label="env=prod"
```
## Obfuscating label values
`vmagent` can obfuscate the values of specified labels before sending metrics to `-remoteWrite.url`
via `-remoteWrite.obfuscateLabels`{{% available_from "#" %}}.
This is useful when one or more `-remoteWrite.url` endpoints point to external monitoring services
outside the organization, and sensitive label values such as `ip`, `host`, `instance`, or `datacenter`
must not be exposed.
Label values are replaced with their SHA-256 hex digests. No salt is applied, so values with a small
or predictable value space can potentially be recovered by brute force.
This feature pairs well with [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange):
use `-remoteWrite.mdx.enable=true` to forward only VictoriaMetrics self-monitoring metrics to an
external vendor while hiding sensitive label values with `-remoteWrite.obfuscateLabels`.
Use `-remoteWrite.obfuscateLabels` to list label names whose values should be obfuscated for the
corresponding `-remoteWrite.url`. Separate multiple label names with `^^`.
```sh
./vmagent \
-remoteWrite.url=http://<external-service1> \
-remoteWrite.obfuscateLabels='instance^^datacenter' \
-remoteWrite.url=http://<external-service2> \
-remoteWrite.obfuscateLabels='instance' \
-remoteWrite.url=http://<internal-service> \
-remoteWrite.obfuscateLabels=''
```
## Automatically generated metrics
`vmagent` automatically generates the following metrics for each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)

View File

@@ -341,7 +341,6 @@ The following variables are available in templating:
| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | `Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})` |
| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | `Visit {{ $externalURL }} for more details` |
| $isPartial or .IsPartial | Indicates whether the latest rule query response from the datasource(that supports returning `isPartial` option, such as vmcluster) could be partial. | `{{ if $isPartial }}WARNING: The latest alert state may be a false alarm due to a partial response from the datasource.{{ end }}` |
| $interval or .Interval | Alerting rule group's evaluation interval. | `http://vm-grafana.com/<dashboard-id>?viewPanel=<panel-id>&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}` |
Additionally, `vmalert` provides some extra templating functions listed in [template functions](#template-functions) and [reusable templates](#reusable-templates).

View File

@@ -404,8 +404,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
Optional TLS server name to use for connections to -remoteWrite.url. By default, the server name from -remoteWrite.url is used
-remoteWrite.url string
Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
-replay.continueWithExecutionErr bool
Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
-replay.disableProgressBar
Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
-replay.maxDatapointsPerQuery int
@@ -413,7 +411,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
-replay.ruleEvaluationConcurrency int
The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery. (default 1)
-replay.ruleRetryAttempts int
Defines how many retries to make before giving up on rule if request for it returns a retriable error. (default 5)
Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
-replay.rulesDelay duration
Delay before evaluating the next rule within the group. Is important for chained rules. Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting. (default 1s)
-replay.timeFrom string

View File

@@ -90,11 +90,7 @@ func newClient(ctx context.Context, sw *ScrapeWork) (*client, error) {
}
tr := httputil.NewTransport(false, "vm_promscrape")
if sw.UnixSocket != "" {
// Unix sockets are direct local endpoints and cannot be reached via HTTP proxies.
// Proxy could be set implicitly via global env variable HTTP_PROXY
tr.Proxy = nil
} else if proxyURLFunc != nil {
if proxyURLFunc != nil {
tr.Proxy = proxyURLFunc
}
tr.TLSHandshakeTimeout = 10 * time.Second

View File

@@ -1118,14 +1118,6 @@ func searchAndMerge[T any](qt *querytracer.Tracer, s *Storage, tr TimeRange, sea
qt = qt.NewChild("search indexDBs: timeRange=%v", &tr)
defer qt.Done()
var zeroValue T
if tr.MinTimestamp < minUnixMilli {
tr.MinTimestamp = minUnixMilli
}
if tr.MaxTimestamp < tr.MinTimestamp {
return zeroValue, nil
}
var idbts []indexDBWithType
ptws := s.tb.GetPartitions(tr)

View File

@@ -402,10 +402,6 @@ func TestStorageDeletePendingSeries(t *testing.T) {
defer testRemoveAll(t)
const numMonths = 10
start := time.Date(1971, 1, 1, 0, 0, 0, 0, time.UTC)
middle := start.AddDate(0, (numMonths-1)/2, 0)
end := start.AddDate(0, numMonths-1, 0)
s := MustOpenStorage(t.Name(), OpenOptions{})
var metricGroupName = []byte("metric")
@@ -460,7 +456,7 @@ func TestStorageDeletePendingSeries(t *testing.T) {
assertCountMonthsWithLabels := func(count int) {
t.Helper()
ts := start
ts := time.Unix(0, 0)
n := 0
for range numMonths {
lns, err := s.SearchLabelNames(nil, nil, TimeRange{ts.UnixMilli(), ts.UnixMilli()}, 1e5, 1e9, noDeadline)
@@ -485,7 +481,7 @@ func TestStorageDeletePendingSeries(t *testing.T) {
var search Search
defer search.MustClose()
search.Init(nil, s, []*TagFilters{tfs}, TimeRange{start.UnixMilli(), math.MaxInt64}, 1e5, noDeadline)
search.Init(nil, s, []*TagFilters{tfs}, TimeRange{0, math.MaxInt64}, 1e5, noDeadline)
n := 0
for search.NextMetricBlock() {
var b Block
@@ -502,6 +498,10 @@ func TestStorageDeletePendingSeries(t *testing.T) {
// Verify no metrics exist
assertCountRows(0)
start := time.Unix(0, 0)
middle := start.AddDate(0, (numMonths-1)/2, 0)
end := start.AddDate(0, numMonths-1, 0)
// Add some rows and flush, so next DeleteSeries() can delete them
addRows(start, middle, false)
s.DebugFlush()
@@ -2843,8 +2843,9 @@ func TestStorageAdjustTimeRange(t *testing.T) {
}
var searchTimeRange TimeRange
// Search time range is the same as globalIndexTimeRange.
searchTimeRange = globalIndexTimeRange
// Zero search time range is adjusted to globalIndexTimeRange regardless
// whether the -disablePerDayIndex flag is set or not.
searchTimeRange = TimeRange{}
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
@@ -2852,6 +2853,9 @@ func TestStorageAdjustTimeRange(t *testing.T) {
// The search time range is smaller than a month (and therefore < 40 days)
// and is fully included into the partition idb time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange. Otherwise it must remain the same
// after the adjustment.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
@@ -2862,6 +2866,11 @@ func TestStorageAdjustTimeRange(t *testing.T) {
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
// The search time range is the same as partition idb time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb: it must remain the same
// - For the partition idb: it must be replaced with globalIndexTimeRange.
searchTimeRange = partitionIDBTimeRange
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
@@ -2870,6 +2879,11 @@ func TestStorageAdjustTimeRange(t *testing.T) {
// The search time range is smaller than 40 days and fully includes the
// partition idb time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb: it must remain the same
// - For the partition idb: it must be replaced with globalIndexTimeRange.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
@@ -2881,6 +2895,10 @@ func TestStorageAdjustTimeRange(t *testing.T) {
// The search time range is 41 days and fully includes the partition idb
// time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise it must be replaced with globalIndexTimeRange for both legacy
// and partition idbs.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + 41*msecPerDay,
@@ -2892,6 +2910,12 @@ func TestStorageAdjustTimeRange(t *testing.T) {
// The search time range is smaller than 40 days and overlaps with partition
// idb time range on the left.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb: it must remain the same
// - For the partition idb: the MinTimestamp must be adjusted to match the
// partition idb time range MinTimestamp.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
@@ -2906,6 +2930,12 @@ func TestStorageAdjustTimeRange(t *testing.T) {
// The search time range is smaller than 40 days and overlaps with partition
// idb time range on the right.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb, it must remain the same
// - For the partition idb: its MaxTimestamp must be adjusted to match the
// partition idb time range MaxTimestamp.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
@@ -2919,7 +2949,7 @@ func TestStorageAdjustTimeRange(t *testing.T) {
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
}
type testStorageSearchWithoutIndexOptions struct {
type testStorageSearchWithoutPerDayIndexOptions struct {
mrs []MetricRow
assertSearchResult func(t *testing.T, s *Storage, tr TimeRange, want any)
alwaysPerTimeRange bool // If true, use wantPerTimeRange instead of wantAll
@@ -2928,15 +2958,16 @@ type testStorageSearchWithoutIndexOptions struct {
wantEmpty any
}
// testStorageSearchWithoutIndex tests how the search behaves when the
// testStorageSearchWithoutPerDayIndex tests how the search behaves when the
// per-day index is disabled. This function is expected to be called by
// functions that test a particular search operation, such as GetTSDBStatus(),
// SearchMetricNames(), etc.
func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutIndexOptions) {
func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWithoutPerDayIndexOptions) {
defer testRemoveAll(t)
// The data is inserted and the search is performed when per-day index is enabled.
t.Run("Add-Global-PerDay/Search-Global-PerDay", func(t *testing.T) {
// The data is inserted and the search is performed when the per-day index
// is enabled.
t.Run("InsertAndSearchWithPerDayIndex", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: false,
})
@@ -2948,8 +2979,9 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
s.MustClose()
})
// The data is inserted and the search is performed when per-day index is disabled.
t.Run("Add-Global-noPerDay/Search-Global-noPerDay", func(t *testing.T) {
// The data is inserted and the search is performed when the per-day index
// is disabled.
t.Run("InsertAndSearchWithoutPerDayIndex", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: true,
})
@@ -2964,9 +2996,9 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
s.MustClose()
})
// The data is inserted when per-day index are enabled.
// The search is performed when per-day index is disabled.
t.Run("Add-Global-PerDay/Search-Global-noPerDay", func(t *testing.T) {
// The data is inserted when the per-day index is enabled but the search is
// performed when the per-day index is disabled.
t.Run("InsertWithPerDayIndexSearchWithout", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: false,
})
@@ -2986,11 +3018,10 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
s.MustClose()
})
// The data is inserted when per-day index is disabled.
// The search is performed when per-day index is enabled.
// This case also shows that registering metric names recovers the per-day
// index.
t.Run("Add-Global-noPerDay/Search-Global-PerDay", func(t *testing.T) {
// The data is inserted when the per-day index is disabled but the search is
// performed when the per-day index is enabled. This case also shows that
// registering metric names recovers the per-day index.
t.Run("InsertWithoutPerDayIndexSearchWith", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: true,
})
@@ -3017,13 +3048,13 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
})
}
func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutIndexOptions{
opts := testStorageSearchWithoutPerDayIndexOptions{
wantEmpty: &TSDBStatus{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: &TSDBStatus{TotalSeries: days * rows},
@@ -3063,16 +3094,16 @@ func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
}
}
testStorageSearchWithoutIndex(t, &opts)
testStorageSearchWithoutPerDayIndex(t, &opts)
}
func TestStorageSearchMetricNamesWithoutIndex(t *testing.T) {
func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutIndexOptions{
opts := testStorageSearchWithoutPerDayIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3124,16 +3155,16 @@ func TestStorageSearchMetricNamesWithoutIndex(t *testing.T) {
}
}
testStorageSearchWithoutIndex(t, &opts)
testStorageSearchWithoutPerDayIndex(t, &opts)
}
func TestStorageSearchLabelNamesWithoutIndex(t *testing.T) {
func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutIndexOptions{
opts := testStorageSearchWithoutPerDayIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3178,17 +3209,17 @@ func TestStorageSearchLabelNamesWithoutIndex(t *testing.T) {
}
}
testStorageSearchWithoutIndex(t, &opts)
testStorageSearchWithoutPerDayIndex(t, &opts)
}
func TestStorageSearchLabelValuesWithoutIndex(t *testing.T) {
func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
const (
days = 4
rows = 10
labelName = "job"
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutIndexOptions{
opts := testStorageSearchWithoutPerDayIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3232,17 +3263,17 @@ func TestStorageSearchLabelValuesWithoutIndex(t *testing.T) {
}
}
testStorageSearchWithoutIndex(t, &opts)
testStorageSearchWithoutPerDayIndex(t, &opts)
}
func TestStorageSearchTagValueSuffixesWithoutIndex(t *testing.T) {
func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
const (
days = 4
rows = 10
tagValuePrefix = "metric."
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutIndexOptions{
opts := testStorageSearchWithoutPerDayIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3282,16 +3313,16 @@ func TestStorageSearchTagValueSuffixesWithoutIndex(t *testing.T) {
}
}
testStorageSearchWithoutIndex(t, &opts)
testStorageSearchWithoutPerDayIndex(t, &opts)
}
func TestStorageSearchGraphitePathsWithoutIndex(t *testing.T) {
func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutIndexOptions{
opts := testStorageSearchWithoutPerDayIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3332,16 +3363,16 @@ func TestStorageSearchGraphitePathsWithoutIndex(t *testing.T) {
}
}
testStorageSearchWithoutIndex(t, &opts)
testStorageSearchWithoutPerDayIndex(t, &opts)
}
func TestStorageQueryWithoutIndex(t *testing.T) {
func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutIndexOptions{
opts := testStorageSearchWithoutPerDayIndexOptions{
wantEmpty: []MetricRow(nil),
wantPerTimeRange: make(map[TimeRange]any),
alwaysPerTimeRange: true,
@@ -3382,191 +3413,54 @@ func TestStorageQueryWithoutIndex(t *testing.T) {
}
}
testStorageSearchWithoutIndex(t, &opts)
testStorageSearchWithoutPerDayIndex(t, &opts)
}
func TestStorageAddRowsWithZeroDate(t *testing.T) {
func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testStorageAddRowsWithZeroDate(t, disablePerDayIndex)
f := func(t *testing.T, disablePerDayIndex bool) {
t.Helper()
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
}
}
defer s.MustClose()
func testStorageAddRowsWithZeroDate(t *testing.T, disablePerDayIndex bool) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
mn := MetricName{MetricGroup: []byte("metric")}
mr := MetricRow{MetricNameRaw: mn.marshalRaw(nil)}
for range 10 {
mr.Timestamp = rand.Int63n(msecPerDay)
mr.Value = float64(rand.Intn(1000))
s.AddRows([]MetricRow{mr}, defaultPrecisionBits)
s.DebugFlush()
// Reset TSID cache so that insertion takes the path that involves
// checking whether the index contains metricName->TSID mapping.
s.resetAndSaveTSIDCache()
}
want := 1
firstUnixDay := TimeRange{
MinTimestamp: 0,
MaxTimestamp: msecPerDay - 1,
}
if got := s.newTimeseriesCreated.Load(); got != uint64(want) {
t.Errorf("unexpected new timeseries count: got %d, want %d", got, want)
}
if got := testCountAllMetricNames(s, firstUnixDay); got != want {
t.Errorf("unexpected metric name count: got %d, want %d", got, want)
}
if got := testCountAllMetricIDs(s, firstUnixDay); got != want {
t.Errorf("unexpected metric id count: got %d, want %d", got, want)
}
}
t.Run("disablePerDayIndex=false", func(t *testing.T) {
f(t, false)
})
t.Run("disablePerDayIndex=true", func(t *testing.T) {
f(t, true)
})
defer s.MustClose()
const numDays = 4
var metricNamesAll []string
labelNamesAll := []string{"__name__", "label"}
var labelValuesAll []string
mrs := make([]MetricRow, numDays)
for day := range numDays {
metricName := fmt.Sprintf("metric_%02d", day)
labelName := fmt.Sprintf("label_%02d", day)
labelValue := fmt.Sprintf("value_%02d", day)
if day != 0 {
metricNamesAll = append(metricNamesAll, metricName)
labelNamesAll = append(labelNamesAll, labelName)
labelValuesAll = append(labelValuesAll, labelValue)
}
mn := MetricName{
MetricGroup: []byte(metricName),
Tags: []Tag{
{Key: []byte(labelName), Value: []byte("value")},
{Key: []byte("label"), Value: []byte(labelValue)},
},
}
mn.sortTags()
mrs[day].MetricNameRaw = mn.marshalRaw(nil)
mrs[day].Timestamp = int64(day * msecPerDay)
}
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
if got, want := s.newTimeseriesCreated.Load(), uint64(numDays-1); got != want {
t.Fatalf("unexpected new timeseries count: got %d, want %d", got, want)
}
if got, want := s.tooSmallTimestampRows.Load(), uint64(1); got != want {
t.Fatalf("unexpected rows with too small timestamp: got %d, want %d", got, want)
}
assertMetricNames := func(tr TimeRange, want []string) {
t.Helper()
tfs := NewTagFilters()
if err := tfs.Add(nil, []byte("metric_.*"), false, true); err != nil {
t.Fatalf("unexpected error in TagFilters.Add: %v", err)
}
got, err := s.SearchMetricNames(nil, []*TagFilters{tfs}, tr, 1e9, noDeadline)
if err != nil {
t.Fatalf("SearchMetricNames(%v, %v) failed unexpectedly: %v", tfs, &tr, err)
}
for i, name := range got {
var mn MetricName
if err := mn.UnmarshalString(name); err != nil {
t.Fatalf("Could not unmarshal metric name %q: %v", name, err)
}
got[i] = string(mn.MetricGroup)
}
slices.Sort(got)
slices.Sort(want)
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("unexpected metric names (-want, +got):\n%s", diff)
}
}
assertLabelNames := func(tr TimeRange, want []string) {
t.Helper()
tfs := NewTagFilters()
if err := tfs.Add(nil, []byte("metric_.*"), false, true); err != nil {
t.Fatalf("unexpected error in TagFilters.Add: %v", err)
}
got, err := s.SearchLabelNames(nil, []*TagFilters{tfs}, tr, 1e9, 1e9, noDeadline)
if err != nil {
t.Fatalf("SearchLabelNames(%v, %v) failed unexpectedly: %s", tfs, &tr, err)
}
slices.Sort(got)
slices.Sort(want)
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("unexpected label names (-want, +got):\n%s", diff)
}
}
assertLabelValues := func(tr TimeRange, want []string) {
t.Helper()
tfs := NewTagFilters()
if err := tfs.Add([]byte("label"), []byte("value_.*"), false, true); err != nil {
t.Fatalf("unexpected error in TagFilters.Add: %v", err)
}
got, err := s.SearchLabelValues(nil, "label", []*TagFilters{tfs}, tr, 1e9, 1e9, noDeadline)
if err != nil {
t.Fatalf("SearchLabelValues(%v, %v) failed unexpectedly: %s", tfs, tr, err)
}
slices.Sort(got)
slices.Sort(want)
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("unexpected label values (-want, +got):\n%s", diff)
}
}
assertData := func(tr TimeRange, want []MetricRow) {
t.Helper()
tfs := NewTagFilters()
if err := tfs.Add(nil, []byte("metric_.*"), false, true); err != nil {
t.Fatalf("TagFilters.Add() failed unexpectedly: %v", err)
}
if err := testAssertSearchResult(s, tr, tfs, want); err != nil {
t.Fatalf("Search(%v, %v) failed unexpectedly: %v", tfs, tr, err)
}
}
var tr TimeRange
// Empty time range.
// Expect empty search results
tr = TimeRange{}
assertMetricNames(tr, nil)
assertLabelNames(tr, []string{})
assertLabelValues(tr, []string{})
assertData(tr, nil)
// First day time range.
// Expect empty search results
tr = TimeRange{
MinTimestamp: 0,
MaxTimestamp: msecPerDay - 1,
}
assertMetricNames(tr, nil)
assertLabelNames(tr, []string{})
assertLabelValues(tr, []string{})
assertData(tr, nil)
// Second day time range.
tr = TimeRange{
MinTimestamp: msecPerDay,
MaxTimestamp: 2*msecPerDay - 1,
}
if disablePerDayIndex {
// Expect index search results for all days if per-day index is
// disabled.
assertMetricNames(tr, metricNamesAll)
assertLabelNames(tr, labelNamesAll)
assertLabelValues(tr, labelValuesAll)
} else {
// Expect index search results on second day only if per-day index is
// enabled.
assertMetricNames(tr, []string{"metric_01"})
assertLabelNames(tr, []string{"__name__", "label", "label_01"})
assertLabelValues(tr, []string{"value_01"})
}
assertData(tr, mrs[1:2])
// First two days time range.
// Expect results on second day only.
tr = TimeRange{
MinTimestamp: 0,
MaxTimestamp: 2*msecPerDay - 1,
}
if disablePerDayIndex {
// Expect index search results for all days if per-day index is
// disabled.
assertMetricNames(tr, metricNamesAll)
assertLabelNames(tr, labelNamesAll)
assertLabelValues(tr, labelValuesAll)
} else {
// Expect index search results on second day only if per-day index is
// enabled.
assertMetricNames(tr, []string{"metric_01"})
assertLabelNames(tr, []string{"__name__", "label", "label_01"})
assertLabelValues(tr, []string{"value_01"})
}
assertData(tr, mrs[1:2])
}
// testSearchMetricIDs returns metricIDs for the given tfss and tr.
@@ -3603,13 +3497,13 @@ func testCountAllMetricIDs(s *Storage, tr TimeRange) int {
return len(ids)
}
func TestStorageRegisterMetricNamesVariousDataPatterns(t *testing.T) {
func TestStorageRegisterMetricNamesForVariousDataPatternsConcurrently(t *testing.T) {
testStorageVariousDataPatternsConcurrently(t, true, func(s *Storage, mrs []MetricRow) {
s.RegisterMetricNames(nil, mrs)
})
}
func TestStorageAddRowsVariousDataPatterns(t *testing.T) {
func TestStorageAddRowsForVariousDataPatternsConcurrently(t *testing.T) {
testStorageVariousDataPatternsConcurrently(t, false, func(s *Storage, mrs []MetricRow) {
s.AddRows(mrs, defaultPrecisionBits)
})
@@ -3626,18 +3520,27 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
const concurrency = 4
for _, disablePerDayIndex := range []bool{false, true} {
prefix := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(prefix+"/serial", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
})
t.Run(prefix+"/concurrentRows", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
})
t.Run(prefix+"/concurrentBatches", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
})
}
disablePerDayIndex := false
t.Run("perDayIndexes/serial", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
})
t.Run("perDayIndexes/concurrentRows", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
})
t.Run("perDayIndexes/concurrentBatches", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
})
disablePerDayIndex = true
t.Run("noPerDayIndexes/serial", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
})
t.Run("noPerDayIndexes/concurrentRows", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
})
t.Run("noPerDayIndexes/concurrentBatches", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
})
}
// testStorageVariousDataPatterns tests the ingestion of different combinations

View File

@@ -429,8 +429,9 @@ func (tb *table) getMinMaxIngestionTimestamps() (int64, int64) {
func (tb *table) getMinMaxTimestampsForAge(minAgeMsecs int64) (int64, int64) {
now := int64(fasttime.UnixTimestamp() * 1000)
minTimestamp := now - minAgeMsecs
if minTimestamp < minUnixMilli {
minTimestamp = minUnixMilli
if minTimestamp < 0 {
// Negative timestamps aren't supported by the storage.
minTimestamp = 0
}
maxTimestamp := int64(maxUnixMilli)
if maxUnixMilli-now > tb.s.futureRetentionMsecs {

View File

@@ -40,6 +40,12 @@ type TimeRange struct {
MaxTimestamp int64
}
// Zero time range and zero date are used to force global index search.
var (
globalIndexTimeRange = TimeRange{}
globalIndexDate = uint64(0)
)
// DateRange returns the date range for the given time range.
func (tr *TimeRange) DateRange() (uint64, uint64) {
minDate := uint64(tr.MinTimestamp) / msecPerDay
@@ -111,29 +117,10 @@ func (tr *TimeRange) contains(timestamp int64) bool {
return tr.MinTimestamp <= timestamp && timestamp <= tr.MaxTimestamp
}
// Zero time range and zero date are used to force global index search.
var (
globalIndexDate = uint64(0)
globalIndexTimeRange = TimeRange{}
)
const (
msecPerDay = 24 * 3600 * 1000
msecPerHour = 3600 * 1000
// minUnixMilli is the min millisecond that is allowed to be used as the
// sample timestamp.
//
// It corresponds to the first millisecond of the second day of the Unix
// Epoch, i.e. 1970-01-02T00:00:00.000Z.
//
// The first day of the Unix Epoch is reserved: zero date and zero time
// range are used for indicating that the the global index search is
// required. See globalIndexDate and globalIndexTimeRange above.
//
// Negative timestamps aren't supported.
minUnixMilli = msecPerDay
// maxUnixMilli is the max millisecond that is allowed to be used as the
// sample timestamp.
//
@@ -143,6 +130,6 @@ const (
// time.UnixMicro(math.MaxInt64/1000) == 2262-04-11 23:47:16.854775 UTC.
//
// Round it to the last millisecond of the last complete partition:
// 2262-03-31T23:59:59.999Z.
// 2262-03-31 23:59:59.999 UTC.
maxUnixMilli = 9222422399999
)

View File

@@ -23,6 +23,22 @@ func TestTryParseUnixTimestamp_Success(t *testing.T) {
f("0", 0)
//smaller than maxValidSecond
f("12", 12000000000)
f("12.0", 12000000000) // check fail, got 12000000000000
f("9223372", 9223372000000000)
f("9223372.0", 9223372000000000) // check fail, got 9223372000000000000
// bigger than maxValidSecond
f("9223373", 9223373000000000)
f("9223373.0", 9223373000000000) // check success
f("1700000000", 1700000000000000000)
f("1700000000.0", 1700000000000000000) // check success
// nanoseconds
f("-1234567890123456789", -1234567890123456789)
f("1234567890123456789", 1234567890123456789)