mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-25 03:27:51 +03:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
e30e8be1f4 |
@@ -565,14 +565,6 @@ func tryPushMetadataToRemoteStorages(at *auth.Token, rwctxs []*remoteWriteCtx, m
|
||||
mm.ProjectID = at.ProjectID
|
||||
}
|
||||
}
|
||||
tmp := mms[:0]
|
||||
for _, mm := range mms {
|
||||
if timeserieslimits.IsMetricMetadataExceeding(&mm) {
|
||||
continue
|
||||
}
|
||||
tmp = append(tmp, mm)
|
||||
}
|
||||
mms = tmp
|
||||
// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
|
||||
// Since metadata is usually small and there is no guarantee that metadata can be sent to
|
||||
// the same remote storage with the corresponding metrics.
|
||||
|
||||
@@ -175,19 +175,14 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
|
||||
}
|
||||
mms := ctx.mms
|
||||
mms = slicesutil.SetLength(mms, len(mmpbs))
|
||||
var cnt int
|
||||
for _, mmpb := range mmpbs {
|
||||
if timeserieslimits.IsMetricMetadataExceeding(&mmpb) {
|
||||
continue
|
||||
}
|
||||
mm := &mms[cnt]
|
||||
for idx, mmpb := range mmpbs {
|
||||
mm := &mms[idx]
|
||||
mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
|
||||
mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
|
||||
mm.Type = mmpb.Type
|
||||
mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
|
||||
cnt++
|
||||
}
|
||||
ctx.mms = mms[:cnt]
|
||||
ctx.mms = mms
|
||||
|
||||
err := vmstorage.VMInsertAPI.WriteMetadata(mms)
|
||||
if err != nil {
|
||||
@@ -206,18 +201,13 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
|
||||
}
|
||||
mms := ctx.mms
|
||||
mms = slicesutil.SetLength(mms, len(mmps))
|
||||
var cnt int
|
||||
for _, mmpb := range mmps {
|
||||
mm := &mms[cnt]
|
||||
if timeserieslimits.IsPrometheusMetadataExceeding(&mmpb) {
|
||||
continue
|
||||
}
|
||||
for idx, mmpb := range mmps {
|
||||
mm := &mms[idx]
|
||||
mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
|
||||
mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
|
||||
mm.Type = mmpb.Type
|
||||
cnt++
|
||||
}
|
||||
ctx.mms = mms[:cnt]
|
||||
ctx.mms = mms
|
||||
|
||||
err := vmstorage.VMInsertAPI.WriteMetadata(mms)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,7 +2,6 @@ package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
@@ -26,11 +25,7 @@ func TestSingleMetricsMetadata(t *testing.T) {
|
||||
if len(resp.Data) != 0 {
|
||||
t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
|
||||
}
|
||||
generateValueExceedLimit := func(prefix string) string {
|
||||
buf := make([]byte, math.MaxUint16+len(prefix))
|
||||
copy(buf, prefix)
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
const ingestTimestamp = 1707123456700
|
||||
prometheusTextDataSet := []string{
|
||||
`# HELP metric_name_1 some help message`,
|
||||
@@ -45,12 +40,6 @@ func TestSingleMetricsMetadata(t *testing.T) {
|
||||
`# TYPE metric_name_3 gauge`,
|
||||
`metric_name_3{label="baz"} 30`,
|
||||
}
|
||||
prometheusTextDataSet = append(prometheusTextDataSet,
|
||||
`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
|
||||
`# TYPE metric_name_4 gauge`,
|
||||
`metric_name_4{label="baz"} 30`,
|
||||
)
|
||||
|
||||
prometheusRemoteWriteDataSet := prompb.WriteRequest{
|
||||
Timeseries: []prompb.TimeSeries{
|
||||
{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
|
||||
@@ -63,9 +52,6 @@ func TestSingleMetricsMetadata(t *testing.T) {
|
||||
{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
|
||||
{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
|
||||
{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
|
||||
{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
|
||||
{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
|
||||
{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -151,11 +137,6 @@ func TestClusterMetricsMetadata(t *testing.T) {
|
||||
if len(resp.Data) != 0 {
|
||||
t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
|
||||
}
|
||||
generateValueExceedLimit := func(prefix string) string {
|
||||
buf := make([]byte, math.MaxUint16+len(prefix))
|
||||
copy(buf, prefix)
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
const ingestTimestamp = 1707123456700
|
||||
prometheusTextDataSet := []string{
|
||||
@@ -171,11 +152,6 @@ func TestClusterMetricsMetadata(t *testing.T) {
|
||||
`# TYPE metric_name_3 gauge`,
|
||||
`metric_name_3{label="baz"} 30`,
|
||||
}
|
||||
prometheusTextDataSet = append(prometheusTextDataSet,
|
||||
`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
|
||||
`# TYPE metric_name_4 gauge`,
|
||||
`metric_name_4{label="baz"} 30`,
|
||||
)
|
||||
prometheusRemoteWriteDataSet := prompb.WriteRequest{
|
||||
Timeseries: []prompb.TimeSeries{
|
||||
{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
|
||||
@@ -188,9 +164,6 @@ func TestClusterMetricsMetadata(t *testing.T) {
|
||||
{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
|
||||
{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
|
||||
{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
|
||||
{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
|
||||
{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
|
||||
{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -517,15 +517,10 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) {
|
||||
"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
|
||||
fmt.Sprintf(`-remoteWrite.url=http://%s/insert/multitenant/prometheus/api/v1/write`, sut.Vminsert.HTTPAddr()),
|
||||
})
|
||||
generateValueExceedLimit := func(prefix string) string {
|
||||
buf := make([]byte, math.MaxUint16+len(prefix))
|
||||
copy(buf, prefix)
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
prometheusRemoteWriteDataSet := prompb.WriteRequest{
|
||||
Metadata: []prompb.MetricMetadata{
|
||||
{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary, AccountID: 100},
|
||||
{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset, AccountID: 100},
|
||||
},
|
||||
}
|
||||
vmagent.PrometheusAPIV1Write(t, prometheusRemoteWriteDataSet, apptest.QueryOpts{Tenant: "multitenant"})
|
||||
|
||||
@@ -59,7 +59,7 @@ services:
|
||||
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
|
||||
restart: always
|
||||
vmanomaly:
|
||||
image: victoriametrics/vmanomaly:v1.29.6
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -14,6 +14,17 @@ aliases:
|
||||
---
|
||||
Please find the changelog for VictoriaMetrics Anomaly Detection below.
|
||||
|
||||
## v1.29.7
|
||||
Released: 2026-06-25
|
||||
|
||||
- UI: updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.1](https://docs.victoriametrics.com/anomaly-detection/ui/#v171) to [v1.7.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v172), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v172) for details.
|
||||
|
||||
- IMPROVEMENT: Increased high-cardinality inference scaling by optionally scattering periodic infer jobs to reduce contention on shared resources (e.g. datasource, CPU, RAM) when `settings.n_workers > 1` and `scheduler.infer_every` is smaller than the total time to fetch and process all queries. This is controlled by new `scatter_infer_jobs` boolean argument of [Periodic Scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#parameters-1) (default: `false`).
|
||||
|
||||
- IMPROVEMENT: Optimized internal batching for reader post-fetch series processing, exposing reader processing queue depth, and clarifying inference skip logs after data fetch timeouts.
|
||||
|
||||
- IMPROVEMENT: Refined `VmReader` and `VLogsReader` logging after datasource request failures by suppressing the follow-up generic "No data" or "No unseen data" warning for failed fetches. Failed requests now keep the original datasource error while empty successful responses still emit the no-data warning.
|
||||
|
||||
## v1.29.6
|
||||
Released: 2026-06-17
|
||||
|
||||
|
||||
@@ -423,7 +423,7 @@ services:
|
||||
# ...
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.29.6
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
# ...
|
||||
restart: always
|
||||
volumes:
|
||||
@@ -641,7 +641,7 @@ options:
|
||||
Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.29.6 && docker image tag victoriametrics/vmanomaly:v1.29.6 vmanomaly
|
||||
docker pull victoriametrics/vmanomaly:v1.29.7 && docker image tag victoriametrics/vmanomaly:v1.29.7 vmanomaly
|
||||
```
|
||||
|
||||
```sh
|
||||
|
||||
@@ -45,7 +45,7 @@ There are 2 types of compatibility to consider when migrating in stateful mode:
|
||||
|
||||
| Group start | Group end | Compatibility | Notes |
|
||||
|---------|--------- |------------|-------|
|
||||
| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.6](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1296) | Fully Compatible | - |
|
||||
| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1297) | Fully Compatible | - |
|
||||
| [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Partially compatible* | Dumped models of class [prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) and [seasonal quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) have problems with loading to [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) due to dropped `pytz` library. **Upgrading directly from v1.28.7 to [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) with a fix is suggested** |
|
||||
| [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
|
||||
| [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |
|
||||
|
||||
@@ -132,7 +132,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
|
||||
1. Pull Docker image:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.29.6
|
||||
docker pull victoriametrics/vmanomaly:v1.29.7
|
||||
```
|
||||
|
||||
2. Create the license file with your license key.
|
||||
@@ -152,7 +152,7 @@ docker run -it \
|
||||
-v ./license:/license \
|
||||
-v ./config.yaml:/config.yaml \
|
||||
-p 8490:8490 \
|
||||
victoriametrics/vmanomaly:v1.29.6 \
|
||||
victoriametrics/vmanomaly:v1.29.7 \
|
||||
/config.yaml \
|
||||
--licenseFile=/license \
|
||||
--loggerLevel=INFO \
|
||||
@@ -169,7 +169,7 @@ docker run -it \
|
||||
-e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
|
||||
-e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
|
||||
-p 8490:8490 \
|
||||
victoriametrics/vmanomaly:v1.29.6 \
|
||||
victoriametrics/vmanomaly:v1.29.7 \
|
||||
/config.yaml \
|
||||
--licenseFile=/license \
|
||||
--loggerLevel=INFO \
|
||||
@@ -182,7 +182,7 @@ services:
|
||||
# ...
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.29.6
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
# ...
|
||||
restart: always
|
||||
volumes:
|
||||
@@ -267,6 +267,7 @@ schedulers:
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler
|
||||
class: 'periodic'
|
||||
infer_every: '5m'
|
||||
scatter_infer_jobs: true
|
||||
fit_every: '1d'
|
||||
fit_window: '4w'
|
||||
|
||||
@@ -298,6 +299,7 @@ reader:
|
||||
datasource_url: "https://play.victoriametrics.com/" # [YOUR_DATASOURCE_URL]
|
||||
tenant_id: '0:0'
|
||||
sampling_period: "5m"
|
||||
series_processing_batch_size: 8 # number of time series to process together while preparing data for fit or infer stages
|
||||
queries:
|
||||
# define your queries with MetricsQL - https://docs.victoriametrics.com/victoriametrics/metricsql/
|
||||
cpu_user:
|
||||
@@ -413,11 +415,13 @@ For optimal service behavior, consider the following tweaks when configuring `vm
|
||||
- Configure the **inference frequency** in the [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) section of the configuration file.
|
||||
- Ensure that `infer_every` aligns with your **minimum required alerting frequency**.
|
||||
- For example, if receiving **alerts every 15 minutes** is sufficient (when `anomaly_score > 1`), set `infer_every` to match `reader.sampling_period` or override it per query via `reader.queries.query_xxx.step` for an optimal setup.
|
||||
- Set `scheduler.scatter_infer_jobs` {{% available_from "v1.29.7" anomaly %}} [arg](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#parameters-1) to `true` to allow for equal distribution of inference jobs across `infer_every` intervals, which can further enhance parallel processing efficiency and reduce resource contention when `reader.queries` contains a large number of queries.
|
||||
|
||||
**Reader**:
|
||||
- Setup the datasource to read data from in the [reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/) section. Include tenant ID if using a [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) (`multitenant` value {{% available_from "v1.16.2" anomaly %}} can be also used here).
|
||||
- Define queries for input data using [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) under `reader.queries` section. Note, it's possible to override reader-level arguments at query level for increased flexibility, e.g. specifying per-query [timezone](https://docs.victoriametrics.com/anomaly-detection/faq/#handling-timezones) or [sampling period](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters).
|
||||
- For longer `fit_window` intervals in scheduler, consider splitting queries into smaller time ranges to avoid excessive memory usage, timeouts and hitting server-side constraints, so they can be queried separately and reconstructed on `vmanomaly` side. Please refer to this [example](https://docs.victoriametrics.com/anomaly-detection/faq/#handling-large-queries-in-vmanomaly) for more details.
|
||||
- Set `reader.series_processing_batch_size` {{% available_from "v1.29.7" anomaly %}} [arg](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters) to a reasonable value (4-16, default is 8) to balance between memory usage and processing speed when preparing data for fit or infer stages.
|
||||
|
||||
> If applicable - consider [`VLogsReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#victorialogs-reader) {{% available_from "v1.26.0" anomaly %}} to perform anomaly detection on **log-derived metrics**. This is particularly useful for scenarios where log data needs to be analyzed for unusual patterns or behaviors, such as error rates or request latencies.
|
||||
|
||||
|
||||
@@ -315,7 +315,7 @@ docker run -it --rm \
|
||||
-e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
|
||||
-p 8080:8080 \
|
||||
-p 8490:8490 \
|
||||
victoriametrics/vmanomaly:v1.29.6 \
|
||||
victoriametrics/vmanomaly:v1.29.7 \
|
||||
vmanomaly_config.yaml
|
||||
```
|
||||
|
||||
@@ -640,6 +640,21 @@ If the **results** look good and the **model configuration should be deployed in
|
||||
|
||||
## Changelog
|
||||
|
||||
### v1.7.2
|
||||
Released: 2026-06-25
|
||||
|
||||
vmanomaly version: [v1.29.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1297)
|
||||
|
||||
- FEATURE: Added controls for selecting server-configured scheduled models (drop-down inside [model wizard](#model-panel)) and browsing scheduled queries from the running vmanomaly instance ("Queries" button, "scheduled queries" tab).
|
||||
|
||||
- IMPROVEMENT: Surfaced datasource fetch failures from ad-hoc VMUI raw queries as query-level errors instead of returning a successful empty result that triggers a generic "No match" warning. Now the user can see the actual error message from the datasource (e.g. "unauthorized", "not found", etc.) and take appropriate action.
|
||||
|
||||
- BUGFIX: Fixed [UI/query-server](#settings-panel) handling of VictoriaMetrics datasource URLs that already include `/select/multitenant/prometheus`. Such URLs are now recognized as cluster datasource URLs, preserving the multitenant path when proxying VMUI requests and allowing `server.use_reader_connection_settings` to reuse [configured reader credentials for authenticated datasources](#authentication).
|
||||
|
||||
- BUGFIX: Fixed [settings](#settings-panel) inputs for server and datasource URLs so editing, deleting, or pasting text is no longer immediately reverted to the previous value before applying changes.
|
||||
|
||||
- BUGFIX: Fixed [model wizard](#model-panel) settings for [`IsolationForestModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate) `contamination`, allowing decimal float values such as `0.1` or `0,1` to be typed or pasted without being collapsed to `0`, while preserving the `"auto"` value.
|
||||
|
||||
### v1.7.1
|
||||
Released: 2026-06-11
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ schedulers:
|
||||
periodic_online: # alias
|
||||
class: 'periodic' # scheduler class
|
||||
infer_every: "30s" # how often to produce anomaly scores for new data
|
||||
scatter_infer_jobs: true # distribute infer jobs evenly across the infer interval to reduce synchronized bursts
|
||||
fit_every: "365d" # how often to re-fit the models, for online models used effectively once, then they are updated with new data and won't require re-fit
|
||||
fit_window: "3d" # how much historical data to use for fit stage
|
||||
start_from: "00:00" # start from specified time, i.e. 00:00 given timezone and do daily fits as `fit_every` is 1 day
|
||||
@@ -56,6 +57,7 @@ schedulers:
|
||||
periodic_offline_1w:
|
||||
class: 'periodic'
|
||||
infer_every: "15m"
|
||||
scatter_infer_jobs: true
|
||||
fit_every: "24h"
|
||||
fit_window: "14d"
|
||||
# if no start_from is specified, jobs will start immediately after service starts
|
||||
@@ -135,6 +137,7 @@ server:
|
||||
port: 8490
|
||||
path_prefix: '/vmanomaly' # optional path prefix for all HTTP routes
|
||||
max_concurrent_tasks: 4 # maximum number of concurrent anomaly detection tasks processed by backend
|
||||
use_reader_connection_settings: True # if True, use reader's datasource_url and credentials for UI requests to datasource
|
||||
uvicorn_config: # optional Uvicorn server configuration
|
||||
log_level: 'warning'
|
||||
```
|
||||
|
||||
@@ -1265,7 +1265,7 @@ monitoring:
|
||||
Let's pull the docker image for `vmanomaly`:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.29.6
|
||||
docker pull victoriametrics/vmanomaly:v1.29.7
|
||||
```
|
||||
|
||||
Now we can run the docker container putting as volumes both config and model file:
|
||||
@@ -1279,7 +1279,7 @@ docker run -it \
|
||||
-v $(PWD)/license:/license \
|
||||
-v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
|
||||
-v $(PWD)/custom.yaml:/config.yaml \
|
||||
victoriametrics/vmanomaly:v1.29.6 /config.yaml \
|
||||
victoriametrics/vmanomaly:v1.29.7 /config.yaml \
|
||||
--licenseFile=/license
|
||||
--watch
|
||||
```
|
||||
|
||||
@@ -458,6 +458,21 @@ Label names [description](#labelnames)
|
||||
<td>The total number of datapoints received from VictoriaMetrics for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
|
||||
<td>
|
||||
|
||||
`url`, `query_key`, `scheduler_alias`, `preset`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`vmanomaly_reader_processing_tasks_queued`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`Gauge`
|
||||
</td>
|
||||
<td>The total number of queued processing tasks {{% available_from "v1.29.7" anomaly %}} (timeseries batches of size `series_processing_batch_size`) for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode. If continuously >0, it may lead to skipped infer runs due to resource contention and timeouts.</td>
|
||||
<td>
|
||||
|
||||
`url`, `query_key`, `scheduler_alias`, `preset`
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
@@ -421,7 +421,20 @@ Optional argument{{% available_from "v1.18.1" anomaly %}} allows defining **vali
|
||||
`60s`
|
||||
</td>
|
||||
<td>
|
||||
Optional argument{{% available_from "v1.25.3" anomaly %}} allows specifying a time offset for all queries in `queries`. Defaults to `0s` (0) if not set and can be overridden on a [per-query basis](#per-query-parameters).
|
||||
Optional argument {{% available_from "v1.25.3" anomaly %}}, allows specifying a time offset for all queries in `queries`. Defaults to `0s` (0) if not set and can be overridden on a [per-query basis](#per-query-parameters).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`series_processing_batch_size`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`8`
|
||||
</td>
|
||||
<td>
|
||||
Optional argument {{% available_from "v1.29.7" anomaly %}}, allows specifying the number of time series to process together while preparing data for fit or infer stages. Defaults to `8`. Suggested values are 4-16 for high-cardinality queries.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
@@ -450,6 +463,7 @@ reader:
|
||||
sampling_period: '1m'
|
||||
query_from_last_seen_timestamp: True # false by default
|
||||
latency_offset: '1ms'
|
||||
series_processing_batch_size: 8
|
||||
```
|
||||
|
||||
### MetricsQL Playground
|
||||
|
||||
@@ -74,40 +74,7 @@ options={`"scheduler.periodic.PeriodicScheduler"`, `"scheduler.oneoff.OneoffSche
|
||||
|
||||
### Parameters
|
||||
|
||||
For periodic scheduler parameters are defined as differences in times, expressed in difference units, e.g. days, hours, minutes, seconds.
|
||||
|
||||
Examples: `"50s"`, `"4m"`, `"3h"`, `"2d"`, `"1w"`.
|
||||
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>Time granularity</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>s</td>
|
||||
<td>seconds</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>m</td>
|
||||
<td>minutes</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>h</td>
|
||||
<td>hours</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>d</td>
|
||||
<td>days</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>w</td>
|
||||
<td>weeks</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
For periodic scheduler parameters are defined as differences in times, expressed in difference units, e.g. days, hours, minutes, seconds. Time granularity is defined by the last characters of a string. Examples: `"50s"` (seconds), `"4m"` (minutes), `"3h"` (hours), `"2d"` (days), `"1w"` (weeks).
|
||||
|
||||
<table class="params">
|
||||
<thead>
|
||||
@@ -188,6 +155,21 @@ Specifies when to initiate the first `fit_every` call. Accepts either an ISO 860
|
||||
Defines the local timezone for the `start_from` parameter, if specified. Defaults to `UTC` if no timezone is provided.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`scatter_infer_jobs`{{% available_from "v1.29.7" anomaly %}}</span>
|
||||
</td>
|
||||
<td>bool, <span style="white-space: nowrap;">Optional</span></td>
|
||||
<td>
|
||||
|
||||
`true` or `false`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
If `true`, distribute infer jobs and their dependent data-fetch jobs evenly across the infer interval. This reduces synchronized read and inference bursts for high-scale configurations. Defaults to `false`. Useful when `settings.n_workers > 1`, `reader.queries` cardinality is high, and `scheduler.infer_every` is small.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -200,6 +182,7 @@ schedulers:
|
||||
# (or class: "scheduler.periodic.PeriodicScheduler" for versions before v1.13.0, without class alias support)
|
||||
fit_window: "14d"
|
||||
infer_every: "1m"
|
||||
scatter_infer_jobs: true # Distribute infer jobs evenly across the infer interval to reduce synchronized bursts.
|
||||
fit_every: "1h"
|
||||
start_from: "20:00" # If launched before 20:00 (local Kyiv time), the first run starts today at 20:00. Otherwise, it starts tomorrow at 20:00.
|
||||
tz: "Europe/Kyiv" # Defaults to 'UTC' if not specified.
|
||||
|
||||
@@ -395,7 +395,7 @@ services:
|
||||
restart: always
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.29.6
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -26,8 +26,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce `65kb` size limit for `metric metadata` fields - `Unit`, `Help` and `MetricFamilyName`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
|
||||
|
||||
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
|
||||
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package metricsmetadata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
@@ -25,7 +24,7 @@ type Row struct {
|
||||
}
|
||||
|
||||
// MarshalTo serializes Row into provided buffer and returns result
|
||||
func (mr *Row) MarshalTo(dst []byte) ([]byte, error) {
|
||||
func (mr *Row) MarshalTo(dst []byte) []byte {
|
||||
dstLen := len(dst)
|
||||
// tenant information (accountID and projectID)
|
||||
dstSize := dstLen + 8
|
||||
@@ -38,20 +37,10 @@ func (mr *Row) MarshalTo(dst []byte) ([]byte, error) {
|
||||
dst = encoding.MarshalUint32(dst, mr.AccountID)
|
||||
dst = encoding.MarshalUint32(dst, mr.ProjectID)
|
||||
dst = encoding.MarshalUint32(dst, uint32(mr.Type))
|
||||
var err error
|
||||
dst, err = marshalBytesFast(dst, mr.MetricFamilyName)
|
||||
if err != nil {
|
||||
return dst, fmt.Errorf("cannot marshal MetricFamilyName: %w", err)
|
||||
}
|
||||
dst, err = marshalBytesFast(dst, mr.Help)
|
||||
if err != nil {
|
||||
return dst, fmt.Errorf("cannot marshal Help: %w", err)
|
||||
}
|
||||
dst, err = marshalBytesFast(dst, mr.Unit)
|
||||
if err != nil {
|
||||
return dst, fmt.Errorf("cannot marshal Unit: %w", err)
|
||||
}
|
||||
return dst, nil
|
||||
dst = marshalBytesFast(dst, mr.MetricFamilyName)
|
||||
dst = marshalBytesFast(dst, mr.Help)
|
||||
dst = marshalBytesFast(dst, mr.Unit)
|
||||
return dst
|
||||
}
|
||||
|
||||
// Unmarshal parses Row from provided buffer and returns tail buffer
|
||||
@@ -137,11 +126,8 @@ func UnmarshalRows(dst []Row, src []byte, maxRows int) ([]Row, []byte, error) {
|
||||
return dst, src, nil
|
||||
}
|
||||
|
||||
func marshalBytesFast(dst []byte, s []byte) ([]byte, error) {
|
||||
if len(s) > math.MaxUint16 {
|
||||
return dst, fmt.Errorf("size of s: %d cannot exceed max uint16", len(s))
|
||||
}
|
||||
func marshalBytesFast(dst []byte, s []byte) []byte {
|
||||
dst = encoding.MarshalUint16(dst, uint16(len(s)))
|
||||
dst = append(dst, s...)
|
||||
return dst, nil
|
||||
return dst
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package timeserieslimits
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
@@ -9,7 +8,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -46,9 +44,6 @@ func Init(inputMaxLabelsPerTimeseries, inputMaxLabelNameLen, inputMaxLabelValueL
|
||||
_ = metrics.GetOrCreateGauge(`vm_rows_ignored_total{reason="too_long_label_value"}`, func() float64 {
|
||||
return float64(ignoredSeriesWithTooLongLabelValue.Load())
|
||||
})
|
||||
_ = metrics.GetOrCreateGauge(`vm_rows_ignored_total{reason="too_long_metric_metadata_value"}`, func() float64 {
|
||||
return float64(ignoredMetricsMetadataWithTooLongValue.Load())
|
||||
})
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -66,8 +61,6 @@ var (
|
||||
|
||||
// ignoredSeriesWithTooLongLabelValue is the number of ignored series which contain labels with too long values
|
||||
ignoredSeriesWithTooLongLabelValue atomicutil.Uint64
|
||||
|
||||
ignoredMetricsMetadataWithTooLongValue atomicutil.Uint64
|
||||
)
|
||||
|
||||
func trackIgnoredSeriesWithTooManyLabels(labels []prompb.Label) {
|
||||
@@ -139,63 +132,3 @@ func IsExceeding(labels []prompb.Label) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
func trackIgnoredMetricMetadataWithTooLongValue(fieldName, metricName string, fieldSize int) {
|
||||
ignoredMetricsMetadataWithTooLongValue.Add(1)
|
||||
select {
|
||||
case <-ignoredSeriesWithTooLongLabelValueLogTicker.C:
|
||||
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
|
||||
logger.Warnf("ignoring metric metadata with metric name %q; field %q value length=%d exceeds %d limit; "+
|
||||
"reduce the size of field at metric metadata source.",
|
||||
metricName, fieldName, fieldSize, metricMetadataMaxFieldValueSize)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// metricMetadataMaxFieldValueSize defines max size of string fields at MetricMetadata
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128 for details
|
||||
const metricMetadataMaxFieldValueSize = math.MaxUint16
|
||||
|
||||
// IsMetricMetadataExceeding checks if passed metricMetadata exceed size limit
|
||||
// for the following fields:
|
||||
// * Help
|
||||
// * MetricFamilyName
|
||||
// * Unit
|
||||
//
|
||||
// increments metrics and shows warning in logs
|
||||
func IsMetricMetadataExceeding(md *prompb.MetricMetadata) bool {
|
||||
if len(md.Help) > metricMetadataMaxFieldValueSize {
|
||||
trackIgnoredMetricMetadataWithTooLongValue("help", md.MetricFamilyName, len(md.Help))
|
||||
return true
|
||||
}
|
||||
if len(md.MetricFamilyName) > metricMetadataMaxFieldValueSize {
|
||||
trackIgnoredMetricMetadataWithTooLongValue("metricFamilyName", md.MetricFamilyName, len(md.MetricFamilyName))
|
||||
|
||||
return true
|
||||
}
|
||||
if len(md.Unit) > metricMetadataMaxFieldValueSize {
|
||||
trackIgnoredMetricMetadataWithTooLongValue("unit", md.MetricFamilyName, len(md.Unit))
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// IsPrometheusMetadataExceeding checks if passed prometheus.Metadata exceed size limit
|
||||
// for the following fields:
|
||||
// * Help
|
||||
// * Metric
|
||||
//
|
||||
// increments metrics and shows warning in logs
|
||||
func IsPrometheusMetadataExceeding(md *prometheus.Metadata) bool {
|
||||
if len(md.Help) > metricMetadataMaxFieldValueSize {
|
||||
trackIgnoredMetricMetadataWithTooLongValue("help", md.Metric, len(md.Help))
|
||||
return true
|
||||
}
|
||||
if len(md.Metric) > metricMetadataMaxFieldValueSize {
|
||||
trackIgnoredMetricMetadataWithTooLongValue("metric", md.Metric, len(md.Metric))
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user