Compare commits
1 Commits
master
...
docs/vmano
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d88f43def |
@@ -30,6 +30,35 @@ groups:
|
||||
summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
|
||||
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5m"
|
||||
|
||||
- alert: SchedulerWorkerDown
|
||||
expr: >
|
||||
(vmanomaly_scheduler_alive{job=~".*vmanomaly.*"} == 0)
|
||||
and on (job, instance, scheduler_alias, preset)
|
||||
(sum(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"})
|
||||
by (job, instance, scheduler_alias, preset) > 0)
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Scheduler {{ $labels.scheduler_alias }} is down in {{ $labels.job }} ({{ $labels.instance }})"
|
||||
description: |
|
||||
Periodic scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} has remained down for more than 2 minutes
|
||||
after entering automatic recovery. The restart policy may be exhausted.
|
||||
Check vmanomaly logs and the vmanomaly_scheduler_restarts_total metric for restart failures.
|
||||
|
||||
- alert: FrequentSchedulerRestarts
|
||||
expr: >
|
||||
sum(increase(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"}[15m]))
|
||||
by (job, instance, scheduler_alias, preset) > 2
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Scheduler {{ $labels.scheduler_alias }} is restarting frequently in {{ $labels.job }} ({{ $labels.instance }})"
|
||||
description: |
|
||||
Scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} had more than two restart attempts
|
||||
during the last 15 minutes. The service may be self-healing, but repeated restarts indicate an unstable
|
||||
scheduler, configuration, datasource, or runtime dependency. Check vmanomaly logs for the original failure.
|
||||
|
||||
# default value of 900 Should be changed to the scrape_interval for pull metrics. For push metrics this should be the lowest fit_every or infer_every in your vmanomaly config.
|
||||
- alert: NoSelfMonitoringMetrics
|
||||
expr: >
|
||||
|
||||
@@ -59,7 +59,7 @@ services:
|
||||
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
|
||||
restart: always
|
||||
vmanomaly:
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
image: victoriametrics/vmanomaly:v1.30.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
schedulers:
|
||||
periodic:
|
||||
infer_every: "1m"
|
||||
fit_every: "1h"
|
||||
fit_window: "2d" # 2d-14d based on the presence of weekly seasonality in your data
|
||||
fit_every: "100w" # the online model keeps learning during inference
|
||||
fit_window: "2w"
|
||||
|
||||
models:
|
||||
prophet:
|
||||
class: "prophet"
|
||||
args:
|
||||
interval_width: 0.98
|
||||
weekly_seasonality: False # comment it if your data has weekly seasonality
|
||||
yearly_seasonality: False
|
||||
temporal_envelope:
|
||||
class: "temporal_envelope"
|
||||
queries: "all"
|
||||
schedulers: "all"
|
||||
seasonalities: ["hod_smooth", "dow_smooth"]
|
||||
alpha: 0.005
|
||||
loss_reactivity: 5
|
||||
iqr_threshold: 2
|
||||
|
||||
reader:
|
||||
datasource_url: "http://victoriametrics:8428/"
|
||||
@@ -26,4 +28,4 @@ writer:
|
||||
monitoring:
|
||||
pull: # Enable /metrics endpoint.
|
||||
addr: "0.0.0.0"
|
||||
port: 8490
|
||||
port: 8490
|
||||
|
||||
@@ -69,18 +69,21 @@ See more details at [VictoriaMetrics/mcp-victoriatraces](https://github.com/Vict
|
||||
REST API and documentation for AI-assisted anomaly detection, model management, and observability insights.
|
||||
|
||||
Capabilities include:
|
||||
- Health Monitoring: Check `vmanomaly` server health and build information
|
||||
- Model Management: List, validate, and configure anomaly detection models (like `zscore_online`, `prophet`, and more)
|
||||
- Configuration Generation: Generate complete `vmanomaly` YAML configurations
|
||||
- Alert Rule Generation: Generate [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules) based on [anomaly score metrics](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) to simplify alerting setup
|
||||
- Documentation Search: Full-text search across embedded `vmanomaly` documentation with fuzzy matching
|
||||
- Check `vmanomaly` health, build information, compatibility, and self-monitoring metrics
|
||||
- Inspect model schemas and validate model or complete service configurations
|
||||
- Profile sampled query results for trends, seasonalities, changepoints, gaps, and intermittent behavior
|
||||
- Run asynchronous autotune tasks and turn their results into data-driven model recommendations
|
||||
- Generate complete `vmanomaly` YAML configurations and [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules)
|
||||
- Search embedded `vmanomaly` documentation with fuzzy matching
|
||||
|
||||
See more details at [VictoriaMetrics/mcp-vmanomaly](https://github.com/VictoriaMetrics/mcp-vmanomaly).
|
||||
# vmanomaly UI Copilot
|
||||
The vmanomaly UI includes an [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) that can assist users with anomaly detection tasks, model configuration, and troubleshooting, changing the UI state based on user queries and providing actionable suggestions through automated data profiling and validation. The AI Copilot is powered by respective [MCP Server](#vmanomaly-mcp-server) and [Agent Skills](#agent-skills), enabling it to understand the context of the user's actions and provide relevant guidance.
|
||||
|
||||
# Agent Skills
|
||||
|
||||
[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate,
|
||||
and troubleshoot VictoriaMetrics observability components, including metrics, logs, and traces.
|
||||
[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate,
|
||||
and troubleshoot VictoriaMetrics observability components, including metrics, logs, traces, and [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/).
|
||||
|
||||
These skills provide predefined workflows and capabilities such as:
|
||||
* Query metrics, logs, traces and alerts
|
||||
@@ -89,6 +92,9 @@ These skills provide predefined workflows and capabilities such as:
|
||||
* Cardinality optimization
|
||||
* Unused metric detection
|
||||
* Stream aggregation configuration
|
||||
* Build validated `vmanomaly` configurations from measured time-series characteristics (e.g., seasonality, changepoints, trends)
|
||||
* Query and operate the `vmanomaly` API
|
||||
* Review existing anomaly detection configurations against real data and identify false-positive or model-data fit issues
|
||||
|
||||
To install the available skills for AI agents, run:
|
||||
```sh
|
||||
@@ -108,4 +114,4 @@ AI code assistants like Claude Code, OpenAI Codex, Gemini CLI, Qwen Code, and Op
|
||||
helps to monitor cost usage, analytics, performance, compliance and improves troubleshooting experience. All major
|
||||
AI coding tools support OpenTelemetry and can be easily integrated into VictoriaMetrics Observability Stack.
|
||||
Please see more details in [Vibe coding tools observability with VictoriaMetrics Stack and OpenTelemetry
|
||||
](https://victoriametrics.com/blog/vibe-coding-observability/).
|
||||
](https://victoriametrics.com/blog/vibe-coding-observability/).
|
||||
|
||||
@@ -9,6 +9,7 @@ tags:
|
||||
- metrics
|
||||
- logs
|
||||
- traces
|
||||
- anomaly detection
|
||||
- AI
|
||||
- AI integration
|
||||
- agent
|
||||
|
||||
@@ -14,6 +14,31 @@ aliases:
|
||||
---
|
||||
Please find the changelog for VictoriaMetrics Anomaly Detection below.
|
||||
|
||||
{{% collapse name="2026" open=true %}}
|
||||
|
||||
## v1.30.0
|
||||
Released: 2026-07-23
|
||||
|
||||
- FEATURE: Added univariate and multivariate online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) models for complex time-series profiles. They combine robust causal trend, compact calendar and learned holiday behavior, changepoint adaptation, residual prediction intervals, future-horizon forecasts, and optional cross-series dependency detection in bounded state.
|
||||
|
||||
- FEATURE: Added asynchronous [`/api/v1/autotune/tasks`](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow) endpoints and improved unsupervised tuning with model-capability-aware objectives, optional causal exact validation for online models, constrained alert-volume selection, fitted-state complexity tie-breaking, and frozen model-specific parameters.
|
||||
|
||||
- FEATURE: Added `/api/v1/timeseries/characteristics` for bounded analysis of trend, calendar seasonality, changepoints, gaps, and intermittent or spiky behavior across sampled query results. This improves automated model selection and search-space suggestions in both agentic workflows and [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance)-backed [UI](https://docs.victoriametrics.com/anomaly-detection/ui/) experiments.
|
||||
|
||||
- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v172) to [v1.8.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v180). Notable mentions are model and query configuration, query prettification, fullscreen charts, out-of-date results alert and improved [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) suggestion and cancellation behavior.
|
||||
|
||||
- FEATURE: Made AI-assisted configuration more reliable with reusable [vmanomaly workflow skills](https://github.com/VictoriaMetrics/skills), data-aware model suggestions, protocol-safe cancellation and synchronized query, model, and anomaly-setting updates in [UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
|
||||
|
||||
- IMPROVEMENT: Fitted history can be retained as a stronger prior through `history_strength` argument, requiring a reduced number of observations (say 2 weeks for weekly seasonality) to be effective if the history window is a good indicator of "normal" data patterns. Models supported: `mad_online`, `zscore_online`, `quantile_online`.
|
||||
|
||||
- IMPROVEMENT: Improved bounded datasource operation with optional ad-hoc series limits, stale-series lookback caps, and propagation of configured per-query sampling periods to models and autotune trials.
|
||||
|
||||
- IMPROVEMENT: Added `reader.fetch_timeout` and `reader.processing_timeout` for [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [`VLogsReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#victorialogs-reader), allowing datasource reads and post-fetch processing to be tuned *independently* while preserving `reader.timeout` as the backward-compatible default for both phases.
|
||||
|
||||
- BUGFIX: Fixed [backtesting runs](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) across multiple fit cycles and for auto-tuned online wrappers, preventing duplicate or missing predictions while retaining causal model updates and restoration of compatible legacy auto-tuned state.
|
||||
|
||||
- BUGFIX: Prevented service shutdown after a [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) worker dies by adding bounded restart attempts with backoff and exposing restart health through [startup metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics).
|
||||
|
||||
## v1.29.7
|
||||
Released: 2026-06-25
|
||||
|
||||
@@ -124,6 +149,10 @@ Released: 2026-01-12
|
||||
|
||||
- BUGFIX: Restored expected behavior when `fit_every` equals `infer_every` in [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) - now full data range `fit_window` is fetched for model trainings instead of a last point from that interval.
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="2025" %}}
|
||||
|
||||
## v1.28.3
|
||||
Released: 2025-12-17
|
||||
|
||||
@@ -395,6 +424,10 @@ Released: 2025-01-20
|
||||
- BUGFIX: Now [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) properly handles the cases where the number of queries processed in parallel (up to `reader.queries` cardinality) exceeds the default limit of 10 HTTP(S) connections, preventing potential data loss from discarded queries. The pool limit will automatically adjust to match `reader.queries` cardinality.
|
||||
- BUGFIX: Corrected the construction of write endpoints for cluster VictoriaMetrics `url`s (`tenant_id` arg is set) in `monitoring.push` [section configurations](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters).
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="2024" %}}
|
||||
|
||||
## v1.18.8
|
||||
Released: 2024-12-03
|
||||
|
||||
@@ -709,6 +742,10 @@ Released: 2024-01-15
|
||||
- IMPROVEMENT: Don't check /health endpoint, check the real /query_range or /import endpoints directly. Users kept getting problems with /health.
|
||||
- DEPRECATION: "health_path" param is deprecated and doesn't do anything in config ([reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), [monitoring.push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters)).
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="2023" %}}
|
||||
|
||||
|
||||
## v1.7.2
|
||||
Released: 2023-12-21
|
||||
@@ -803,6 +840,12 @@ Released: 2023-01-23
|
||||
Released: 2023-01-06
|
||||
- BUGFIX: prophet model incorrectly predicted two points in case of only one
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
{{% collapse name="2022" %}}
|
||||
|
||||
## v1.0.0-beta
|
||||
Released: 2022-12-08
|
||||
- First public release is available
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
@@ -125,7 +125,7 @@ Please refer to the [state restoration section](https://docs.victoriametrics.com
|
||||
|
||||
## Config hot-reloading
|
||||
|
||||
`vmanomaly` supports [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to automatically reload configurations on config files changes. It can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) and allows for configuration updates without explicit service restarts.
|
||||
`vmanomaly` supports [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
|
||||
|
||||
## Environment variables
|
||||
|
||||
@@ -141,20 +141,31 @@ For information on migrating between different versions of `vmanomaly`, please r
|
||||
|
||||
## Choosing the right model for vmanomaly
|
||||
|
||||
> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on selecting the best model and its configuration for your use case. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
|
||||
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect:
|
||||
|
||||
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet).
|
||||
- Use [Online MAD](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) for simple, mostly stationary data with no-to-slow trend, when robustness to outliers is important.
|
||||
- Use [Online Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) for simple, light-tailed data where standard-deviation units are meaningful.
|
||||
- Use [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} for complex data with trends, calendar patterns, holidays, or persistent shifts. It is the preferred *online* alternative to Prophet (which will be deprecated in the future releases).
|
||||
- Use multivariate Temporal Envelope when normal relationships between aligned metrics matter. This should replace [Isolation Forest](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate) used in previous versions of `vmanomaly`, which will be deprecated in future releases.
|
||||
- Use [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) when Prophet-specific decomposition outputs, or offline batch behavior are required. Consider using Temporal Envelope instead, as it is more efficient and provides better results in most cases.
|
||||
|
||||
Also, there is an option to auto-tune the most important (hyper)parameters of selected model class {{% available_from "v1.12.0" anomaly %}}, find [the details here](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned).
|
||||
There is also an option to auto-tune the most important parameters of a selected model class {{% available_from "v1.12.0" anomaly %}}. {{% available_from "v1.30.0" anomaly %}} The asynchronous autotune API can first profile a bounded sample through `/api/v1/timeseries/characteristics`, then tune a shared concrete configuration through `/api/v1/autotune/tasks`. See the [autotune workflow](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow).
|
||||
|
||||
Please refer to [respective blogpost on anomaly types and alerting heuristics](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/) for more details.
|
||||
|
||||
Still not 100% sure what to use? We are [here to help](https://docs.victoriametrics.com/anomaly-detection/#get-in-touch).
|
||||
|
||||
## Incorporating domain knowledge
|
||||
## Can AI help configure vmanomaly?
|
||||
|
||||
> [!TIP]
|
||||
> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on incorporating domain knowledge into your anomaly detection models. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
|
||||
Yes. The available tools serve different workflows:
|
||||
|
||||
- [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) provides interactive guidance and can apply query, model, and alerting changes in the UI.
|
||||
- The [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server) gives compatible AI clients access to live schemas, documentation, time-series characteristics, configuration validation, and autotune tasks.
|
||||
- [Agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) provide repeatable workflows for investigating data and generating or reviewing `vmanomaly` and `vmalert` configurations.
|
||||
|
||||
Treat AI-generated configuration as a proposal. Review it and validate it through the UI or with [`--dryRun`](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) before deployment.
|
||||
|
||||
## Incorporating domain knowledge
|
||||
|
||||
Anomaly detection models can significantly improve when incorporating business-specific assumptions about the data and what constitutes an anomaly. `vmanomaly` supports various [business-side configuration parameters](https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args) across all built-in models to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** and **align model behavior with business needs**, for example:
|
||||
|
||||
@@ -298,7 +309,7 @@ Configuration above will produce N intervals of full length (`fit_window`=14d +
|
||||
|
||||
## Forecasting
|
||||
|
||||
`vmanomaly` can generate future forecasts (e.g. using [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}), which is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
|
||||
`vmanomaly` can generate future forecasts using [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} or [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}. This is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
|
||||
|
||||
> However, please note that this mode should be used with care, as the model will produce `yhat_{h}` (and probably `yhat_lower_{h}`, and `yhat_upper_{h}`) time series **for each timeseries returned by input queries and for each forecasting horizon specified in `forecast_at` argument, which can lead to a significant increase in the number of active timeseries in VictoriaMetrics TSDB**.
|
||||
|
||||
@@ -309,12 +320,12 @@ Here's an example of how to produce forecasts using `vmanomaly` and combine it w
|
||||
schedulers:
|
||||
periodic_5m: # this scheduler will be used to produce anomaly scores each 5 minutes using "regular" simple model
|
||||
class: 'periodic'
|
||||
fit_every: '30d'
|
||||
fit_every: '100w'
|
||||
fit_window: '3d'
|
||||
infer_every: '5m'
|
||||
periodic_forecast: # this scheduler will be used to produce forecasts each 24h using "daily" model
|
||||
class: 'periodic'
|
||||
fit_every: '7d'
|
||||
fit_every: '1000w'
|
||||
fit_window: '730d' # to fit the model on 2 years of data to account for seasonality and holidays
|
||||
infer_every: '24h'
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader
|
||||
@@ -356,20 +367,18 @@ models:
|
||||
quantiles: [0.25, 0.5, 0.75] # to produce median and upper quartiles
|
||||
iqr_threshold: 2.0
|
||||
|
||||
prophet_1d:
|
||||
class: 'prophet'
|
||||
envelope_1d:
|
||||
class: 'temporal_envelope'
|
||||
queries: ['disk_usage_perc_1d']
|
||||
schedulers: ['periodic_forecast']
|
||||
clip_predictions: True
|
||||
detection_direction: 'above_expected' # as we are interested in spikes in capacity planning
|
||||
forecast_at: ['3d', '7d'] # this will produce forecasts for 3 and 7 days ahead
|
||||
provide_series: ['yhat', 'yhat_upper'] # to write forecasts back to VictoriaMetrics, omitting `yhat_lower` as it is not needed in this example
|
||||
# other model params, yearly_seasonality may stay
|
||||
|
||||
# https://facebook.github.io/prophet/docs/quick_start#python-api
|
||||
args:
|
||||
interval_width: 0.98 # see https://facebook.github.io/prophet/docs/uncertainty_intervals
|
||||
country_holidays: 'US'
|
||||
seasonalities: [dow_smooth]
|
||||
holidays:
|
||||
countries: [US]
|
||||
group: true
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer
|
||||
writer:
|
||||
class: 'vm'
|
||||
@@ -423,7 +432,7 @@ services:
|
||||
# ...
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
image: victoriametrics/vmanomaly:v1.30.0
|
||||
# ...
|
||||
restart: always
|
||||
volumes:
|
||||
@@ -460,6 +469,8 @@ With the introduction of [online models](https://docs.victoriametrics.com/anomal
|
||||
- **Optimized resource utilization**: By spreading the computational load over time and reducing peak demands, online models make more efficient use of resources and inducing less data transfer from VictoriaMetrics TSDB, improving overall system performance.
|
||||
- **Faster convergence**: Online models can adapt {{% available_from "v1.23.0" anomaly %}} to changes in data patterns more quickly, which is particularly beneficial in dynamic environments where data characteristics may shift frequently. See `decay` argument description [here](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay).
|
||||
|
||||
{{% available_from "v1.30.0" anomaly %}} Online models expose a **common** `min_n_samples_seen` warmup: they continue learning, but emit `anomaly_score: 0` until enough observations have been seen. Online MAD, Z-score, and Seasonal Quantile also support `history_strength`, which retains fitted history as a stronger prior while inference continues to adapt the state.
|
||||
|
||||
> {{% available_from "v1.24.0" anomaly %}} Online models are best used in conjunction with [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the model state across service restarts. This allows the model to continue adapting to new data without losing previously learned patterns, thus avoiding the need for a full `fit` stage to start working again. {{% available_from "v1.28.1" anomaly %}} Additionally, setting [retention policies](https://docs.victoriametrics.com/anomaly-detection/components/settings/#retention) helps manage disk space or RAM used by periodical cleanup of old model instances.
|
||||
|
||||
Here's an example of how we can switch from (offline) [Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score) to [Online Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score):
|
||||
@@ -641,7 +652,7 @@ options:
|
||||
Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.29.7 && docker image tag victoriametrics/vmanomaly:v1.29.7 vmanomaly
|
||||
docker pull victoriametrics/vmanomaly:v1.30.0 && docker image tag victoriametrics/vmanomaly:v1.30.0 vmanomaly
|
||||
```
|
||||
|
||||
```sh
|
||||
|
||||
@@ -45,7 +45,7 @@ There are 2 types of compatibility to consider when migrating in stateful mode:
|
||||
|
||||
| Group start | Group end | Compatibility | Notes |
|
||||
|---------|--------- |------------|-------|
|
||||
| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1297) | Fully Compatible | - |
|
||||
| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300) | Fully Compatible | v1.30.0 adds new [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model state without changing the compatibility of existing model and data artifacts. |
|
||||
| [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Partially compatible* | Dumped models of class [prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) and [seasonal quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) have problems with loading to [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) due to dropped `pytz` library. **Upgrading directly from v1.28.7 to [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) with a fix is suggested** |
|
||||
| [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
|
||||
| [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |
|
||||
|
||||
@@ -132,7 +132,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
|
||||
1. Pull Docker image:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.29.7
|
||||
docker pull victoriametrics/vmanomaly:v1.30.0
|
||||
```
|
||||
|
||||
2. Create the license file with your license key.
|
||||
@@ -152,7 +152,7 @@ docker run -it \
|
||||
-v ./license:/license \
|
||||
-v ./config.yaml:/config.yaml \
|
||||
-p 8490:8490 \
|
||||
victoriametrics/vmanomaly:v1.29.7 \
|
||||
victoriametrics/vmanomaly:v1.30.0 \
|
||||
/config.yaml \
|
||||
--licenseFile=/license \
|
||||
--loggerLevel=INFO \
|
||||
@@ -169,7 +169,7 @@ docker run -it \
|
||||
-e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
|
||||
-e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
|
||||
-p 8490:8490 \
|
||||
victoriametrics/vmanomaly:v1.29.7 \
|
||||
victoriametrics/vmanomaly:v1.30.0 \
|
||||
/config.yaml \
|
||||
--licenseFile=/license \
|
||||
--loggerLevel=INFO \
|
||||
@@ -182,7 +182,7 @@ services:
|
||||
# ...
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
image: victoriametrics/vmanomaly:v1.30.0
|
||||
# ...
|
||||
restart: always
|
||||
volumes:
|
||||
@@ -245,7 +245,7 @@ Before deploying, check the correctness of your configuration validate config fi
|
||||
|
||||
### Example
|
||||
|
||||
Here is an example of a config file that will run the [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) model on `vm_cache_entries` metric, with periodic scheduler that runs inference every minute and fits the model every day. The model will be trained on the last 2 weeks of data each time it is (re)fitted. The model will produce `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging purposes. The model will be timezone-aware and will use cyclical encoding for the hour of the day and day of the week seasonality.
|
||||
Here is an example of a config file that runs the online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model on a CPU metric. The scheduler runs inference every five minutes and performs a full refit only every 100 weeks; between refits the model updates causally from each inference batch. The initial fit uses four weeks of data. The model produces `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging, and its hour-of-day and day-of-week profiles follow the query timezone and daylight-saving-time changes.
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
@@ -260,38 +260,30 @@ settings:
|
||||
# scheduler: INFO
|
||||
# reader: INFO
|
||||
# writer: INFO
|
||||
model.prophet: WARNING
|
||||
model.online.temporal_envelope: WARNING
|
||||
|
||||
schedulers:
|
||||
1d_5m:
|
||||
100w_5m:
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler
|
||||
class: 'periodic'
|
||||
infer_every: '5m'
|
||||
scatter_infer_jobs: true
|
||||
fit_every: '1d'
|
||||
# Temporal Envelope learns online between full refits.
|
||||
fit_every: '100w'
|
||||
fit_window: '4w'
|
||||
|
||||
models:
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet
|
||||
prophet_model:
|
||||
class: 'prophet'
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope
|
||||
temporal_envelope_model:
|
||||
class: 'temporal_envelope'
|
||||
queries: ['cpu_user']
|
||||
schedulers: ['100w_5m']
|
||||
provide_series: ['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper'] # for debugging
|
||||
tz_aware: True # set to True if your data is timezone-aware, to deal with DST changes correctly
|
||||
tz_use_cyclical_encoding: True
|
||||
tz_seasonalities: # intra-day + intra-week seasonality
|
||||
- name: 'hod' # intra-day seasonality, hour of the day
|
||||
fourier_order: 4 # keep it 3-8 based on intraday pattern complexity
|
||||
prior_scale: 10
|
||||
- name: 'dow' # intra-week seasonality, time of the week
|
||||
fourier_order: 2 # keep it 2-4, as dependencies are learned separately for each weekday
|
||||
compression: # available since v1.28.1
|
||||
window: "30m" # downsample 5m data into 30m intervals before fitting
|
||||
agg_method: "mean" # use mean aggregation within each window
|
||||
adjust_boundaries: true # adjust confidence intervals after downsampling
|
||||
# inner model args (key-value pairs) accepted by
|
||||
# https://facebook.github.io/prophet/docs/quick_start#python-api
|
||||
args:
|
||||
interval_width: 0.98 # see https://facebook.github.io/prophet/docs/uncertainty_intervals
|
||||
seasonalities: ['hod_smooth', 'dow_smooth']
|
||||
alpha: 0.005 # trend reactivity; try 0.0025-0.02
|
||||
loss_reactivity: 5 # try 1-5; lower values reduce the influence of spikes
|
||||
iqr_threshold: 2 # try 1-4 to adjust data-driven interval width
|
||||
min_n_samples_seen: 16
|
||||
|
||||
reader:
|
||||
class: 'vm' # use VictoriaMetrics as a data source
|
||||
@@ -299,6 +291,7 @@ reader:
|
||||
datasource_url: "https://play.victoriametrics.com/" # [YOUR_DATASOURCE_URL]
|
||||
tenant_id: '0:0'
|
||||
sampling_period: "5m"
|
||||
tz: 'UTC' # set the IANA timezone that defines local calendar patterns, e.g. 'America/New_York'
|
||||
series_processing_batch_size: 8 # number of time series to process together while preparing data for fit or infer stages
|
||||
queries:
|
||||
# define your queries with MetricsQL - https://docs.victoriametrics.com/victoriametrics/metricsql/
|
||||
@@ -317,7 +310,7 @@ writer:
|
||||
|
||||
### UI
|
||||
|
||||
{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI can be used for prototyping and interactive experimenting to produce vmanomaly's and vmalert's configuration files. Please refer to the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for detailed instructions and examples. {{% available_from "v1.29.0" anomaly %}} Connect MCP server to the UI to benefit from better response quality and tool access in the UI Copilot, which provides AI-assisted configuration generation and debugging capabilities. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) for instructions on how to set it up.
|
||||
{{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI supports prototyping and interactive generation of `vmanomaly` and `vmalert` configuration files. See the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for instructions and examples. For optional AI-assisted workflows, use the [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance), connect the [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server), or follow the published [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills).
|
||||
|
||||

|
||||
> [!TIP]
|
||||
@@ -409,7 +402,7 @@ For optimal service behavior, consider the following tweaks when configuring `vm
|
||||
|
||||
- Set up **state restoration** {{% available_from "v1.24.0" anomaly %}} to resume from the last known state for long-term stability. This is controlled by the `settings.restore_state` boolean [arg](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration).
|
||||
|
||||
- Set up **config hot-reloading** {{% available_from "v1.25.0" anomaly %}} to automatically reload configurations on config files changes. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) and allows for configuration updates without explicit service restarts.
|
||||
- Set up **configuration hot reload** {{% available_from "v1.25.0" anomaly %}} to apply configuration-file changes automatically. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) to update the service without an explicit restart.
|
||||
|
||||
**Schedulers**:
|
||||
- Configure the **inference frequency** in the [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) section of the configuration file.
|
||||
|
||||
@@ -58,7 +58,7 @@ Get started with VictoriaMetrics Anomaly Detection by following our guides and i
|
||||
|
||||
- **Quickstart**: Learn how to quickly set up `vmanomaly` by following the [Quickstart Guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/).
|
||||
- **UI**: Explore anomaly detection configurations through the [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
|
||||
- **MCP**: Allow AI to assist you in generating service and alerting configurations, answering questions, planning migration with the [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly). Find the setup guide how to setup and use it [here](https://github.com/VictoriaMetrics/mcp-vmanomaly?tab=readme-ov-file#installation).
|
||||
- **AI-assisted setup (optional)**: Use the [UI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance), [vmanomaly MCP server](https://docs.victoriametrics.com/ai-tools/#vmanomaly-mcp-server), or [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) to inspect time-series characteristics, select and autotune models, and generate validated configurations.
|
||||
- **Integration**: Integrate anomaly detection into your existing observability stack. Find detailed steps [here](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/).
|
||||
- **Anomaly Detection Presets**: Enable anomaly detection on predefined sets of metrics. Learn more [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
|
||||
|
||||
|
||||
@@ -135,6 +135,8 @@ These alerting rules complements the [dashboard](#grafana-dashboard) to monitor
|
||||
`vmanomaly-health` alerting group:
|
||||
- **`TooManyRestarts`**: Triggers if an instance restarts more than twice within 15 minutes, suggesting the process might be crashlooping and needs investigation.
|
||||
- **`ServiceDown`**: Alerts if an instance is down for more than 5 minutes, indicating a service outage.
|
||||
- **`SchedulerWorkerDown`**: {{% available_from "v1.30.0" anomaly %}} Alerts when an individual periodic scheduler remains down for more than 2 minutes after automatic recovery attempts. This catches partial service failures that a process-level `ServiceDown` alert cannot detect.
|
||||
- **`FrequentSchedulerRestarts`**: {{% available_from "v1.30.0" anomaly %}} Warns when an individual scheduler has more than two restart attempts within 15 minutes. A single successful self-heal does not alert, while repeated attempts indicate a flapping scheduler or dependency.
|
||||
- **`ProcessNearFDLimits`**: Alerts when the number of available file descriptors falls below 100, which could lead to severe degradation if the limit is exhausted.
|
||||
- **`TooHighCPUUsage`**: Alerts when CPU usage exceeds 90% for a continuous 5-minute period, indicating possible resource exhaustion and the need to adjust resource allocation or load.
|
||||
- **`TooHighMemoryUsage`**: Alerts when RAM usage exceeds 85% for a continuous 5-minute period and the need to adjust resource allocation or load.
|
||||
|
||||
@@ -20,13 +20,13 @@ aliases:
|
||||
|
||||
## Introduction
|
||||
|
||||
{{% available_from "v1.26.0" anomaly %}} `vmanomaly` is shipped with a built-in [vmui-like](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) [UI](https://en.wikipedia.org/wiki/Graphical_user_interface) that provides an intuitive interface for rapid exploration of how anomaly detection models, their configurations and included domain knowledge impacts the results of anomaly detection, before such configurations are deployed in production.
|
||||
{{% available_from "v1.26.0" anomaly %}} `vmanomaly` includes a built-in [vmui-like](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) [UI](https://en.wikipedia.org/wiki/Graphical_user_interface) for exploring queries, comparing anomaly detection models, and tuning model or domain settings before production deployment.
|
||||
|
||||

|
||||
|
||||
## Accessing the UI
|
||||
|
||||
The UI is available at `http://<vmanomaly-host>:8490` by default, however, the port can be changed in `server` [section](https://docs.victoriametrics.com/anomaly-detection/components/server/) of the [configuration file](https://docs.victoriametrics.com/anomaly-detection/components/) using the `port` parameter:
|
||||
The UI is available at `http://<vmanomaly-host>:8490` by default. Change the port with `server.port` in the [configuration file](https://docs.victoriametrics.com/anomaly-detection/components/):
|
||||
|
||||
```yaml
|
||||
server:
|
||||
@@ -39,7 +39,7 @@ For impactful parameters please refer to [optimize resource usage](#optimize-res
|
||||
|
||||
## Playgrounds
|
||||
|
||||
To start exploring the UI, you can use embedded demo with preconfigured queries and models down below on public playgrounds (VictoriaMetrics, VictoriaLogs and VictoriaTraces):
|
||||
Try the UI with preconfigured queries and models in the public VictoriaMetrics, VictoriaLogs, and VictoriaTraces playgrounds:
|
||||
|
||||
{{% collapse name="Playground on VictoriaMetrics Datasource" %}}
|
||||
|
||||
@@ -151,10 +151,11 @@ users:
|
||||
Then, on [settings panel](#settings-panel) of the UI, set the URLs accordingly, also check the option to forward auth headers to the datasource:
|
||||
|
||||

|
||||
{class="w-50 mx-auto"}
|
||||
|
||||
### Pre-configured Datasource
|
||||
|
||||
{{% available_from "v1.28.2" anomaly %}} It is possible to disable the datasource selectors from UI (e.g. at purpose to serve internal teams) by using pre-configured one with respective environment variables at `vmanomaly` startup:
|
||||
{{% available_from "v1.28.2" anomaly %}} For a shared deployment with a fixed datasource, set these environment variables at `vmanomaly` startup. The UI then hides its datasource selectors:
|
||||
|
||||
- `VMANOMALY_UI_DATASOURCE_URL` - to set static datasource URL
|
||||
- `VMANOMALY_UI_DATASOURCE_TYPE` - to set datasource type, supported options are `vm` for VictoriaMetrics, `vmlogs` for both VictoriaLogs and VictoriaTraces.
|
||||
@@ -165,10 +166,9 @@ export VMANOMALY_UI_DATASOURCE_URL=https://play.victoriametrics.com/select/0:0/p
|
||||
export VMANOMALY_UI_DATASOURCE_TYPE=vm
|
||||
```
|
||||
|
||||
After that, start `vmanomaly` instance as usual, and the datasource selectors will be hidden from UI, while the pre-configured datasource will be used for all queries:
|
||||

|
||||
Start `vmanomaly` as usual. All UI queries will use the configured datasource:
|
||||
|
||||

|
||||

|
||||
|
||||
## Preset
|
||||
|
||||
@@ -193,9 +193,10 @@ The best applications of this mode are:
|
||||
|
||||
### What you can do with Copilot
|
||||
|
||||
- **Ask questions** about any model (e.g. [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
|
||||
- **Ask questions** about any model (e.g. [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope), [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
|
||||
- **Improve detection quality** - describe what's wrong ("too many false positives", "missing spikes") and Copilot reads the config, searches the docs, and proposes a validated configuration change to fix the issue.
|
||||
- **Get config suggestions inline** - suggestions appear as interactive cards with an explanation and a YAML diff; click **Apply** to write the change directly to your current settings, or **Decline** to keep the conversation going.
|
||||
- {{% available_from "v1.30.0" anomaly %}} **Profile and tune the real query** - with [mcp-vmanomaly](#mcp-tools-server) connected, Copilot can inspect bounded time-series characteristics, recommend an online model, start an asynchronous autotune task, and apply its validated query and model suggestions.
|
||||
|
||||
### How it works
|
||||
|
||||
@@ -212,7 +213,7 @@ AI Assistant is disabled by default; enable it with `VMANOMALY_COPILOT_ENABLED=t
|
||||
Supported providers and model formats:
|
||||
|
||||
- **Anthropic** - set `ANTHROPIC_API_KEY`; model format: `anthropic:<model>`
|
||||
- Examples: `claude-haiku-4-5`, `claude-sonnet-4-6`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
|
||||
- Examples: `claude-haiku-4-5`, `claude-sonnet-5`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
|
||||
- **OpenAI** - set `OPENAI_API_KEY`; model format: `openai:<model>` or `openai-responses:<model>`
|
||||
- Examples: `gpt-5-mini`, `gpt-5.2`; see [full list](https://platform.openai.com/docs/models)
|
||||
- {{% available_from "v1.29.1" anomaly %}} OpenAI-compatible non-OpenAI providers are supported through `OPENAI_BASE_URL` + `OPENAI_API_KEY`
|
||||
@@ -315,14 +316,14 @@ docker run -it --rm \
|
||||
-e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
|
||||
-p 8080:8080 \
|
||||
-p 8490:8490 \
|
||||
victoriametrics/vmanomaly:v1.29.7 \
|
||||
victoriametrics/vmanomaly:v1.30.0 \
|
||||
vmanomaly_config.yaml
|
||||
```
|
||||
|
||||
|
||||
## UI Navigation
|
||||
|
||||
The vmanomaly UI provides a user-friendly interface for exploring and configuring anomaly detection models. The main components of the UI include:
|
||||
The UI has four main areas:
|
||||
|
||||
- [**Query Explorer**](#query-explorer): A vmui-like interface for typing and executing MetricsQL/LogsQL queries to visualize data.
|
||||
- [**Model Panel**](#model-panel): A form for editing anomaly detection model hyperparameters and applying domain knowledge settings.
|
||||
@@ -331,7 +332,7 @@ The vmanomaly UI provides a user-friendly interface for exploring and configurin
|
||||
|
||||
### Query Explorer
|
||||
|
||||
The Query Explorer provides a vmui-like interface for typing and executing MetricsQL/LogsQL queries to visualize data.
|
||||
Use Query Explorer to run MetricsQL or LogsQL queries and visualize input data.
|
||||
|
||||

|
||||
|
||||
@@ -345,23 +346,23 @@ Users can:
|
||||
|
||||
### Visualization Panel
|
||||
|
||||
The Visualization Panel has 2 modes of displaying data - either raw queried data or data with detected anomalies, depending on the action taken in the Model Panel.
|
||||
The Visualization Panel displays either raw query results or model output, depending on the selected action.
|
||||
|
||||
**Visualizations of the queried data** ("Execute Query" button)
|
||||
After selecting **Execute Query**:
|
||||
|
||||

|
||||
|
||||
> All the metrics are shown in a single plot, similar to vmui, with zooming and panning capabilities.
|
||||
All returned series appear in one vmui-like plot with zooming and panning.
|
||||
|
||||
**Initial data with detected anomalies** ("Detect Anomalies" button)
|
||||
After selecting **Detect Anomalies**:
|
||||
|
||||

|
||||
|
||||
> The plot shows the queried data, **grouped by individual series**, iterated over legend, with the actual values (`y`) compared to the expected values (model predictions, `y_hat`), confidence intervals (`y_hat_lower`, `y_hat_upper`), and detected anomalies. The anomalies are marked with red circles, and hovering over them provides additional information such as the anomaly score and associated labels.
|
||||
The plot groups model output by input series and compares actual values (`y`) with predictions (`yhat`), confidence intervals (`yhat_lower`, `yhat_upper`), and detected anomalies. Hover over an anomaly marker to inspect its score and labels.
|
||||
|
||||
Also, timeseries (such as `y`, `y_hat`, etc.) can be toggled on/off by clicking on the legend items.
|
||||
Toggle individual output series from the legend.
|
||||
|
||||
{{% available_from "v1.29.2" anomaly %}} Seeing model [business-boundaries](https://docs.victoriametrics.com/anomaly-detection/faq/#incorporating-domain-knowledge), such as [detection direction](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) and minimal deviation from expected ([absolute](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-deviation-from-expected) and [relative](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-relative-deviation-from-expected) combined) can be turned on with "business boundaries" toggle. Showing/hiding individual bands can be done by clicking on the respective legend items, while showing/hiding all business boundaries at once can be done with "business boundaries" toggle.
|
||||
{{% available_from "v1.29.2" anomaly %}} Enable **Business Boundaries** to overlay the configured [detection direction](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) and combined [absolute](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-deviation-from-expected) and [relative](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-relative-deviation-from-expected) minimum-deviation bands. Toggle individual bands from the legend or all bands with the main control.
|
||||
|
||||
[Back to UI navigation](#ui-navigation)
|
||||
|
||||
@@ -371,18 +372,18 @@ Also, timeseries (such as `y`, `y_hat`, etc.) can be toggled on/off by clicking
|
||||
|
||||
The Model Panel provides:
|
||||
|
||||
Parameters, such as "Fit Every", "Fit Window" and {{% available_from "v1.28.0" anomaly %}} "Infer Every" to imitate [production scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler), as well as overriding default [anomaly detection threshold](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) (1.0).
|
||||
- Scheduling controls such as **Fit Every**, **Fit Window**, and {{% available_from "v1.28.0" anomaly %}} **Infer Every** for imitating [production scheduling](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler).
|
||||
- An override for the default [anomaly detection threshold](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) of `1.0`.
|
||||
- Actions for running or canceling detection, downloading results, and exporting model configurations or example alerting rules.
|
||||
|
||||
> {{% available_from "v1.28.0" anomaly %}} "Exact" mode checkbox is used in combination with "Infer Every" control for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) such as `mad_online` or `quantile_online`, to provide unbiased estimates of how production scheduler would perform anomaly detection on incoming data streams. In "exact" mode, the model is updated exactly at every "infer every" micro-batch interval, at a cost of increased computation time.
|
||||
> {{% available_from "v1.28.0" anomaly %}} For [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models), combine **Exact** mode with **Infer Every** to reproduce causal production micro-batches. This gives a more representative backtest at the cost of additional computation.
|
||||
|
||||
Controls for running/canceling anomaly detection on the queried data, downloading the results as CSV/JSON, accessing and downloading the model configuration or example alerting rules in YAML format.
|
||||
A form-based menu configures model hyperparameters and domain knowledge:
|
||||
|
||||
A form-based menu for finetuning model hyperparameters and applying domain knowledge settings:
|
||||
|
||||
- Model type selection (e.g., rolling quantile, Prophet, etc.)
|
||||
- Model selection, for example Temporal Envelope or Online MAD.
|
||||

|
||||
- Wizard with **model-agnostic parameters** (e.g., detection direction, data range, scale, clipping, minimum deviation from expected, etc.) and **model-specific hyperparameters** for chosen model type (e.g., quantile and window steps for [rolling quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-quantile) model). {{% available_from "v1.27.0" anomaly %}} autocomplete of example parameters by hitting Tab key is supported.
|
||||

|
||||
- A wizard with **model-agnostic settings** such as detection direction, data range, clipping, and minimum deviation, plus the selected model's hyperparameters. {{% available_from "v1.27.0" anomaly %}} Press **Tab** to autocomplete suggested values.
|
||||
<a class="content-image d-flex justify-content-center" data-bs-target="#image-modal" data-bs-toggle="modal" href="/anomaly-detection/vmanomaly-ui-model-config-wizard.webp"><img alt="vmanomaly-ui-model-config-wizard" class="w-75 mx-auto" src="/anomaly-detection/vmanomaly-ui-model-config-wizard.webp" style="min-width: 0;" /></a>
|
||||
|
||||
[Back to UI navigation](#ui-navigation)
|
||||
|
||||
@@ -397,6 +398,7 @@ The vmui-like "Settings" panel allows users to configure global settings and pre
|
||||
- {{% available_from "v1.27.0" anomaly %}} Auth Headers forwarding to datasource (VictoriaMetrics, VictoriaLogs).
|
||||
|
||||

|
||||
{class="w-50 mx-auto"}
|
||||
|
||||
[Back to navigation](#ui-navigation)
|
||||
|
||||
@@ -585,6 +587,7 @@ Set the "Fit Every" and "Fit Window" parameters to control how often and over wh
|
||||
Tune the model hyperparameters and apply domain knowledge settings using the form-based menu in the Model Panel. See (i) tooltips for parameter descriptions and [model documentation](https://docs.victoriametrics.com/anomaly-detection/components/models/) link for recommended values and guidelines.
|
||||
|
||||

|
||||
{class="w-75 mx-auto"}
|
||||
|
||||
For example, for a **MAD online** [model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad), that should be run on a query, returning per-mode CPU utilization (as fractions of 1, data range `[0, 1]`), where you are interested in capturing **spikes of at least 6% deviations** from expected behavior:
|
||||
|
||||
@@ -640,6 +643,31 @@ If the **results** look good and the **model configuration should be deployed in
|
||||
|
||||
## Changelog
|
||||
|
||||
{{% collapse name="Release history" %}}
|
||||
|
||||
### v1.8.0
|
||||
Released: 2026-07-23
|
||||
|
||||
vmanomaly version: [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300)
|
||||
|
||||
- FEATURE: UX Improvements, refreshed interface with clearer model and query forms, inline configuration previews, and fullscreen chart controls.
|
||||
|
||||
- FEATURE: Server (production-running) models and queries can be accessed and selected from the UI, with a new "Queries" button, while model selection now includes a drop-down for server-configured scheduled models in model wizard.
|
||||
|
||||
- IMPROVEMENT: Boosted [AI Copilot](#ai-assistance) stability and suggestions quality, aligned with MCP/skills toolset, and proper handling of canceled or incomplete tool calls.
|
||||
|
||||
- IMPROVEMENT: Added query prettification and stable value formatting, including platform-aware keyboard-shortcut hints.
|
||||
|
||||
- IMPROVEMENT: Results now visibly switch to an out-of-date state after the query, time range, or model configuration changes. The action updates to rerun detection and remains visible in both expanded and collapsed model views.
|
||||
|
||||
- IMPROVEMENT: Chart range navigation commits one query on interaction completion instead of issuing many intermediate requests.
|
||||
|
||||
- IMPROVEMENT: Compact consecutive identical [AI Copilot](#ai-assistance) tool calls, keep query/model/anomaly suggestions synchronized, and recover cleanly from canceled or incomplete tool calls.
|
||||
|
||||
- BUGFIX: Fixed exact UI backtesting across multiple fit cycles and for auto-tuned online wrappers, preventing *duplicate or missing predictions* while retaining causal model updates and restoration of compatible legacy auto-tuned state.
|
||||
|
||||
- BUGFIX: Kept automatic trailing-slash redirects for configured path prefixes and `/vmui` relative to the public origin, preventing internal backend hostnames from leaking through reverse proxies such as `vmauth`.
|
||||
|
||||
### v1.7.2
|
||||
Released: 2026-06-25
|
||||
|
||||
@@ -809,3 +837,5 @@ Released: 2025-10-02
|
||||
vmanomaly version: [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1260)
|
||||
|
||||
Initial public release of the vmanomaly UI.
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
@@ -6,7 +6,7 @@ build:
|
||||
sitemap:
|
||||
disable: true
|
||||
---
|
||||
This chapter describes different components, that correspond to respective sections of a config to launch VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/) service:
|
||||
This chapter describes the configuration sections used to run VictoriaMetrics Anomaly Detection, or [`vmanomaly`](https://docs.victoriametrics.com/anomaly-detection/):
|
||||
|
||||
- [Model(s) section](https://docs.victoriametrics.com/anomaly-detection/components/models/) - Required
|
||||
- [Reader section](https://docs.victoriametrics.com/anomaly-detection/components/reader/) - Required
|
||||
@@ -16,9 +16,9 @@ This chapter describes different components, that correspond to respective secti
|
||||
- [Settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/) - Optional
|
||||
- [Server section](https://docs.victoriametrics.com/anomaly-detection/components/server/) - Optional
|
||||
|
||||
> Once the service starts, automated config validation is performed {{% available_from "v1.7.2" anomaly %}}. Please see container logs for errors that need to be fixed to create fully valid config, visiting sections above for examples and documentation.
|
||||
> The service validates its configuration at startup{{% available_from "v1.7.2" anomaly %}}. Check the container logs for validation errors and use the sections above for field descriptions and examples.
|
||||
|
||||
> Components' class {{% available_from "v1.13.0" anomaly %}} can be referenced by a short alias instead of a full class path - i.e. `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, `scheduler.periodic.PeriodicScheduler` becomes `periodic`, etc. Please see according sections for the details.
|
||||
> Component classes{{% available_from "v1.13.0" anomaly %}} can be referenced by short aliases instead of full import paths. For example, `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, and `scheduler.periodic.PeriodicScheduler` becomes `periodic`.
|
||||
|
||||
> `preset` modes are available {{% available_from "v1.13.0" anomaly %}} for `vmanomaly`. Please find the guide [here](https://docs.victoriametrics.com/anomaly-detection/presets/).
|
||||
|
||||
@@ -32,7 +32,7 @@ Below, you will find an example illustrating how the components of `vmanomaly` i
|
||||
|
||||
## Example config
|
||||
|
||||
Here's a minimalistic full config example, demonstrating many-to-many configuration (actual for [latest version](https://docs.victoriametrics.com/anomaly-detection/changelog/)):
|
||||
The following minimal configuration demonstrates current many-to-many model, query, and scheduler mapping:
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
@@ -52,7 +52,7 @@ schedulers:
|
||||
scatter_infer_jobs: true # distribute infer jobs evenly across the infer interval to reduce synchronized bursts
|
||||
fit_every: "365d" # how often to re-fit the models, for online models used effectively once, then they are updated with new data and won't require re-fit
|
||||
fit_window: "3d" # how much historical data to use for fit stage
|
||||
start_from: "00:00" # start from specified time, i.e. 00:00 given timezone and do daily fits as `fit_every` is 1 day
|
||||
start_from: "00:00" # align the annual fit schedule to midnight in the configured timezone
|
||||
tz: "Europe/Kyiv" # timezone to use for start_from
|
||||
periodic_offline_1w:
|
||||
class: 'periodic'
|
||||
@@ -146,14 +146,14 @@ server:
|
||||
|
||||
> This feature is better used in conjunction with [stateful service](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the state of the models and schedulers between restarts and reuse what can be reused, thus avoiding unnecessary re-training of models, re-initialization of schedulers and re-reading of data.
|
||||
|
||||
{{% available_from "v1.25.0" anomaly %}} Service supports hot reload of configuration files, which allows for automatic reloading of configurations on config files change without the need of explicit service restart. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). `vmanomaly_config_reload_enabled` flag in [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) will be set to 1 (if enabled) or 0 (if disabled).
|
||||
{{% available_from "v1.25.0" anomaly %}} The service supports hot reload of configuration files, applying changes without an explicit restart. Enable it with the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). The `vmanomaly_config_reload_enabled` [self-monitoring metric](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) is `1` when hot reload is enabled and `0` otherwise.
|
||||
|
||||
> [!NOTE]
|
||||
> {{% deprecated_from "v1.29.5" anomaly %}} File system event-based hot reload has been deprecated in favor of content-based polling with configurable `-configCheckInterval` due to reliability issues with Kubernetes ConfigMap symlink rotations and other filesystems where event delivery can be inconsistent. If you were using file system event-based hot reload, please switch to content-based polling by enabling `--watch` flag and configuring `-configCheckInterval` as needed.
|
||||
|
||||
### How it works
|
||||
|
||||
It works by checking watched `.yml|.yaml` file contents in the specified files or directories on the configured interval `-configCheckInterval` (default is `30s`) {{% available_from "v1.29.5" anomaly %}}. When a content change is detected, the service will attempt to reload the configuration files after the existing debounce window, rebuild the [global config](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration) and reinitialize the components. If the reload is successful, the `vmanomaly_config_reloads_total` metric will be incremented for `status="success"` label, otherwise it will be incremented with `status="failure"` label and a respective error message on config validation failure(s) will be logged.
|
||||
The service checks watched `.yml` and `.yaml` files at the `-configCheckInterval` interval (default `30s`){{% available_from "v1.29.5" anomaly %}}. When it detects a content change, it waits for the debounce window, rebuilds the [global configuration](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration), and reinitializes the components. The `vmanomaly_config_reloads_total` metric is incremented with `status="success"` or `status="failure"`; validation failures are also logged.
|
||||
|
||||
> If the reload fails, the service will log an error message indicating the reason for the failure, and the **previous configuration will remain active until a successful reload occurs** to preserve the service's stability. This means that if there are errors in the new configuration, the service will continue to operate with the last valid configuration until the issues are resolved.
|
||||
|
||||
|
||||
@@ -304,7 +304,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
|
||||
<span style="white-space: nowrap;">`vmanomaly_available_memory_bytes`</span>
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>Virtual memory size in bytes, available to the process{{% available_from "v1.18.4" anomaly %}}.</td>
|
||||
<td>Effective memory capacity available to the process in bytes{{% available_from "v1.18.4" anomaly %}}. The value honors cgroup limits when available, then process address-space limits, and otherwise reports host physical memory. It does not represent currently unused memory.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
@@ -312,7 +312,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
|
||||
<span style="white-space: nowrap;">`vmanomaly_cpu_cores_available`</span>
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>Number of (logical) CPU cores available to the process{{% available_from "v1.18.4" anomaly %}}.</td>
|
||||
<td>Effective CPU capacity available to the process{{% available_from "v1.18.4" anomaly %}}, constrained by host logical CPUs, process affinity, and cgroup quota. The value can be fractional when a fractional CPU quota is configured.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
@@ -320,7 +320,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
|
||||
<span style="white-space: nowrap;">`vmanomaly_config_entities`</span>
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>Number of [sub-configs](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#sub-configuration) **available** (`{scope="total"}`) and **used** for particular [shard](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability) (`{scope="shard"}`) {{% available_from "v1.21.0" anomaly %}}</td>
|
||||
<td>Number of [sub-configs](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#sub-configuration) **available** (`scope="total"`) and **used** by the current [shard](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability) (`scope="shard"`){{% available_from "v1.21.0" anomaly %}}, labeled by `preset` and `scope`.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
@@ -352,6 +352,34 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
|
||||
<td>Gauge</td>
|
||||
<td>Timestamp of the last successful config [hot-reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) in seconds since epoch {{% available_from "v1.25.1" anomaly %}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<span style="white-space: nowrap;">`vmanomaly_scheduler_alive`</span>
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>Whether the scheduler worker thread identified by `scheduler_alias` and `preset` is alive (`1`) or not (`0`) {{% available_from "v1.30.0" anomaly %}}.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<span style="white-space: nowrap;">`vmanomaly_scheduler_restarts_total`</span>
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>Number of bounded scheduler restart attempts {{% available_from "v1.30.0" anomaly %}}, labeled by `scheduler_alias`, `preset`, and `status` (`success` or `failure`).</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<span style="white-space: nowrap;">`vm_license_expires_at`</span>
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>License expiration time as a Unix timestamp in seconds. See the [licensing section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#licensing) for example alerts.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<span style="white-space: nowrap;">`vm_license_expires_in_seconds`</span>
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>Time remaining until license expiration in seconds. See the [licensing section](https://docs.victoriametrics.com/anomaly-detection/quickstart/#licensing) for warning and critical alert examples.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -425,7 +453,7 @@ Label names [description](#labelnames)
|
||||
|
||||
`Histogram` (was `Summary`{{% deprecated_from "v1.17.0" anomaly %}})
|
||||
</td>
|
||||
<td>The total time (in seconds) taken for data parsing at each `step` (json, dataframe) for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
|
||||
<td>The total time (in seconds) taken for data parsing at each `step` (`json` or `df`) for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
|
||||
<td>
|
||||
|
||||
`step`, `url`, `query_key`, `scheduler_alias`, `preset`
|
||||
@@ -521,7 +549,7 @@ Label names [description](#labelnames)
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`Histogram`</span> (was `Summary`{{% deprecated_from "v1.17.0" anomaly %}}) </td>
|
||||
<td>The total time (in seconds) taken by model invocations during the `stage` (`fit`, `infer`, `fit_infer`), based on the results of the `query_key` query, for models of class `model_alias`, within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
|
||||
<td>The model-service stage duration in seconds for `fit`, `infer`, or combined `fit_infer` execution, based on the results of the `query_key` query for `model_alias`. Reader and writer I/O durations are reported by their respective metrics.</td>
|
||||
<td>
|
||||
|
||||
`stage`, `query_key`, `model_alias`, `scheduler_alias`, `preset`
|
||||
@@ -536,7 +564,7 @@ Label names [description](#labelnames)
|
||||
|
||||
`Counter`
|
||||
</td>
|
||||
<td>The number of datapoints accepted (excluding NaN or Inf values) by models of class `model_alias` from the results of the `query_key` query during the `stage` (`infer`, `fit_infer`), within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
|
||||
<td>The number of valid datapoints accepted by `model_alias`, excluding NaN and Inf values, during `fit`, `infer`, or combined `fit_infer` execution for the `query_key` query.</td>
|
||||
<td>
|
||||
|
||||
`stage`, `query_key`, `model_alias`, `scheduler_alias`, `preset`
|
||||
@@ -641,7 +669,7 @@ Label names [description](#labelnames)
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`vmanomaly_writer_responses`</span> (named `vmanomaly_reader_response_count`{{% deprecated_from "v1.17.0" anomaly %}})
|
||||
<span style="white-space: nowrap;">`vmanomaly_writer_responses`</span> (named `vmanomaly_writer_response_count`{{% deprecated_from "v1.17.0" anomaly %}})
|
||||
</td>
|
||||
<td>
|
||||
|
||||
@@ -720,14 +748,14 @@ Label names [description](#labelnames)
|
||||
|
||||
### Labelnames
|
||||
|
||||
* `stage` - stage of model - 'fit', 'infer' or 'fit_infer' for models that do it simultaneously, see [model types](https://docs.victoriametrics.com/anomaly-detection/components/models/#model-types).
|
||||
* `stage` - model execution stage: `fit`, `infer`, or `fit_infer` for a combined fit/inference scheduler run. See [model types](https://docs.victoriametrics.com/anomaly-detection/components/models/#model-types).
|
||||
* `query_key` - query alias from [`reader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/) config section.
|
||||
* `model_alias` - model alias from [`models`](https://docs.victoriametrics.com/anomaly-detection/components/models/) config section{{% available_from "v1.10.0" anomaly %}}.
|
||||
* `scheduler_alias` - scheduler alias from [`schedulers`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) config section{{% available_from "v1.11.0" anomaly %}}.
|
||||
* `preset` - preset alias for [`preset`](https://docs.victoriametrics.com/anomaly-detection/presets/) mode of `vmanomaly`{{% available_from "v1.12.0" anomaly %}}.
|
||||
* `url` - writer or reader url endpoint.
|
||||
* `code` - response status code or `connection_error`, `timeout`.
|
||||
* `step` - json or dataframe reading step.
|
||||
* `code` - HTTP response status code or `connection_error`, `timeout`, `ssl_error`, or `io_error`.
|
||||
* `step` - reader parsing step: `json` or `df`.
|
||||
|
||||
[Back to metric sections](#metrics-generated-by-vmanomaly)
|
||||
|
||||
@@ -902,14 +930,14 @@ The `model` component (wrapped in service) logs operations during the fitting an
|
||||
|
||||
**Skipped runs**. When there are insufficient valid data points to fit or infer using a model, the run is skipped and a `warning` log is generated. This can occur when the query returns no new data or when the data contains invalid values (e.g., `NaN`, `INF`). The skipped run is also reflected in the `vmanomaly_model_runs_skipped` metric. Log messages:
|
||||
|
||||
When there are insufficient valid data points (at least 1 for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) and 2 for [offline models](https://docs.victoriametrics.com/anomaly-detection/components/models/#offline-models))
|
||||
When there are no valid datapoints available for fitting
|
||||
```text
|
||||
[Scheduler {{scheduler_alias}}] Skipping run for stage 'fit' for model '{{model_alias}}' (query_key: {{query_key}}): Not enough valid data to fit: {{valid_values_cnt}}
|
||||
```
|
||||
|
||||
When all the received timestamps during an `infer` call have already been processed, meaning the [`anomaly_score`](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) has already been produced for those points
|
||||
```text
|
||||
[Scheduler {{scheduler_alias}}] Skipping run for stage 'infer' for model '{{model_alias}}' (query_key: {{query_key}}): No unseen data to infer on.
|
||||
[Scheduler {{scheduler_alias}}] Skipping run for stage 'infer' for model '{{model_alias}}' (query_key: {{query_key}}): No unseen valid data to infer on.
|
||||
```
|
||||
When the model fails to produce any valid or finite outputs (such as [`anomaly_score`](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score))
|
||||
```text
|
||||
@@ -937,14 +965,14 @@ When the model fails to produce any valid or finite outputs (such as [`anomaly_s
|
||||
|
||||
**Successful model runs**. When a model successfully fits, logs track the number of valid datapoints processed and the time taken for the operation. These logs are accompanied by updates to [self-monitoring metrics](#models-behaviour-metrics) like `vmanomaly_model_runs`, `vmanomaly_model_run_duration_seconds`, `vmanomaly_model_datapoints_accepted`, and `vmanomaly_model_datapoints_produced`. Log messages:
|
||||
|
||||
For [non-rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#non-rolling-models)
|
||||
For a `fit` stage
|
||||
```text
|
||||
[Scheduler {{scheduler_alias}}] Fitting on {{valid_values_cnt}}/{{total_values_cnt}} valid datapoints for "{{query_key}}" using model "{{model_alias}}".
|
||||
```
|
||||
```text
|
||||
[Scheduler {{scheduler_alias}}] Model '{{model_alias}}' fit completed in {{model_run_duration}} seconds for {{query_key}}.
|
||||
```
|
||||
For [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) (combined stage)
|
||||
For a combined `fit_infer` stage
|
||||
```text
|
||||
[Scheduler {{scheduler_alias}}] Fit-Infer on {{datapoint_count}} points for "{{query_key}}" using model "{{model_alias}}".
|
||||
```
|
||||
@@ -1032,6 +1060,6 @@ The label set for the metric {{query_key}} contains multi-tenancy labels, but th
|
||||
|
||||
- `vmanomaly_writer_timeseries_sent`: Tracks the number of timeseries sent to VictoriaMetrics.
|
||||
|
||||
**Metrics skipped in case of failures**. If an error occurs (connection, timeout, or I/O error), only `vmanomaly_writer_request_duration_seconds` is updated with appropriate error code.
|
||||
**Metrics updated in case of failures**. If an SSL, connection, timeout, or I/O error occurs, `vmanomaly_writer_request_duration_seconds` records the complete failed request duration, including connection retries, and `vmanomaly_writer_responses` records the corresponding value in its `code` label. Successfully sent byte and datapoint counters are not incremented.
|
||||
|
||||
[Back to logging sections](#logs-generated-by-vmanomaly)
|
||||
|
||||
@@ -12,15 +12,14 @@ aliases:
|
||||
- /anomaly-detection/components/reader.html
|
||||
---
|
||||
|
||||
VictoriaMetrics Anomaly Detection (`vmanomaly`) has an input of Prometheus-compatible metrics from either [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) accessed with [VmReader](#vm-reader) with [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) queries or from [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) / [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) accessed with [VLogsReader](#victorialogs-reader) with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries.
|
||||
VictoriaMetrics Anomaly Detection (`vmanomaly`) reads Prometheus-compatible metrics from [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) through [VmReader](#vm-reader) and [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/). It can also derive metrics from [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) or [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) through [VLogsReader](#victorialogs-reader) and [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/).
|
||||
|
||||
Future updates will introduce additional readers, expanding the range of data sources `vmanomaly` can work with.
|
||||
|
||||
## Playgrounds
|
||||
|
||||
To ease the development and testing of queries for `vmanomaly`'s input data, following playgrounds can be used for experimenting with MetricsQL and LogsQL queries:
|
||||
Use the following playgrounds to develop and test input queries:
|
||||
|
||||
Please see respective sections below for specific reader:
|
||||
- [MetricsQL playground](#metricsql-playground) for `VmReader`
|
||||
- [LogsQL playground](#logsql-playground) for `VLogsReader`
|
||||
|
||||
@@ -28,7 +27,7 @@ Please see respective sections below for specific reader:
|
||||
|
||||
{{% collapse name="Queries format migration (to v1.13.0+)" %}}
|
||||
|
||||
> There is backward-compatible change{{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg of [VmReader](#vm-reader). New format allows to specify per-query parameters, like `step` to reduce amount of data read from VictoriaMetrics TSDB and to allow config flexibility. Please see [per-query parameters](#per-query-parameters) section for the details.
|
||||
> The backward-compatible `queries` format introduced in v1.13.0 allows [VmReader](#vm-reader) parameters such as `step` to be configured per query. This can reduce the amount of data read from VictoriaMetrics. See [per-query parameters](#per-query-parameters) for details.
|
||||
|
||||
Old format like
|
||||
|
||||
@@ -263,7 +262,33 @@ BasicAuth password. If set, it will be used to authenticate the request.
|
||||
`30s`
|
||||
</td>
|
||||
<td>
|
||||
Timeout for the requests, passed as a string
|
||||
Backward-compatible timeout used for both datasource fetches and post-fetch processing when `fetch_timeout` or `processing_timeout` are not set.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`fetch_timeout`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Not set (`timeout` fallback)
|
||||
</td>
|
||||
<td>
|
||||
Optional timeout {{% available_from "v1.30.0" anomaly %}} for each datasource read request. Use values such as `5s`, `30s`, or `1m`.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`processing_timeout`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Not set (`timeout` fallback)
|
||||
</td>
|
||||
<td>
|
||||
Optional timeout {{% available_from "v1.30.0" anomaly %}} for post-fetch processing that prepares returned data for fit or inference. High-cardinality queries may need a larger processing timeout than their datasource fetch timeout.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -362,6 +387,19 @@ If True, then query will be performed from the last seen timestamp for a given s
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`query_last_seen_max_lookback`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`None`
|
||||
</td>
|
||||
<td>
|
||||
Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far `query_from_last_seen_timestamp` may move a query start into the past to recover skipped inference intervals. When configured below the query step, the effective cap is raised to one step. Examples: `5m`, `1h`.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`latency_offset`</span>
|
||||
</td>
|
||||
<td>
|
||||
@@ -395,7 +433,7 @@ Optional arg{{% available_from "v1.17.0" anomaly %}} overrides how `search.maxPo
|
||||
`UTC`
|
||||
</td>
|
||||
<td>
|
||||
Optional argument{{% available_from "v1.18.0" anomaly %}} specifies the [IANA](https://nodatime.org/TimeZones) timezone to account for local shifts, like [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models sensitive to seasonal patterns (e.g., [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)). Defaults to `UTC` if not set and can be overridden on a [per-query basis](#per-query-parameters).
|
||||
Optional argument {{% available_from "v1.18.0" anomaly %}} specifies the [IANA](https://nodatime.org/TimeZones) timezone to account for local shifts, like [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models sensitive to seasonal patterns (e.g., [`TemporalEnvelopeModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope), [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)). Defaults to `UTC` if not set and can be overridden on a [per-query basis](#per-query-parameters).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -461,6 +499,9 @@ reader:
|
||||
# tenant_id: '1:0' # if set, overrides reader-level tenant_id
|
||||
# offset: '-15s' # if set, overrides reader-level offset
|
||||
sampling_period: '1m'
|
||||
timeout: '30s' # backward-compatible default for both phases
|
||||
fetch_timeout: '30s' # timeout for each datasource request, overrides `timeout` if set
|
||||
processing_timeout: '1m' # timeout for preparing fetched series for fit/infer, overrides `timeout` if set
|
||||
query_from_last_seen_timestamp: True # false by default
|
||||
latency_offset: '1ms'
|
||||
series_processing_batch_size: 8
|
||||
@@ -530,7 +571,7 @@ reader:
|
||||
|
||||
### Healthcheck metrics
|
||||
|
||||
`VmReader` exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
|
||||
`VmReader` exposes [several health metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
|
||||
|
||||
|
||||
## VictoriaLogs reader
|
||||
@@ -800,7 +841,33 @@ Frequency of the points returned. Will be converted to `/select/stats_query_rang
|
||||
`30s`
|
||||
</td>
|
||||
<td>
|
||||
(Optional) Specifies the maximum duration to wait for a query to complete before timing out. Can be set on a [per-query basis](#per-query-parameters-1) to override the reader-level setting.
|
||||
(Optional) Backward-compatible timeout used for both datasource fetches and post-fetch processing when `fetch_timeout` or `processing_timeout` are not set.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`fetch_timeout`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Not set (`timeout` fallback)
|
||||
</td>
|
||||
<td>
|
||||
Optional timeout {{% available_from "v1.30.0" anomaly %}} for each datasource read request. Use values such as `5s`, `30s`, or `1m`.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`processing_timeout`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Not set (`timeout` fallback)
|
||||
</td>
|
||||
<td>
|
||||
Optional timeout {{% available_from "v1.30.0" anomaly %}} for post-fetch processing that prepares returned data for fit or inference. High-cardinality results may need a larger processing timeout than their datasource fetch timeout.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -906,6 +973,19 @@ If a path to a CA bundle file (like `ca.crt`), it will verify the certificate us
|
||||
Optional argument {{% available_from "v1.29.7" anomaly %}}, allows specifying the number of time series to process together while preparing data for fit or infer stages. Defaults to `8`. Suggested values are 4-16 for high-cardinality queries.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
<span style="white-space: nowrap;">`query_last_seen_max_lookback`</span>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`None`
|
||||
</td>
|
||||
<td>
|
||||
Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far last-seen recovery may move a query start into the past. Examples: `5m`, `1h`.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -927,7 +1007,9 @@ reader:
|
||||
series_processing_batch_size: 8
|
||||
data_range: [0, 'inf'] # reader-level
|
||||
offset: '0s' # reader-level
|
||||
timeout: '30s'
|
||||
timeout: '30s' # backward-compatible default for both phases
|
||||
fetch_timeout: '30s' # timeout for each datasource request, overrides `timeout` if set
|
||||
processing_timeout: '1m' # timeout for preparing fetched series for fit/infer, overrides `timeout` if set
|
||||
queries:
|
||||
# one query returning 1 result fields (avg_duration), it will have __name__ label (series name) as `duration_30m__avg`
|
||||
duration_avg_30m:
|
||||
@@ -960,4 +1042,4 @@ Please refer to the [mTLS protection](#mtls-protection) section above for detail
|
||||
|
||||
### Healthcheck metrics
|
||||
|
||||
Similarly to `VmReader`, `VLogsReader` also exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
|
||||
Like `VmReader`, `VLogsReader` exposes [several health metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
|
||||
|
||||
@@ -72,6 +72,8 @@ options={`"scheduler.periodic.PeriodicScheduler"`, `"scheduler.oneoff.OneoffSche
|
||||
|
||||
> If `start_from` [parameter](#parameters-1) is used, it's suggested to also set `restore_state: true` in the [Settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) of a config, so that the scheduler can restore its state from the previous run **if terminated or restarted in between scheduled runs** and continue producing anomaly scores without interruptions, otherwise the service will be idle until future `start_from` time is reached. E.g. if `start_from` is set to `20:00` and the service is started and then terminated and restarted at `20:30`, it will not produce any anomaly scores until the next day's `20:00` is reached (+23:30 of being idle), which introduces inconvenience for the users.
|
||||
|
||||
> {{% available_from "v1.30.0" anomaly %}} If a periodic scheduler worker exits unexpectedly, the service attempts bounded restarts with exponential backoff instead of shutting down unrelated schedulers. Monitor [`vmanomaly_scheduler_alive`](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) and `vmanomaly_scheduler_restarts_total` to alert on persistent failures.
|
||||
|
||||
### Parameters
|
||||
|
||||
For periodic scheduler parameters are defined as differences in times, expressed in difference units, e.g. days, hours, minutes, seconds. Time granularity is defined by the last characters of a string. Examples: `"50s"` (seconds), `"4m"` (minutes), `"3h"` (hours), `"2d"` (days), `"1w"` (weeks).
|
||||
|
||||
@@ -64,3 +64,15 @@ reader:
|
||||
After starting the `vmanomaly` server with the above configuration, UI can be accessed at `<vmanomaly-host>:8490/vmanomaly/vmui/` (e.g. `http://localhost:8490/vmanomaly/vmui/`).
|
||||
|
||||
Rest API endpoints (e.g. `/metrics`) can be accessed at `<vmanomaly-host>:8490/vmanomaly/metrics` (e.g. `http://localhost:8490/vmanomaly/metrics`).
|
||||
|
||||
### Time-series analysis and autotune API
|
||||
|
||||
{{% available_from "v1.30.0" anomaly %}} The server exposes bounded endpoints for UI, MCP, and automation workflows:
|
||||
|
||||
- `GET /api/v1/timeseries/characteristics` samples the supplied query and summarizes trend, calendar seasonality, changepoints, gaps, and intermittent or spiky behavior. Use `limit` (default 100) to cap sampled series and pass the production `step` and timezone.
|
||||
- `POST /api/v1/autotune/tasks` starts asynchronous shared-model tuning. The request contains the query, candidate `tuned_class_name`, expected `anomaly_percentage`, data-source settings, and optimization parameters.
|
||||
- `GET /api/v1/autotune/tasks/{task_id}` returns progress and the concrete suggested `modelConfig` when complete.
|
||||
- `DELETE /api/v1/autotune/tasks/{task_id}` cancels pending work cooperatively.
|
||||
|
||||
> [!TIP]
|
||||
> For a complete request and recommended workflow, see [Shared asynchronous autotune workflow](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow). OpenAPI schemas for the running version are available at `/docs` endpoint of a running `vmanomaly` instance.
|
||||
|
||||
@@ -307,7 +307,7 @@ This means that the service upon restart:
|
||||
|
||||
## Retention
|
||||
|
||||
{{% available_from "v1.28.1" anomaly %}} The `retention` argument allows to set a [time-to-live](https://en.wikipedia.org/wiki/Time_to_live) (TTL) for service artifacts, such as stored model instances and their training data. When enabled, the service will periodically check (controlled by `check_interval` period) and clean up model instances that have not been used for inference or refitting within the specified period of time (defined in `ttl` argument as a valid period). This helps to manage resources in long-running deployments by removing stale or unused artifacts.
|
||||
{{% available_from "v1.28.1" anomaly %}} The `retention` argument sets a [time to live](https://en.wikipedia.org/wiki/Time_to_live) (TTL) for service artifacts such as stored model instances and training data. At each `check_interval`, the service removes artifacts that have not been used for inference or refitting within `ttl`. This bounds stale resource usage in long-running deployments.
|
||||
|
||||
### Use Cases
|
||||
- With **[online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models)** as they continuously create model instances for new timeseries over time during inference calls, especially when combined with [periodic schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) with infrequent `fit_every` (say, `90d`).
|
||||
@@ -322,7 +322,7 @@ The section is **backward-compatible and disabled by default**, meaning that all
|
||||
|
||||
`ttl` argument defines the time-to-live period for model instances and their training data. It should be a valid period string (e.g., `7d` for 7 days, `30d` for 30 days, etc.). If a model instance or its training data has not been used for inference or refitting within this period, it will be considered stale and eligible for cleanup.
|
||||
|
||||
> If set higher than respective scheduler's `fit_every` period, the ttl will have no effect, as models will always be refitted before they become stale.
|
||||
> If `ttl` is greater than a scheduler's `fit_every`, the model is refitted before it becomes stale and the TTL has no effect.
|
||||
|
||||
`check_interval` argument defines how often the service should check for stale artifacts. It should be a valid period string (e.g., `1h` for 1 hour, `24h` for 24 hours, etc.). During each check, the service will evaluate all stored model instances and their training data against the defined `ttl` and remove those that are stale.
|
||||
|
||||
@@ -401,4 +401,4 @@ settings:
|
||||
model: WARNING # applies to all components with 'model' prefix, such as 'model.zscore_online', 'model.prophet', etc.
|
||||
# once commented out in hot-reload mode, will use the default logger level set by --loggerLevel command line argument
|
||||
# monitoring.push: critical
|
||||
```
|
||||
```
|
||||
|
||||
@@ -334,7 +334,7 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
|
||||
|
||||
### Healthcheck metrics
|
||||
|
||||
`VmWriter` exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics).
|
||||
`VmWriter` exposes [several health metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics).
|
||||
|
||||
### Metrics formatting
|
||||
|
||||
|
||||
@@ -395,7 +395,7 @@ services:
|
||||
restart: always
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.29.7
|
||||
image: victoriametrics/vmanomaly:v1.30.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
|
Before Width: | Height: | Size: 189 KiB After Width: | Height: | Size: 357 KiB |
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 17 KiB |
|
Before Width: | Height: | Size: 206 KiB After Width: | Height: | Size: 82 KiB |
|
Before Width: | Height: | Size: 428 KiB After Width: | Height: | Size: 85 KiB |
|
Before Width: | Height: | Size: 37 KiB After Width: | Height: | Size: 11 KiB |
|
Before Width: | Height: | Size: 740 KiB After Width: | Height: | Size: 82 KiB |
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 17 KiB |
|
Before Width: | Height: | Size: 225 KiB After Width: | Height: | Size: 58 KiB |
|
Before Width: | Height: | Size: 189 KiB After Width: | Height: | Size: 280 KiB |
|
Before Width: | Height: | Size: 26 KiB After Width: | Height: | Size: 17 KiB |
|
Before Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 160 KiB After Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 93 KiB After Width: | Height: | Size: 40 KiB |
|
Before Width: | Height: | Size: 425 KiB After Width: | Height: | Size: 235 KiB |
|
Before Width: | Height: | Size: 514 KiB After Width: | Height: | Size: 122 KiB |
|
Before Width: | Height: | Size: 193 KiB After Width: | Height: | Size: 44 KiB |