mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-05-30 07:10:55 +03:00
Compare commits
66 Commits
v1.102.0-c
...
storage-no
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1272a7f743 | ||
|
|
2b39ee785c | ||
|
|
842bf78cb1 | ||
|
|
5420989018 | ||
|
|
9ff8b312bb | ||
|
|
130b9cd04e | ||
|
|
88bfad9535 | ||
|
|
e44c6f38c2 | ||
|
|
96a62a275a | ||
|
|
20b9c8007b | ||
|
|
3df456dd35 | ||
|
|
7402ee0801 | ||
|
|
5ac1e77520 | ||
|
|
71e729f3f8 | ||
|
|
8729ec174b | ||
|
|
84184b707a | ||
|
|
41e217423f | ||
|
|
8d8073a24d | ||
|
|
00b1ee6b5c | ||
|
|
9a3f44e79c | ||
|
|
5dc7ec058f | ||
|
|
d2a825279b | ||
|
|
75411db0c7 | ||
|
|
eb2a33b6b3 | ||
|
|
f320efd5ad | ||
|
|
a135a4dcfa | ||
|
|
83d53dfb0d | ||
|
|
edc16e171e | ||
|
|
e4195f1b47 | ||
|
|
d7ce94c20d | ||
|
|
f17265b67d | ||
|
|
1951555ffe | ||
|
|
8e2ff15203 | ||
|
|
d25f88b912 | ||
|
|
3f85306738 | ||
|
|
571ecbb610 | ||
|
|
61f9e3112d | ||
|
|
85d624d6c2 | ||
|
|
b8d80ddae5 | ||
|
|
9a051fc80f | ||
|
|
6b97044d8a | ||
|
|
4a01709b90 | ||
|
|
e0c62e5c50 | ||
|
|
9f5eb25150 | ||
|
|
bafdc9dd78 | ||
|
|
53c08cb8d7 | ||
|
|
591c42fdbe | ||
|
|
b65f32350e | ||
|
|
339821f5ce | ||
|
|
b31bd5613f | ||
|
|
8f3fd62f50 | ||
|
|
37fdba6897 | ||
|
|
28af963940 | ||
|
|
a50a29500f | ||
|
|
4e3acfbe9a | ||
|
|
3dd239fb87 | ||
|
|
5436911abb | ||
|
|
be2a61c244 | ||
|
|
7eb24d98e1 | ||
|
|
12393c64f7 | ||
|
|
13b9a2cfc1 | ||
|
|
3cda945138 | ||
|
|
7e0fff224e | ||
|
|
47f62f2e75 | ||
|
|
009f71fc4c | ||
|
|
c57c169dc4 |
5
Makefile
5
Makefile
@@ -270,8 +270,9 @@ copy-docs:
|
||||
fi
|
||||
echo "---" >> ${DST}
|
||||
cat ${SRC} >> ${DST}
|
||||
sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
|
||||
sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"/' ${DST}
|
||||
sed -i='.tmp' 's/<img src=\"docs\//<img src=\"\//' ${DST}
|
||||
sed -i='.tmp' 's/<source srcset=\"docs\//<source srcset=\"\//' ${DST}
|
||||
sed -i='.tmp' 's/](docs\//](/' ${DST}
|
||||
rm -rf docs/*.tmp
|
||||
|
||||
# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
# Cluster version
|
||||
|
||||
<picture>
|
||||
<source srcset="docs/logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
<source srcset="docs/logo.webp" media="(prefers-color-scheme: light)">
|
||||
@@ -50,7 +48,7 @@ Each service may scale independently and may run on the most suitable hardware.
|
||||
This is a [shared nothing architecture](https://en.wikipedia.org/wiki/Shared-nothing_architecture).
|
||||
It increases cluster availability, and simplifies cluster maintenance as well as cluster scaling.
|
||||
|
||||
<img src="docs/Cluster-VictoriaMetrics_cluster-scheme.webp">
|
||||

|
||||
|
||||
## Multitenancy
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"files": {
|
||||
"main.css": "./static/css/main.1041c3d4.css",
|
||||
"main.js": "./static/js/main.62a82f4a.js",
|
||||
"main.js": "./static/js/main.8451d9b9.js",
|
||||
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
|
||||
"index.html": "./index.html"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.1041c3d4.css",
|
||||
"static/js/main.62a82f4a.js"
|
||||
"static/js/main.8451d9b9.js"
|
||||
]
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.62a82f4a.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8451d9b9.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
@@ -10,9 +10,6 @@ aliases:
|
||||
- /ExtendedPromQL.html
|
||||
- /MetricsQL.html
|
||||
---
|
||||
|
||||
# MetricsQL
|
||||
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
|
||||
@@ -220,9 +217,11 @@ See also [descent_over_time](#descent_over_time).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [median_over_time](#median_over_time).
|
||||
See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
|
||||
|
||||
#### changes
|
||||
|
||||
@@ -262,6 +261,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
|
||||
|
||||
#### count_gt_over_time
|
||||
@@ -272,6 +273,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
|
||||
|
||||
#### count_le_over_time
|
||||
@@ -282,6 +285,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
|
||||
|
||||
#### count_ne_over_time
|
||||
@@ -292,7 +297,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
See also [count_over_time](#count_over_time).
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### count_over_time
|
||||
|
||||
@@ -313,6 +320,8 @@ The results are calculated independently per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
|
||||
|
||||
#### decreases_over_time
|
||||
@@ -432,6 +441,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### histogram_over_time
|
||||
|
||||
`histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
@@ -443,11 +454,15 @@ For example, the following query calculates median temperature by country over t
|
||||
|
||||
`histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### hoeffding_bound_lower
|
||||
|
||||
`hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
|
||||
#### hoeffding_bound_upper
|
||||
@@ -455,6 +470,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
`hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
|
||||
#### holt_winters
|
||||
@@ -462,8 +479,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
|
||||
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
|
||||
Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
Both `sf` and `tf` must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
@@ -495,13 +513,14 @@ See also [deriv](#deriv).
|
||||
|
||||
`increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
|
||||
@@ -510,12 +529,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
|
||||
|
||||
`increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
|
||||
#### increase_pure
|
||||
@@ -524,6 +544,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
|
||||
while [increase](#increase) ignores the first value in a series if it is too big.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
|
||||
|
||||
#### increases_over_time
|
||||
|
||||
`increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -540,15 +564,18 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### irate
|
||||
|
||||
`irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
|
||||
the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [rate](#rate) and [rollup_rate](#rollup_rate).
|
||||
@@ -587,6 +614,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
#### max_over_time
|
||||
@@ -594,9 +623,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
|
||||
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmax_over_time](#tmax_over_time).
|
||||
See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
|
||||
|
||||
#### median_over_time
|
||||
|
||||
@@ -604,6 +635,8 @@ See also [tmax_over_time](#tmax_over_time).
|
||||
on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [avg_over_time](#avg_over_time).
|
||||
|
||||
#### min_over_time
|
||||
@@ -611,9 +644,11 @@ See also [avg_over_time](#avg_over_time).
|
||||
`min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmin_over_time](#tmin_over_time).
|
||||
See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
|
||||
|
||||
#### mode_over_time
|
||||
|
||||
@@ -622,6 +657,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
values are discrete.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### outlier_iqr_over_time
|
||||
|
||||
`outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
|
||||
@@ -632,6 +669,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
|
||||
The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
|
||||
For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [outliers_iqr](#outliers_iqr).
|
||||
|
||||
#### predict_linear
|
||||
@@ -659,6 +698,8 @@ This function is supported by PromQL.
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The `phi` value must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [quantiles_over_time](#quantiles_over_time).
|
||||
@@ -670,6 +711,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [quantile_over_time](#quantile_over_time).
|
||||
|
||||
#### range_over_time
|
||||
@@ -680,11 +723,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rate
|
||||
|
||||
`rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
|
||||
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
|
||||
@@ -703,6 +747,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
on the given lookbehind window `d`. The calculations are performed individually per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
#### resets
|
||||
@@ -710,10 +756,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
`resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
|
||||
of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### rollup
|
||||
@@ -725,6 +772,10 @@ These values are calculated individually per each time series returned from the
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_candlestick
|
||||
|
||||
`rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
|
||||
@@ -736,6 +787,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rollup_delta
|
||||
|
||||
`rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -762,6 +815,8 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_increase
|
||||
|
||||
`rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -774,12 +829,17 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_rate
|
||||
|
||||
`rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
|
||||
for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
|
||||
and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
|
||||
when to use `rollup_rate()`.
|
||||
@@ -787,10 +847,12 @@ when to use `rollup_rate()`.
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
|
||||
|
||||
#### rollup_scrape_interval
|
||||
|
||||
`rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
|
||||
@@ -824,6 +886,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### share_le_over_time
|
||||
@@ -838,6 +902,8 @@ the share of time series values for the last 24 hours when memory usage was belo
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### share_eq_over_time
|
||||
@@ -849,6 +915,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### stale_samples_over_time
|
||||
@@ -866,6 +934,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stdvar_over_time](#stdvar_over_time).
|
||||
@@ -877,6 +947,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stddev_over_time](#stddev_over_time).
|
||||
@@ -888,6 +960,8 @@ values equal to `eq` on the given lookbehind window `d` per each time series ret
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### sum_gt_over_time
|
||||
@@ -897,6 +971,8 @@ values bigger than `gt` on the given lookbehind window `d` per each time series
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### sum_le_over_time
|
||||
@@ -906,6 +982,8 @@ values smaller or equal to `le` on the given lookbehind window `d` per each time
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### sum_over_time
|
||||
@@ -915,6 +993,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### sum2_over_time
|
||||
@@ -924,6 +1004,8 @@ values on the given lookbehind window `d` per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### timestamp
|
||||
|
||||
`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
|
||||
@@ -1001,6 +1083,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
|
||||
@@ -2230,7 +2314,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
|
||||
Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
|
||||
For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
|
||||
since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
|
||||
VictoriaMetrics performs subqueries in the following way:
|
||||
|
||||
@@ -2263,5 +2348,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
|
||||
For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
|
||||
* If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
@@ -84,9 +84,6 @@ type UserInfo struct {
|
||||
concurrencyLimitCh chan struct{}
|
||||
concurrencyLimitReached *metrics.Counter
|
||||
|
||||
// Whether to use backend host header in requests to backend.
|
||||
useBackendHostHeader bool
|
||||
|
||||
rt http.RoundTripper
|
||||
|
||||
requests *metrics.Counter
|
||||
@@ -96,8 +93,9 @@ type UserInfo struct {
|
||||
|
||||
// HeadersConf represents config for request and response headers.
|
||||
type HeadersConf struct {
|
||||
RequestHeaders []*Header `yaml:"headers,omitempty"`
|
||||
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
|
||||
RequestHeaders []*Header `yaml:"headers,omitempty"`
|
||||
ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
|
||||
KeepOriginalHost *bool `yaml:"keep_original_host,omitempty"`
|
||||
}
|
||||
|
||||
func (ui *UserInfo) beginConcurrencyLimit() error {
|
||||
@@ -152,15 +150,6 @@ func (h *Header) MarshalYAML() (any, error) {
|
||||
return h.sOriginal, nil
|
||||
}
|
||||
|
||||
func hasEmptyHostHeader(headers []*Header) bool {
|
||||
for _, h := range headers {
|
||||
if h.Name == "Host" && h.Value == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// URLMap is a mapping from source paths to target urls.
|
||||
type URLMap struct {
|
||||
// SrcPaths is an optional list of regular expressions, which must match the request path.
|
||||
@@ -472,7 +461,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
|
||||
}
|
||||
|
||||
// Slow path - select other backend urls.
|
||||
n := atomicCounter.Add(1)
|
||||
n := atomicCounter.Add(1) - 1
|
||||
|
||||
for i := uint32(0); i < uint32(len(bus)); i++ {
|
||||
idx := (n + i) % uint32(len(bus))
|
||||
@@ -608,7 +597,7 @@ func initAuthConfig() {
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
|
||||
sighupCh := procutil.NewSighupChan()
|
||||
|
||||
_, err := loadAuthConfig()
|
||||
_, err := reloadAuthConfig()
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot load auth config: %s", err)
|
||||
}
|
||||
@@ -640,7 +629,7 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
|
||||
|
||||
updateFn := func() {
|
||||
configReloads.Inc()
|
||||
updated, err := loadAuthConfig()
|
||||
updated, err := reloadAuthConfig()
|
||||
if err != nil {
|
||||
logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
|
||||
configSuccess.Set(0)
|
||||
@@ -666,27 +655,45 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
|
||||
}
|
||||
}
|
||||
|
||||
// authConfigData stores the yaml definition for this config.
|
||||
// authConfigData needs to be updated each time authConfig is updated.
|
||||
var authConfigData atomic.Pointer[[]byte]
|
||||
|
||||
var (
|
||||
authConfig atomic.Pointer[AuthConfig]
|
||||
authUsers atomic.Pointer[map[string]*UserInfo]
|
||||
// authConfigData stores the yaml definition for this config.
|
||||
// authConfigData needs to be updated each time authConfig is updated.
|
||||
authConfigData atomic.Pointer[[]byte]
|
||||
|
||||
// authConfig contains the currently loaded auth config
|
||||
authConfig atomic.Pointer[AuthConfig]
|
||||
|
||||
// authUsers contains the currently loaded auth users
|
||||
authUsers atomic.Pointer[map[string]*UserInfo]
|
||||
|
||||
authConfigWG sync.WaitGroup
|
||||
stopCh chan struct{}
|
||||
)
|
||||
|
||||
// loadAuthConfig loads and applies the config from *authConfigPath.
|
||||
// reloadAuthConfig loads and applies the config from *authConfigPath.
|
||||
// It returns bool value to identify if new config was applied.
|
||||
// The config can be not applied if there is a parsing error
|
||||
// or if there are no changes to the current authConfig.
|
||||
func loadAuthConfig() (bool, error) {
|
||||
func reloadAuthConfig() (bool, error) {
|
||||
data, err := fscore.ReadFileOrHTTP(*authConfigPath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
|
||||
}
|
||||
|
||||
ok, err := reloadAuthConfigData(data)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
|
||||
}
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
mp := authUsers.Load()
|
||||
logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func reloadAuthConfigData(data []byte) (bool, error) {
|
||||
oldData := authConfigData.Load()
|
||||
if oldData != nil && bytes.Equal(data, *oldData) {
|
||||
// there are no updates in the config - skip reloading.
|
||||
@@ -695,14 +702,13 @@ func loadAuthConfig() (bool, error) {
|
||||
|
||||
ac, err := parseAuthConfig(data)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
|
||||
return false, fmt.Errorf("failed to parse auth config: %w", err)
|
||||
}
|
||||
|
||||
m, err := parseAuthConfigUsers(ac)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to parse users from -auth.config=%q: %w", *authConfigPath, err)
|
||||
return false, fmt.Errorf("failed to parse users from auth config: %w", err)
|
||||
}
|
||||
logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
|
||||
|
||||
acPrev := authConfig.Load()
|
||||
if acPrev != nil {
|
||||
@@ -749,7 +755,6 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ui.useBackendHostHeader = hasEmptyHostHeader(ui.HeadersConf.RequestHeaders)
|
||||
|
||||
metricLabels, err := ui.getMetricLabels()
|
||||
if err != nil {
|
||||
@@ -814,7 +819,6 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
|
||||
_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
|
||||
return float64(len(ui.concurrencyLimitCh))
|
||||
})
|
||||
ui.useBackendHostHeader = hasEmptyHostHeader(ui.HeadersConf.RequestHeaders)
|
||||
|
||||
rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
|
||||
if err != nil {
|
||||
@@ -1005,6 +1009,14 @@ func getAuthTokensFromRequest(r *http.Request) []string {
|
||||
}
|
||||
}
|
||||
|
||||
// Authorization via http://user:pass@hosname/path
|
||||
if u := r.URL.User; u != nil && u.Username() != "" {
|
||||
username := u.Username()
|
||||
password, _ := u.Password()
|
||||
at := getHTTPAuthBasicToken(username, password)
|
||||
ats = append(ats, at)
|
||||
}
|
||||
|
||||
return ats
|
||||
}
|
||||
|
||||
@@ -1030,10 +1042,6 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
|
||||
}
|
||||
|
||||
func sanitizeURLPrefix(urlPrefix *url.URL) (*url.URL, error) {
|
||||
// Remove trailing '/' from urlPrefix
|
||||
for strings.HasSuffix(urlPrefix.Path, "/") {
|
||||
urlPrefix.Path = urlPrefix.Path[:len(urlPrefix.Path)-1]
|
||||
}
|
||||
// Validate urlPrefix
|
||||
if urlPrefix.Scheme != "http" && urlPrefix.Scheme != "https" {
|
||||
return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, urlPrefix.Scheme)
|
||||
|
||||
@@ -82,6 +82,14 @@ users:
|
||||
headers: foobar
|
||||
`)
|
||||
|
||||
// Invalid keep_original_host value
|
||||
f(`
|
||||
users:
|
||||
- username: foo
|
||||
url_prefix: http://foo.bar
|
||||
keep_original_host: foobar
|
||||
`)
|
||||
|
||||
// empty url_prefix
|
||||
f(`
|
||||
users:
|
||||
@@ -410,7 +418,7 @@ users:
|
||||
HeadersConf: HeadersConf{
|
||||
RequestHeaders: []*Header{
|
||||
mustNewHeader("'foo: bar'"),
|
||||
mustNewHeader("'xxx: y'"),
|
||||
mustNewHeader("'xxx:'"),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -429,7 +437,7 @@ users:
|
||||
url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
|
||||
headers:
|
||||
- "foo: bar"
|
||||
- "xxx: y"
|
||||
- "xxx:"
|
||||
`, map[string]*UserInfo{
|
||||
getHTTPAuthBearerToken("foo"): sharedUserInfo,
|
||||
getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
|
||||
@@ -458,6 +466,7 @@ users:
|
||||
})
|
||||
|
||||
// with default url
|
||||
keepOriginalHost := true
|
||||
f(`
|
||||
users:
|
||||
- bearer_token: foo
|
||||
@@ -469,6 +478,7 @@ users:
|
||||
headers:
|
||||
- "foo: bar"
|
||||
- "xxx: y"
|
||||
keep_original_host: true
|
||||
default_url:
|
||||
- http://default1/select/0/prometheus
|
||||
- http://default2/select/0/prometheus
|
||||
@@ -491,6 +501,7 @@ users:
|
||||
mustNewHeader("'foo: bar'"),
|
||||
mustNewHeader("'xxx: y'"),
|
||||
},
|
||||
KeepOriginalHost: &keepOriginalHost,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -517,6 +528,7 @@ users:
|
||||
mustNewHeader("'foo: bar'"),
|
||||
mustNewHeader("'xxx: y'"),
|
||||
},
|
||||
KeepOriginalHost: &keepOriginalHost,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -536,12 +548,17 @@ users:
|
||||
metric_labels:
|
||||
dc: eu
|
||||
team: dev
|
||||
keep_original_host: true
|
||||
- username: foo-same
|
||||
password: bar
|
||||
url_prefix: https://bar/x
|
||||
metric_labels:
|
||||
backend_env: test
|
||||
team: accounting
|
||||
headers:
|
||||
- "foo: bar"
|
||||
response_headers:
|
||||
- "Abc: def"
|
||||
`, map[string]*UserInfo{
|
||||
getHTTPAuthBasicToken("foo-same", "baz"): {
|
||||
Username: "foo-same",
|
||||
@@ -551,6 +568,9 @@ users:
|
||||
"dc": "eu",
|
||||
"team": "dev",
|
||||
},
|
||||
HeadersConf: HeadersConf{
|
||||
KeepOriginalHost: &keepOriginalHost,
|
||||
},
|
||||
},
|
||||
getHTTPAuthBasicToken("foo-same", "bar"): {
|
||||
Username: "foo-same",
|
||||
@@ -560,6 +580,14 @@ users:
|
||||
"backend_env": "test",
|
||||
"team": "accounting",
|
||||
},
|
||||
HeadersConf: HeadersConf{
|
||||
RequestHeaders: []*Header{
|
||||
mustNewHeader("'foo: bar'"),
|
||||
},
|
||||
ResponseHeaders: []*Header{
|
||||
mustNewHeader("'Abc: def'"),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -695,9 +723,9 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
|
||||
|
||||
fn := func(ns ...int) {
|
||||
t.Helper()
|
||||
bus := up.bus.Load()
|
||||
pbus := *bus
|
||||
for i, b := range pbus {
|
||||
pbus := up.bus.Load()
|
||||
bus := *pbus
|
||||
for i, b := range bus {
|
||||
got := int(b.concurrentRequests.Load())
|
||||
exp := ns[i]
|
||||
if got != exp {
|
||||
@@ -707,39 +735,39 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
|
||||
}
|
||||
|
||||
up.getBackendURL()
|
||||
fn(0, 1, 0)
|
||||
fn(1, 0, 0)
|
||||
up.getBackendURL()
|
||||
fn(0, 1, 1)
|
||||
fn(1, 1, 0)
|
||||
up.getBackendURL()
|
||||
fn(1, 1, 1)
|
||||
|
||||
up.getBackendURL()
|
||||
up.getBackendURL()
|
||||
fn(1, 2, 2)
|
||||
fn(2, 2, 1)
|
||||
|
||||
bus := up.bus.Load()
|
||||
pbus := *bus
|
||||
pbus[0].concurrentRequests.Add(2)
|
||||
pbus[2].concurrentRequests.Add(5)
|
||||
fn(3, 2, 7)
|
||||
fn(4, 2, 6)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(3, 3, 7)
|
||||
fn(4, 3, 6)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(3, 4, 7)
|
||||
fn(4, 4, 6)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(4, 4, 7)
|
||||
fn(4, 5, 6)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(5, 4, 7)
|
||||
fn(5, 5, 6)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(5, 5, 7)
|
||||
fn(6, 5, 6)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(6, 5, 7)
|
||||
fn(6, 6, 6)
|
||||
|
||||
up.getBackendURL()
|
||||
fn(6, 6, 7)
|
||||
|
||||
@@ -117,8 +117,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
|
||||
http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
|
||||
handleMissingAuthorizationError(w)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -189,12 +188,11 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
// to a route that is not in the configuration for unauthorized user.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5236
|
||||
if ui.BearerToken == "" && ui.Username == "" && len(*authUsers.Load()) > 0 {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
|
||||
http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
|
||||
handleMissingAuthorizationError(w)
|
||||
return
|
||||
}
|
||||
missingRouteRequests.Inc()
|
||||
httpserver.Errorf(w, r, "missing route for %q", u.String())
|
||||
httpserver.Errorf(w, r, "missing route for %s", u.String())
|
||||
return
|
||||
}
|
||||
up, hc = ui.DefaultURL, ui.HeadersConf
|
||||
@@ -220,7 +218,15 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
} else { // Update path for regular routes.
|
||||
targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
|
||||
}
|
||||
ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
|
||||
|
||||
wasLocalRetry := false
|
||||
again:
|
||||
ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
|
||||
if needLocalRetry && !wasLocalRetry {
|
||||
wasLocalRetry = true
|
||||
goto again
|
||||
}
|
||||
|
||||
bu.put()
|
||||
if ok {
|
||||
return
|
||||
@@ -228,29 +234,28 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
|
||||
bu.setBroken()
|
||||
}
|
||||
err := &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Err: fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
|
||||
StatusCode: http.StatusBadGateway,
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
ui.backendErrors.Inc()
|
||||
}
|
||||
|
||||
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
|
||||
// This code has been copied from net/http/httputil/reverseproxy.go
|
||||
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
|
||||
req := sanitizeRequestHeaders(r)
|
||||
req.URL = targetURL
|
||||
|
||||
if req.URL.Scheme == "https" || ui.useBackendHostHeader {
|
||||
// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
|
||||
// so it expects the targetURL.Host in the request.
|
||||
// There is no need in overriding the req.Host for http requests, since it is expected that backend server
|
||||
// may properly process queries with the original req.Host.
|
||||
req.Host = targetURL.Host
|
||||
}
|
||||
req.URL = targetURL
|
||||
req.Header.Set("User-Agent", "vmauth")
|
||||
updateHeadersByConfig(req.Header, hc.RequestHeaders)
|
||||
var trivialRetries int
|
||||
if hc.KeepOriginalHost == nil || !*hc.KeepOriginalHost {
|
||||
if host := getHostHeader(hc.RequestHeaders); host != "" {
|
||||
req.Host = host
|
||||
} else {
|
||||
req.Host = targetURL.Host
|
||||
}
|
||||
}
|
||||
|
||||
rtb, rtbOK := req.Body.(*readTrackingBody)
|
||||
again:
|
||||
res, err := ui.rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
@@ -262,7 +267,7 @@ again:
|
||||
// Timed out request must be counted as errors, since this usually means that the backend is slow.
|
||||
ui.backendErrors.Inc()
|
||||
}
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
if !rtbOK || !rtb.canRetry() {
|
||||
// Request body cannot be re-sent to another backend. Return the error to the client then.
|
||||
@@ -272,19 +277,19 @@ again:
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
ui.backendErrors.Inc()
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
// Retry request on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
|
||||
if (netutil.IsTrivialNetworkError(err) || errors.Is(err, io.EOF)) && trivialRetries < 1 {
|
||||
trivialRetries++
|
||||
goto again
|
||||
if netutil.IsTrivialNetworkError(err) {
|
||||
// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
|
||||
return false, true
|
||||
}
|
||||
|
||||
// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
// NOTE: do not use httpserver.GetRequestURI
|
||||
// it explicitly reads request body, which may fail retries.
|
||||
logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
if slices.Contains(retryStatusCodes, res.StatusCode) {
|
||||
_ = res.Body.Close()
|
||||
@@ -298,7 +303,7 @@ again:
|
||||
}
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
ui.backendErrors.Inc()
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
// Retry requests at other backends if it matches retryStatusCodes.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
|
||||
@@ -307,7 +312,7 @@ again:
|
||||
// it explicitly reads request body, which may fail retries.
|
||||
logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
|
||||
remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
removeHopHeaders(res.Header)
|
||||
copyHeader(w.Header(), res.Header)
|
||||
@@ -323,9 +328,9 @@ again:
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
requestURI := httpserver.GetRequestURI(r)
|
||||
logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
var copyBufPool bytesutil.ByteBufferPool
|
||||
@@ -338,12 +343,21 @@ func copyHeader(dst, src http.Header) {
|
||||
}
|
||||
}
|
||||
|
||||
func updateHeadersByConfig(headers http.Header, config []*Header) {
|
||||
for _, h := range config {
|
||||
func getHostHeader(headers []*Header) string {
|
||||
for _, h := range headers {
|
||||
if h.Name == "Host" {
|
||||
return h.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func updateHeadersByConfig(dst http.Header, src []*Header) {
|
||||
for _, h := range src {
|
||||
if h.Value == "" {
|
||||
headers.Del(h.Name)
|
||||
dst.Del(h.Name)
|
||||
} else {
|
||||
headers.Set(h.Name, h.Value)
|
||||
dst.Set(h.Name, h.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -483,6 +497,11 @@ See the docs at https://docs.victoriametrics.com/vmauth/ .
|
||||
flagutil.Usage(s)
|
||||
}
|
||||
|
||||
func handleMissingAuthorizationError(w http.ResponseWriter) {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
|
||||
http.Error(w, "missing 'Authorization' request header", http.StatusUnauthorized)
|
||||
}
|
||||
|
||||
func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
w.Header().Add("Retry-After", "10")
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
@@ -537,10 +556,24 @@ func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
|
||||
maxBodySize = 0
|
||||
}
|
||||
rtb.maxBodySize = maxBodySize
|
||||
|
||||
if r == nil {
|
||||
// This is GET request without request body
|
||||
r = (*zeroReader)(nil)
|
||||
}
|
||||
rtb.r = r
|
||||
return rtb
|
||||
}
|
||||
|
||||
type zeroReader struct{}
|
||||
|
||||
func (r *zeroReader) Read(_ []byte) (int, error) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
func (r *zeroReader) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func putReadTrackingBody(rtb *readTrackingBody) {
|
||||
rtb.reset()
|
||||
readTrackingBodyPool.Put(rtb)
|
||||
@@ -560,7 +593,7 @@ func (rtb *readTrackingBody) Read(p []byte) (int, error) {
|
||||
if rtb.bufComplete {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return 0, fmt.Errorf("cannot read data after closing the reader")
|
||||
return 0, fmt.Errorf("cannot read client request body after closing client reader")
|
||||
}
|
||||
|
||||
n, err := rtb.r.Read(p)
|
||||
|
||||
@@ -2,10 +2,511 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
)
|
||||
|
||||
func TestRequestHandler(t *testing.T) {
|
||||
f := func(cfgStr, requestURL string, backendHandler http.HandlerFunc, responseExpected string) {
|
||||
t.Helper()
|
||||
|
||||
ts := httptest.NewServer(backendHandler)
|
||||
defer ts.Close()
|
||||
|
||||
cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
|
||||
responseExpected = strings.ReplaceAll(responseExpected, "{BACKEND}", ts.URL)
|
||||
|
||||
cfgOrigP := authConfigData.Load()
|
||||
if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
|
||||
t.Fatalf("cannot load config data: %s", err)
|
||||
}
|
||||
defer func() {
|
||||
cfgOrig := []byte("unauthorized_user:\n url_prefix: http://foo/bar")
|
||||
if cfgOrigP != nil {
|
||||
cfgOrig = *cfgOrigP
|
||||
}
|
||||
_, err := reloadAuthConfigData(cfgOrig)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot load the original config: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
r, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot initialize http request: %s", err)
|
||||
}
|
||||
|
||||
r.RequestURI = r.URL.RequestURI()
|
||||
r.RemoteAddr = "42.2.3.84:6789"
|
||||
r.Header.Set("X-Forwarded-For", "12.34.56.78")
|
||||
r.Header.Set("Connection", "Some-Header,Other-Header")
|
||||
r.Header.Set("Some-Header", "foobar")
|
||||
r.Header.Set("Pass-Header", "abc")
|
||||
|
||||
w := &fakeResponseWriter{}
|
||||
if !requestHandler(w, r) {
|
||||
t.Fatalf("unexpected false is returned from requestHandler")
|
||||
}
|
||||
|
||||
response := w.getResponse()
|
||||
response = strings.ReplaceAll(response, "\r\n", "\n")
|
||||
response = strings.TrimSpace(response)
|
||||
responseExpected = strings.TrimSpace(responseExpected)
|
||||
if response != responseExpected {
|
||||
t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// regular url_prefix
|
||||
cfgStr := `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo?bar=baz`
|
||||
requestURL := "http://some-host.com/abc/def?some_arg=some_value"
|
||||
backendHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("Connection", "close")
|
||||
h.Set("Foo", "bar")
|
||||
|
||||
var bb bytes.Buffer
|
||||
if err := r.Header.Write(&bb); err != nil {
|
||||
panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
|
||||
}
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
|
||||
}
|
||||
responseExpected := `
|
||||
statusCode=200
|
||||
Foo: bar
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz&some_arg=some_value
|
||||
Pass-Header: abc
|
||||
User-Agent: vmauth
|
||||
X-Forwarded-For: 12.34.56.78, 42.2.3.84`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// keep_original_host
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
keep_original_host: true`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url=http://some-host.com/foo/abc/def?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// override user-agent header
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "User-Agent: foobar"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz
|
||||
User-Agent=foobar`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// delete user-agent header
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "User-Agent:"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz
|
||||
User-Agent=Go-http-client/1.1`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// override request host with non-empty host
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "Host: other-host:12345"
|
||||
- "abc:"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url=http://other-host:12345/foo/abc/def?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// override request host with empty host
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo?bar=baz"
|
||||
headers:
|
||||
- "Host:"`
|
||||
requestURL = "http://some-host.com/abc/def"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/abc/def?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// /-/reload handler failure
|
||||
origAuthKey := reloadAuthKey.Get()
|
||||
if err := reloadAuthKey.Set("secret"); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: "{BACKEND}/foo"`
|
||||
requestURL = "http://some-host.com/-/reload"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
The provided authKey doesn't match -reloadAuthKey`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
if err := reloadAuthKey.Set(origAuthKey); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
// missing authorization
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://some-host.com/a/b"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Www-Authenticate: Basic realm="Restricted"
|
||||
missing 'Authorization' request header`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// incorrect authorization
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
password: secret
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://foo:invalid-secret@some-host.com/a/b"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Unauthorized`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// incorrect authorization with logging invalid auth tokens
|
||||
origLogInvalidAuthTokens := *logInvalidAuthTokens
|
||||
*logInvalidAuthTokens = true
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
password: secret
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://foo:invalid-secret@some-host.com/a/b?c=d"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /a/b?c=d; cannot authorize request with auth tokens ["http_auth:Basic Zm9vOmludmFsaWQtc2VjcmV0"]`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
*logInvalidAuthTokens = origLogInvalidAuthTokens
|
||||
|
||||
// correct authorization
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: foo
|
||||
password: secret
|
||||
url_prefix: "{BACKEND}/bar"`
|
||||
requestURL = "http://foo:secret@some-host.com/a/b"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/bar/a/b`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo?bar=baz`
|
||||
requestURL = "http://some-host.com/../../a//.///bar/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/a/bar/?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works for url without path
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo?bar=baz`
|
||||
requestURL = "http://some-host.com/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works for url without path if url_prefix path ends with /
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/foo/?bar=baz`
|
||||
requestURL = "http://some-host.com/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/foo/?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify how path cleanup works for url without path and the url_prefix without path prefix
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/?bar=baz`
|
||||
requestURL = "http://some-host.com/"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/?bar=baz`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify routing to default_url
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/
|
||||
default_url: {BACKEND}/404.html`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/404.html?request_path=http%3A%2F%2Fsome-host.com%2Fabc%3Fde%3Dfg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// verify routing to default url_prefix
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/
|
||||
url_prefix: {BACKEND}/default`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/default/abc?de=fg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// missing default_url and default url_prefix for unauthorized user
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=400
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for http://some-host.com/abc?de=fg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// missing default_url and default url_prefix for unauthorized user when there are configs for authorized users
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: some-user
|
||||
url_map:
|
||||
- src_paths: ["/foo/.+"]
|
||||
url_prefix: {BACKEND}/x-foo/
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/abc/.*"]
|
||||
url_prefix: {BACKEND}/x-bar`
|
||||
requestURL = "http://some-host.com/abc?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=401
|
||||
Www-Authenticate: Basic realm="Restricted"
|
||||
missing 'Authorization' request header`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// all the backend_urls are unavailable for unauthorized user
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_map:
|
||||
- src_paths: ["/foo/.*"]
|
||||
url_prefix:
|
||||
- http://127.0.0.1:1/
|
||||
- http://127.0.0.1:2/`
|
||||
requestURL = "http://some-host.com/foo/?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// all the backend_urls are unavailable for authorized user
|
||||
cfgStr = `
|
||||
users:
|
||||
- username: some-user
|
||||
url_map:
|
||||
- src_paths: ["/foo/.*"]
|
||||
url_prefix:
|
||||
- http://127.0.0.1:1/
|
||||
- http://127.0.0.1:2/`
|
||||
requestURL = "http://some-user@some-host.com/foo/?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "some-user" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
|
||||
// zero discovered backend IPs
|
||||
customResolver := &fakeResolver{
|
||||
Resolver: &net.Resolver{},
|
||||
lookupIPAddrResults: map[string][]net.IPAddr{
|
||||
"some-addr": {},
|
||||
},
|
||||
}
|
||||
origResolver := netutil.Resolver
|
||||
netutil.Resolver = customResolver
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: ['http://some-addr:1234/foo/bar']
|
||||
discover_backend_ips: true`
|
||||
requestURL = "http://abc.com/def/?de=fg"
|
||||
backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
|
||||
panic(fmt.Errorf("backend handler shouldn't be called"))
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /def/?de=fg; all the 0 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
netutil.Resolver = origResolver
|
||||
|
||||
// retry_status_codes failure
|
||||
var retries atomic.Int64
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
|
||||
retry_status_codes: [500, 502]`
|
||||
requestURL = "http://some-host.com/foo/?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, _ *http.Request) {
|
||||
retries.Add(1)
|
||||
w.WriteHeader(500)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=502
|
||||
remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
if n := retries.Load(); n != 2 {
|
||||
t.Fatalf("unexpected number of retries; got %d; want 2", n)
|
||||
}
|
||||
|
||||
// retry_status_codes success
|
||||
retries.Store(0)
|
||||
cfgStr = `
|
||||
unauthorized_user:
|
||||
url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
|
||||
retry_status_codes: [500, 502]`
|
||||
requestURL = "http://some-host.com/foo/?de=fg"
|
||||
backendHandler = func(w http.ResponseWriter, r *http.Request) {
|
||||
if n := retries.Add(1); n < 2 {
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
|
||||
}
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
requested_url={BACKEND}/path2/foo/?de=fg`
|
||||
f(cfgStr, requestURL, backendHandler, responseExpected)
|
||||
if n := retries.Load(); n != 2 {
|
||||
t.Fatalf("unexpected number of retries; got %d; want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeResponseWriter struct {
|
||||
h http.Header
|
||||
|
||||
bb bytes.Buffer
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) getResponse() string {
|
||||
return w.bb.String()
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) Header() http.Header {
|
||||
if w.h == nil {
|
||||
w.h = http.Header{}
|
||||
}
|
||||
return w.h
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) Write(p []byte) (int, error) {
|
||||
return w.bb.Write(p)
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) WriteHeader(statusCode int) {
|
||||
fmt.Fprintf(&w.bb, "statusCode=%d\n", statusCode)
|
||||
if w.h == nil {
|
||||
return
|
||||
}
|
||||
err := w.h.WriteSubset(&w.bb, map[string]bool{
|
||||
"Content-Length": true,
|
||||
"Content-Type": true,
|
||||
"Date": true,
|
||||
"X-Content-Type-Options": true,
|
||||
})
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("cannot marshal headers: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadTrackingBody_RetrySuccess(t *testing.T) {
|
||||
f := func(s string, maxBodySize int) {
|
||||
t.Helper()
|
||||
|
||||
@@ -335,8 +335,8 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
|
||||
t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
|
||||
}
|
||||
|
||||
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
|
||||
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
|
||||
f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
|
||||
f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write")
|
||||
// unsuccessful dns resolve
|
||||
f(ui, `/test`, "http://non-exist-dns-addr/test")
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/netstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
@@ -12,7 +14,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/clusternative/stream"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -25,7 +26,7 @@ var (
|
||||
func InsertHandler(c net.Conn) error {
|
||||
// There is no need in response compression, since
|
||||
// lower-level vminsert sends only small packets to upper-level vminsert.
|
||||
bc, err := handshake.VMInsertServer(c, 0)
|
||||
bc, err := handshake.VMInsertServer(c, 0, netstorage.GetNodeID())
|
||||
if err != nil {
|
||||
if errors.Is(err, handshake.ErrIgnoreHealthcheck) {
|
||||
return nil
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
package netstorage
|
||||
|
||||
import (
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
// See the following docs:
|
||||
// - https://www.eecs.umich.edu/techreports/cse/96/CSE-TR-316-96.pdf
|
||||
// - https://github.com/dgryski/go-rendezvous
|
||||
@@ -13,14 +9,10 @@ type consistentHash struct {
|
||||
nodeHashes []uint64
|
||||
}
|
||||
|
||||
func newConsistentHash(nodes []string, hashSeed uint64) *consistentHash {
|
||||
nodeHashes := make([]uint64, len(nodes))
|
||||
for i, node := range nodes {
|
||||
nodeHashes[i] = xxhash.Sum64([]byte(node))
|
||||
}
|
||||
func newConsistentHash(ids []uint64, hashSeed uint64) *consistentHash {
|
||||
return &consistentHash{
|
||||
hashSeed: hashSeed,
|
||||
nodeHashes: nodeHashes,
|
||||
nodeHashes: ids,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,16 +4,18 @@ import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
func TestConsistentHash(t *testing.T) {
|
||||
r := rand.New(rand.NewSource(1))
|
||||
|
||||
nodes := []string{
|
||||
"node1",
|
||||
"node2",
|
||||
"node3",
|
||||
"node4",
|
||||
nodes := []uint64{
|
||||
xxhash.Sum64String("node1"),
|
||||
xxhash.Sum64String("node2"),
|
||||
xxhash.Sum64String("node3"),
|
||||
xxhash.Sum64String("node4"),
|
||||
}
|
||||
rh := newConsistentHash(nodes, 0)
|
||||
|
||||
|
||||
@@ -4,16 +4,19 @@ import (
|
||||
"math/rand"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
func BenchmarkConsistentHash(b *testing.B) {
|
||||
nodes := []string{
|
||||
"node1",
|
||||
"node2",
|
||||
"node3",
|
||||
"node4",
|
||||
nodes := []uint64{
|
||||
xxhash.Sum64String("node1"),
|
||||
xxhash.Sum64String("node2"),
|
||||
xxhash.Sum64String("node3"),
|
||||
xxhash.Sum64String("node4"),
|
||||
}
|
||||
rh := newConsistentHash(nodes, 0)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(benchKeys)))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
@@ -12,7 +14,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
// InsertCtx is a generic context for inserting data.
|
||||
|
||||
@@ -6,10 +6,14 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/consts"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
@@ -21,8 +25,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -43,6 +45,9 @@ var (
|
||||
"On the other side, disabled re-routing minimizes the number of active time series in the cluster "+
|
||||
"during rolling restarts and during spikes in series churn rate. "+
|
||||
"See also -disableRerouting")
|
||||
usePersistentStorageNodeID = flag.Bool("vmstorageUsePersistentID", false, "Whether to use persistent storage node ID for -storageNode instances. "+
|
||||
"If set to false uses storage node address in order to generate an ID. "+
|
||||
"Using persistent node ID is useful if vmstorage node address changes over time, e.g. due to dynamic IP addresses or DNS names. ")
|
||||
)
|
||||
|
||||
var errStorageReadOnly = errors.New("storage node is read only")
|
||||
@@ -278,7 +283,7 @@ func (sn *storageNode) checkHealth() {
|
||||
}
|
||||
return
|
||||
}
|
||||
logger.Infof("successfully dialed -storageNode=%q", sn.dialer.Addr())
|
||||
logger.Infof("successfully dialed -storageNode=%q (node ID: %d)", sn.dialer.Addr(), sn.id.Load())
|
||||
sn.lastDialErr = nil
|
||||
sn.bc = bc
|
||||
sn.isBroken.Store(false)
|
||||
@@ -398,15 +403,24 @@ func (sn *storageNode) dial() (*handshake.BufferedConn, error) {
|
||||
if *disableRPCCompression {
|
||||
compressionLevel = 0
|
||||
}
|
||||
bc, err := handshake.VMInsertClient(c, compressionLevel)
|
||||
bc, id, err := handshake.VMInsertClient(c, compressionLevel)
|
||||
if err != nil {
|
||||
_ = c.Close()
|
||||
sn.handshakeErrors.Inc()
|
||||
return nil, fmt.Errorf("handshake error: %w", err)
|
||||
}
|
||||
sn.id.CompareAndSwap(0, id)
|
||||
return bc, nil
|
||||
}
|
||||
|
||||
func (sn *storageNode) getID() uint64 {
|
||||
// Ensure that the id is populated
|
||||
if sn.id.Load() == 0 {
|
||||
sn.checkHealth()
|
||||
}
|
||||
return sn.id.Load()
|
||||
}
|
||||
|
||||
// storageNode is a client sending data to vmstorage node.
|
||||
type storageNode struct {
|
||||
// isBroken is set to true if the given vmstorage node is temporarily unhealthy.
|
||||
@@ -473,6 +487,9 @@ type storageNode struct {
|
||||
// The total duration spent for sending data to vmstorage node.
|
||||
// This metric is useful for determining the saturation of vminsert->vmstorage link.
|
||||
sendDurationSeconds *metrics.FloatCounter
|
||||
|
||||
// id is a unique identifier for the storage node.
|
||||
id atomic.Uint64
|
||||
}
|
||||
|
||||
type storageNodesBucket struct {
|
||||
@@ -515,14 +532,31 @@ func MustStop() {
|
||||
mustStopStorageNodes(snb)
|
||||
}
|
||||
|
||||
// GetNodeID returns unique identifier for underlying storage nodes.
|
||||
func GetNodeID() uint64 {
|
||||
snb := getStorageNodesBucket()
|
||||
snIDs := make([]uint64, 0, len(snb.sns))
|
||||
for _, sn := range snb.sns {
|
||||
snIDs = append(snIDs, sn.getID())
|
||||
}
|
||||
slices.Sort(snIDs)
|
||||
idsM := make([]byte, 0)
|
||||
for _, id := range snIDs {
|
||||
idsM = encoding.MarshalUint64(idsM, id)
|
||||
}
|
||||
|
||||
return xxhash.Sum64(idsM)
|
||||
}
|
||||
|
||||
func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
|
||||
if len(addrs) == 0 {
|
||||
logger.Panicf("BUG: addrs must be non-empty")
|
||||
}
|
||||
ms := metrics.NewSet()
|
||||
nodesHash := newConsistentHash(addrs, hashSeed)
|
||||
sns := make([]*storageNode, 0, len(addrs))
|
||||
brokenNodes := make([]*storageNode, 0)
|
||||
stopCh := make(chan struct{})
|
||||
nodeIDs := make([]uint64, 0, len(addrs))
|
||||
for _, addr := range addrs {
|
||||
if _, _, err := net.SplitHostPort(addr); err != nil {
|
||||
// Automatically add missing port.
|
||||
@@ -568,10 +602,22 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
|
||||
}
|
||||
return 0
|
||||
})
|
||||
var nodeID uint64
|
||||
if *usePersistentStorageNodeID {
|
||||
nodeID = sn.getID()
|
||||
if nodeID == 0 {
|
||||
brokenNodes = append(brokenNodes, sn)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
nodeID = xxhash.Sum64String(addr)
|
||||
}
|
||||
nodeIDs = append(nodeIDs, nodeID)
|
||||
sns = append(sns, sn)
|
||||
}
|
||||
nodesHash := newConsistentHash(nodeIDs, hashSeed)
|
||||
|
||||
maxBufSizePerStorageNode = memory.Allowed() / 8 / len(sns)
|
||||
maxBufSizePerStorageNode = memory.Allowed() / 8 / len(addrs)
|
||||
if maxBufSizePerStorageNode > consts.MaxInsertPacketSizeForVMInsert {
|
||||
maxBufSizePerStorageNode = consts.MaxInsertPacketSizeForVMInsert
|
||||
}
|
||||
@@ -586,7 +632,12 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
|
||||
wg: &wg,
|
||||
}
|
||||
|
||||
for idx, sn := range sns {
|
||||
// add broken nodes to the end of the list
|
||||
// this is needed because consistent hash slots will be populated with IDs of available
|
||||
// storage nodes (if there are any) and indexes of consistent hash must be linked to healthy storage nodes
|
||||
snb.sns = append(snb.sns, brokenNodes...)
|
||||
|
||||
for idx, sn := range snb.sns {
|
||||
wg.Add(1)
|
||||
go func(sn *storageNode, idx int) {
|
||||
sn.run(snb, idx)
|
||||
@@ -594,6 +645,28 @@ func initStorageNodes(addrs []string, hashSeed uint64) *storageNodesBucket {
|
||||
}(sn, idx)
|
||||
}
|
||||
|
||||
// Watch for node become healthy and rebuild snb.
|
||||
for _, sn := range brokenNodes {
|
||||
wg.Add(1)
|
||||
sn := sn
|
||||
go watchStorageNodeHealthy(sn, func() {
|
||||
defer wg.Done()
|
||||
// rebuild snb in order to update consistent hash with an ID of the healthy storage node
|
||||
for {
|
||||
currentSnb := getStorageNodesBucket()
|
||||
newSnb := initStorageNodes(addrs, hashSeed)
|
||||
if !storageNodes.CompareAndSwap(currentSnb, newSnb) {
|
||||
// snb has been changed, so we need to stop the newSnb and try again
|
||||
mustStopStorageNodes(newSnb)
|
||||
continue
|
||||
}
|
||||
// stop previous snb and exit
|
||||
mustStopStorageNodes(currentSnb)
|
||||
break
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return snb
|
||||
}
|
||||
|
||||
@@ -606,6 +679,34 @@ func mustStopStorageNodes(snb *storageNodesBucket) {
|
||||
metrics.UnregisterSet(snb.ms, true)
|
||||
}
|
||||
|
||||
// watchStorageNodeHealthy watches for sn become healthy and calls cb once it is ready.
|
||||
func watchStorageNodeHealthy(sn *storageNode, cb func()) {
|
||||
for {
|
||||
sn.brLock.Lock()
|
||||
for !sn.isReady() {
|
||||
select {
|
||||
case <-sn.stopCh:
|
||||
sn.brLock.Unlock()
|
||||
return
|
||||
default:
|
||||
sn.brCond.Wait()
|
||||
}
|
||||
}
|
||||
sn.brLock.Unlock()
|
||||
|
||||
select {
|
||||
case <-sn.stopCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if sn.isReady() {
|
||||
cb()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rerouteRowsToReadyStorageNodes reroutes src from not ready snSource to ready storage nodes.
|
||||
//
|
||||
// The function blocks until src is fully re-routed.
|
||||
|
||||
@@ -31,7 +31,9 @@ var (
|
||||
|
||||
// NewVMSelectServer starts new server at the given addr, which serves vmselect requests from netstorage.
|
||||
func NewVMSelectServer(addr string) (*vmselectapi.Server, error) {
|
||||
api := &vmstorageAPI{}
|
||||
api := &vmstorageAPI{
|
||||
nodeID: netstorage.GetNodeID(),
|
||||
}
|
||||
limits := vmselectapi.Limits{
|
||||
MaxLabelNames: *maxTagKeys,
|
||||
MaxLabelValues: *maxTagValues,
|
||||
@@ -45,7 +47,9 @@ func NewVMSelectServer(addr string) (*vmselectapi.Server, error) {
|
||||
}
|
||||
|
||||
// vmstorageAPI impelements vmselectapi.API
|
||||
type vmstorageAPI struct{}
|
||||
type vmstorageAPI struct {
|
||||
nodeID uint64
|
||||
}
|
||||
|
||||
func (api *vmstorageAPI) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
|
||||
denyPartialResponse := httputils.GetDenyPartialResponse(nil)
|
||||
@@ -112,6 +116,10 @@ func (api *vmstorageAPI) RegisterMetricNames(qt *querytracer.Tracer, mrs []stora
|
||||
return netstorage.RegisterMetricNames(qt, mrs, dl)
|
||||
}
|
||||
|
||||
func (api *vmstorageAPI) GetID() uint64 {
|
||||
return api.nodeID
|
||||
}
|
||||
|
||||
// blockIterator implements vmselectapi.BlockIterator
|
||||
type blockIterator struct {
|
||||
workCh chan workItem
|
||||
|
||||
@@ -2107,6 +2107,9 @@ type storageNode struct {
|
||||
|
||||
// The number of list tenants errors to storageNode.
|
||||
tenantsErrors *metrics.Counter
|
||||
|
||||
// id is the unique identifier for the storageNode.
|
||||
id uint64
|
||||
}
|
||||
|
||||
func (sn *storageNode) registerMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline searchutils.Deadline) error {
|
||||
@@ -2954,6 +2957,12 @@ func getStorageNodes() []*storageNode {
|
||||
return snb.sns
|
||||
}
|
||||
|
||||
// GetNodeID returns unique identifier of vmselect
|
||||
func GetNodeID() uint64 {
|
||||
// Returns a 0 as persistent IDs are not intended to use with multi-level setup
|
||||
return 0
|
||||
}
|
||||
|
||||
// Init initializes storage nodes' connections to the given addrs.
|
||||
//
|
||||
// MustStop must be called when the initialized connections are no longer needed.
|
||||
@@ -3015,6 +3024,7 @@ func newStorageNode(ms *metrics.Set, group *storageNodesGroup, addr string) *sto
|
||||
sn := &storageNode{
|
||||
group: group,
|
||||
connPool: connPool,
|
||||
id: connPool.GetTargetNodeID(),
|
||||
|
||||
concurrentQueries: ms.NewCounter(fmt.Sprintf(`vm_concurrent_queries{name="vmselect", addr=%q}`, addr)),
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"files": {
|
||||
"main.css": "./static/css/main.fce049bf.css",
|
||||
"main.js": "./static/js/main.4b213512.js",
|
||||
"main.js": "./static/js/main.36983a8a.js",
|
||||
"static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
|
||||
"static/media/MetricsQL.md": "./static/media/MetricsQL.d46c42c8e891f06298c4.md",
|
||||
"index.html": "./index.html"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.fce049bf.css",
|
||||
"static/js/main.4b213512.js"
|
||||
"static/js/main.36983a8a.js"
|
||||
]
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.4b213512.js"></script><link href="./static/css/main.fce049bf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.36983a8a.js"></script><link href="./static/css/main.fce049bf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
2
app/vmselect/vmui/static/js/main.36983a8a.js
Normal file
2
app/vmselect/vmui/static/js/main.36983a8a.js
Normal file
File diff suppressed because one or more lines are too long
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* @remix-run/router v1.15.1
|
||||
* @remix-run/router v1.17.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
@@ -16,7 +16,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* React Router DOM v6.22.1
|
||||
* React Router DOM v6.24.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
@@ -27,7 +27,7 @@
|
||||
*/
|
||||
|
||||
/**
|
||||
* React Router v6.22.1
|
||||
* React Router v6.24.0
|
||||
*
|
||||
* Copyright (c) Remix Software Inc.
|
||||
*
|
||||
File diff suppressed because one or more lines are too long
@@ -10,9 +10,6 @@ aliases:
|
||||
- /ExtendedPromQL.html
|
||||
- /MetricsQL.html
|
||||
---
|
||||
|
||||
# MetricsQL
|
||||
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
|
||||
@@ -220,9 +217,11 @@ See also [descent_over_time](#descent_over_time).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [median_over_time](#median_over_time).
|
||||
See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
|
||||
|
||||
#### changes
|
||||
|
||||
@@ -262,6 +261,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
|
||||
|
||||
#### count_gt_over_time
|
||||
@@ -272,6 +273,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
|
||||
|
||||
#### count_le_over_time
|
||||
@@ -282,6 +285,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
|
||||
|
||||
#### count_ne_over_time
|
||||
@@ -292,7 +297,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
See also [count_over_time](#count_over_time).
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### count_over_time
|
||||
|
||||
@@ -313,6 +320,8 @@ The results are calculated independently per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
|
||||
|
||||
#### decreases_over_time
|
||||
@@ -432,6 +441,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### histogram_over_time
|
||||
|
||||
`histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
@@ -443,11 +454,15 @@ For example, the following query calculates median temperature by country over t
|
||||
|
||||
`histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### hoeffding_bound_lower
|
||||
|
||||
`hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
|
||||
#### hoeffding_bound_upper
|
||||
@@ -455,6 +470,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
`hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
|
||||
#### holt_winters
|
||||
@@ -462,8 +479,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
|
||||
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
|
||||
Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
Both `sf` and `tf` must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
@@ -495,13 +513,14 @@ See also [deriv](#deriv).
|
||||
|
||||
`increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
|
||||
@@ -510,12 +529,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
|
||||
|
||||
`increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
|
||||
#### increase_pure
|
||||
@@ -524,6 +544,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
|
||||
while [increase](#increase) ignores the first value in a series if it is too big.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
|
||||
|
||||
#### increases_over_time
|
||||
|
||||
`increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -540,15 +564,18 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### irate
|
||||
|
||||
`irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
|
||||
the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [rate](#rate) and [rollup_rate](#rollup_rate).
|
||||
@@ -587,6 +614,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
#### max_over_time
|
||||
@@ -594,9 +623,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
|
||||
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmax_over_time](#tmax_over_time).
|
||||
See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
|
||||
|
||||
#### median_over_time
|
||||
|
||||
@@ -604,6 +635,8 @@ See also [tmax_over_time](#tmax_over_time).
|
||||
on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [avg_over_time](#avg_over_time).
|
||||
|
||||
#### min_over_time
|
||||
@@ -611,9 +644,11 @@ See also [avg_over_time](#avg_over_time).
|
||||
`min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmin_over_time](#tmin_over_time).
|
||||
See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
|
||||
|
||||
#### mode_over_time
|
||||
|
||||
@@ -622,6 +657,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
values are discrete.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### outlier_iqr_over_time
|
||||
|
||||
`outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
|
||||
@@ -632,6 +669,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
|
||||
The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
|
||||
For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [outliers_iqr](#outliers_iqr).
|
||||
|
||||
#### predict_linear
|
||||
@@ -659,6 +698,8 @@ This function is supported by PromQL.
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The `phi` value must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [quantiles_over_time](#quantiles_over_time).
|
||||
@@ -670,6 +711,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [quantile_over_time](#quantile_over_time).
|
||||
|
||||
#### range_over_time
|
||||
@@ -680,11 +723,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rate
|
||||
|
||||
`rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
|
||||
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
|
||||
@@ -703,6 +747,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
on the given lookbehind window `d`. The calculations are performed individually per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
#### resets
|
||||
@@ -710,10 +756,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
`resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
|
||||
of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### rollup
|
||||
@@ -725,6 +772,10 @@ These values are calculated individually per each time series returned from the
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_candlestick
|
||||
|
||||
`rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
|
||||
@@ -736,6 +787,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rollup_delta
|
||||
|
||||
`rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -762,6 +815,8 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_increase
|
||||
|
||||
`rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -774,12 +829,17 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_rate
|
||||
|
||||
`rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
|
||||
for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
|
||||
and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
|
||||
when to use `rollup_rate()`.
|
||||
@@ -787,10 +847,12 @@ when to use `rollup_rate()`.
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
|
||||
|
||||
#### rollup_scrape_interval
|
||||
|
||||
`rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
|
||||
@@ -824,6 +886,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### share_le_over_time
|
||||
@@ -838,6 +902,8 @@ the share of time series values for the last 24 hours when memory usage was belo
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### share_eq_over_time
|
||||
@@ -849,6 +915,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### stale_samples_over_time
|
||||
@@ -866,6 +934,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stdvar_over_time](#stdvar_over_time).
|
||||
@@ -877,6 +947,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stddev_over_time](#stddev_over_time).
|
||||
@@ -888,6 +960,8 @@ values equal to `eq` on the given lookbehind window `d` per each time series ret
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### sum_gt_over_time
|
||||
@@ -897,6 +971,8 @@ values bigger than `gt` on the given lookbehind window `d` per each time series
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### sum_le_over_time
|
||||
@@ -906,6 +982,8 @@ values smaller or equal to `le` on the given lookbehind window `d` per each time
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### sum_over_time
|
||||
@@ -915,6 +993,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### sum2_over_time
|
||||
@@ -924,6 +1004,8 @@ values on the given lookbehind window `d` per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### timestamp
|
||||
|
||||
`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
|
||||
@@ -1001,6 +1083,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
|
||||
@@ -2230,7 +2314,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
|
||||
Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
|
||||
For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
|
||||
since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
|
||||
VictoriaMetrics performs subqueries in the following way:
|
||||
|
||||
@@ -2263,5 +2348,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
|
||||
For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
|
||||
* If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
@@ -111,8 +111,8 @@ func main() {
|
||||
blocksCount := tm.SmallBlocksCount + tm.BigBlocksCount
|
||||
rowsCount := tm.SmallRowsCount + tm.BigRowsCount
|
||||
sizeBytes := tm.SmallSizeBytes + tm.BigSizeBytes
|
||||
logger.Infof("successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
|
||||
*storageDataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
|
||||
logger.Infof("successfully opened storage %q (node ID: %d) in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
|
||||
*storageDataPath, strg.GetID(), time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
|
||||
|
||||
// register storage metrics
|
||||
storageMetrics := metrics.NewSet()
|
||||
|
||||
@@ -101,7 +101,7 @@ func (s *VMInsertServer) run() {
|
||||
// There is no need in response compression, since
|
||||
// vmstorage sends only small packets to vminsert.
|
||||
compressionLevel := 0
|
||||
bc, err := handshake.VMInsertServer(c, compressionLevel)
|
||||
bc, err := handshake.VMInsertServer(c, compressionLevel, s.storage.GetID())
|
||||
if err != nil {
|
||||
if s.isStopping() {
|
||||
// c is stopped inside VMInsertServer.MustStop
|
||||
|
||||
@@ -195,6 +195,10 @@ func (api *vmstorageAPI) setupTfss(qt *querytracer.Tracer, sq *storage.SearchQue
|
||||
return tfss, nil
|
||||
}
|
||||
|
||||
func (api *vmstorageAPI) GetID() uint64 {
|
||||
return api.s.GetID()
|
||||
}
|
||||
|
||||
// blockIterator implements vmselectapi.BlockIterator
|
||||
type blockIterator struct {
|
||||
sr storage.Search
|
||||
|
||||
@@ -6,7 +6,7 @@ COPY web/ /build/
|
||||
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
|
||||
GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
|
||||
|
||||
FROM alpine:3.20.1
|
||||
FROM alpine:3.20.2
|
||||
USER root
|
||||
|
||||
COPY --from=build-web-stage /build/web-amd64 /app/web
|
||||
|
||||
@@ -10,9 +10,6 @@ aliases:
|
||||
- /ExtendedPromQL.html
|
||||
- /MetricsQL.html
|
||||
---
|
||||
|
||||
# MetricsQL
|
||||
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
|
||||
@@ -220,9 +217,11 @@ See also [descent_over_time](#descent_over_time).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [median_over_time](#median_over_time).
|
||||
See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
|
||||
|
||||
#### changes
|
||||
|
||||
@@ -262,6 +261,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
|
||||
|
||||
#### count_gt_over_time
|
||||
@@ -272,6 +273,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
|
||||
|
||||
#### count_le_over_time
|
||||
@@ -282,6 +285,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
|
||||
|
||||
#### count_ne_over_time
|
||||
@@ -292,7 +297,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
See also [count_over_time](#count_over_time).
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### count_over_time
|
||||
|
||||
@@ -313,6 +320,8 @@ The results are calculated independently per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
|
||||
|
||||
#### decreases_over_time
|
||||
@@ -432,6 +441,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### histogram_over_time
|
||||
|
||||
`histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
@@ -443,11 +454,15 @@ For example, the following query calculates median temperature by country over t
|
||||
|
||||
`histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### hoeffding_bound_lower
|
||||
|
||||
`hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
|
||||
#### hoeffding_bound_upper
|
||||
@@ -455,6 +470,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
|
||||
`hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
|
||||
upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
|
||||
#### holt_winters
|
||||
@@ -462,8 +479,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
|
||||
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
|
||||
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
|
||||
Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
Both `sf` and `tf` must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
@@ -495,13 +513,14 @@ See also [deriv](#deriv).
|
||||
|
||||
`increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
|
||||
@@ -510,12 +529,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
|
||||
|
||||
`increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
|
||||
#### increase_pure
|
||||
@@ -524,6 +544,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
|
||||
of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
|
||||
while [increase](#increase) ignores the first value in a series if it is too big.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [increase](#increas) and [increase_prometheus](#increase_prometheus).
|
||||
|
||||
#### increases_over_time
|
||||
|
||||
`increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -540,15 +564,18 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### irate
|
||||
|
||||
`irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
|
||||
the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [rate](#rate) and [rollup_rate](#rollup_rate).
|
||||
@@ -587,6 +614,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
|
||||
over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
#### max_over_time
|
||||
@@ -594,9 +623,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
|
||||
`max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmax_over_time](#tmax_over_time).
|
||||
See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
|
||||
|
||||
#### median_over_time
|
||||
|
||||
@@ -604,6 +635,8 @@ See also [tmax_over_time](#tmax_over_time).
|
||||
on the given lookbehind window `d` per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [avg_over_time](#avg_over_time).
|
||||
|
||||
#### min_over_time
|
||||
@@ -611,9 +644,11 @@ See also [avg_over_time](#avg_over_time).
|
||||
`min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [tmin_over_time](#tmin_over_time).
|
||||
See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
|
||||
|
||||
#### mode_over_time
|
||||
|
||||
@@ -622,6 +657,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
values are discrete.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### outlier_iqr_over_time
|
||||
|
||||
`outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
|
||||
@@ -632,6 +669,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
|
||||
The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
|
||||
For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [outliers_iqr](#outliers_iqr).
|
||||
|
||||
#### predict_linear
|
||||
@@ -659,6 +698,8 @@ This function is supported by PromQL.
|
||||
on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The `phi` value must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [quantiles_over_time](#quantiles_over_time).
|
||||
@@ -670,6 +711,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [quantile_over_time](#quantile_over_time).
|
||||
|
||||
#### range_over_time
|
||||
@@ -680,11 +723,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rate
|
||||
|
||||
`rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
|
||||
over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
|
||||
passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
|
||||
@@ -703,6 +747,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
on the given lookbehind window `d`. The calculations are performed individually per each time series returned
|
||||
from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
#### resets
|
||||
@@ -710,10 +756,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
`resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
|
||||
of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
|
||||
per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### rollup
|
||||
@@ -725,6 +772,10 @@ These values are calculated individually per each time series returned from the
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_candlestick
|
||||
|
||||
`rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
|
||||
@@ -736,6 +787,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### rollup_delta
|
||||
|
||||
`rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -762,6 +815,8 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_increase
|
||||
|
||||
`rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
@@ -774,12 +829,17 @@ See also [label_match](#label_match).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rollup_rate
|
||||
|
||||
`rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
|
||||
for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
|
||||
on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
|
||||
and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
|
||||
when to use `rollup_rate()`.
|
||||
@@ -787,10 +847,12 @@ when to use `rollup_rate()`.
|
||||
Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
|
||||
See also [label_match](#label_match).
|
||||
|
||||
The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
|
||||
|
||||
See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
|
||||
|
||||
#### rollup_scrape_interval
|
||||
|
||||
`rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
|
||||
@@ -824,6 +886,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### share_le_over_time
|
||||
@@ -838,6 +902,8 @@ the share of time series values for the last 24 hours when memory usage was belo
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### share_eq_over_time
|
||||
@@ -849,6 +915,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### stale_samples_over_time
|
||||
@@ -866,6 +934,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stdvar_over_time](#stdvar_over_time).
|
||||
@@ -877,6 +947,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [stddev_over_time](#stddev_over_time).
|
||||
@@ -888,6 +960,8 @@ values equal to `eq` on the given lookbehind window `d` per each time series ret
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
|
||||
|
||||
#### sum_gt_over_time
|
||||
@@ -897,6 +971,8 @@ values bigger than `gt` on the given lookbehind window `d` per each time series
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
|
||||
|
||||
#### sum_le_over_time
|
||||
@@ -906,6 +982,8 @@ values smaller or equal to `le` on the given lookbehind window `d` per each time
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
|
||||
|
||||
#### sum_over_time
|
||||
@@ -915,6 +993,8 @@ on the given lookbehind window `d` per each time series returned from the given
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
#### sum2_over_time
|
||||
@@ -924,6 +1004,8 @@ values on the given lookbehind window `d` per each time series returned from the
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
#### timestamp
|
||||
|
||||
`timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
|
||||
@@ -1001,6 +1083,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
|
||||
|
||||
See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
|
||||
|
||||
|
||||
@@ -2230,7 +2314,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
|
||||
Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
|
||||
For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
|
||||
since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
|
||||
VictoriaMetrics performs subqueries in the following way:
|
||||
|
||||
@@ -2263,5 +2348,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
|
||||
For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
|
||||
* If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
|
||||
is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
|
||||
For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
|
||||
This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
|
||||
starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
|
||||
|
||||
@@ -48,9 +48,9 @@ const ExploreLogs: FC = () => {
|
||||
|
||||
const newPeriod = getPeriod();
|
||||
setPeriod(newPeriod);
|
||||
fetchLogs(newPeriod);
|
||||
fetchLogHits(newPeriod);
|
||||
|
||||
fetchLogs(newPeriod).then((isSuccess) => {
|
||||
isSuccess && fetchLogHits(newPeriod);
|
||||
}).catch(e => e);
|
||||
setSearchParamsFromKeys( {
|
||||
query,
|
||||
"g0.range_input": duration,
|
||||
@@ -90,7 +90,7 @@ const ExploreLogs: FC = () => {
|
||||
onRun={handleRunQuery}
|
||||
onChangeMarkdownParsing={handleChangeMarkdownParsing}
|
||||
/>
|
||||
{isLoading && <Spinner />}
|
||||
{isLoading && <Spinner message={"Loading logs..."}/>}
|
||||
{error && <Alert variant="error">{error}</Alert>}
|
||||
{!error && (
|
||||
<ExploreLogsBarChart
|
||||
|
||||
@@ -57,7 +57,10 @@ const ExploreLogsBarChart: FC<Props> = ({ logHits, period, error, isLoading }) =
|
||||
"vm-block_mobile": isMobile,
|
||||
})}
|
||||
>
|
||||
{isLoading && <Spinner containerStyles={{ position: "absolute" }}/>}
|
||||
{isLoading && <Spinner
|
||||
message={"Loading hits stats..."}
|
||||
containerStyles={{ position: "absolute" }}
|
||||
/>}
|
||||
{!error && noDataMessage && (
|
||||
<div className="vm-explore-logs-chart__empty">
|
||||
<Alert variant="info">{noDataMessage}</Alert>
|
||||
|
||||
@@ -52,20 +52,23 @@ export const useFetchLogs = (server: string, query: string, limit: number) => {
|
||||
setError(text);
|
||||
setLogs([]);
|
||||
setIsLoading(prev => ({ ...prev, [id]: false }));
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
const lines = text.split("\n").filter(line => line).slice(0, limit);
|
||||
const data = lines.map(parseLineToJSON).filter(line => line) as Logs[];
|
||||
setLogs(data);
|
||||
setIsLoading(prev => ({ ...prev, [id]: false }));
|
||||
return true;
|
||||
} catch (e) {
|
||||
setIsLoading(prev => ({ ...prev, [id]: false }));
|
||||
if (e instanceof Error && e.name !== "AbortError") {
|
||||
setError(String(e));
|
||||
console.error(e);
|
||||
setLogs([]);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
setIsLoading(prev => ({ ...prev, [id]: false }));
|
||||
}, [url, query, limit]);
|
||||
|
||||
return {
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
DOCKER_NAMESPACE ?= victoriametrics
|
||||
|
||||
ROOT_IMAGE ?= alpine:3.20.1
|
||||
ROOT_IMAGE ?= alpine:3.20.2
|
||||
ROOT_IMAGE_SCRATCH ?= scratch
|
||||
CERTS_IMAGE := alpine:3.20.1
|
||||
CERTS_IMAGE := alpine:3.20.2
|
||||
|
||||
GO_BUILDER_IMAGE := golang:1.22.5-alpine
|
||||
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
|
||||
|
||||
@@ -4,7 +4,7 @@ services:
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:v1.101.0
|
||||
image: victoriametrics/vmagent:v1.102.0
|
||||
depends_on:
|
||||
- "vminsert"
|
||||
ports:
|
||||
@@ -39,7 +39,7 @@ services:
|
||||
# where N is number of vmstorages (2 in this case).
|
||||
vmstorage-1:
|
||||
container_name: vmstorage-1
|
||||
image: victoriametrics/vmstorage:v1.101.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.102.0-cluster
|
||||
ports:
|
||||
- 8482
|
||||
- 8400
|
||||
@@ -51,7 +51,7 @@ services:
|
||||
restart: always
|
||||
vmstorage-2:
|
||||
container_name: vmstorage-2
|
||||
image: victoriametrics/vmstorage:v1.101.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.102.0-cluster
|
||||
ports:
|
||||
- 8482
|
||||
- 8400
|
||||
@@ -66,7 +66,7 @@ services:
|
||||
# pre-process them and distributes across configured vmstorage shards.
|
||||
vminsert:
|
||||
container_name: vminsert
|
||||
image: victoriametrics/vminsert:v1.101.0-cluster
|
||||
image: victoriametrics/vminsert:v1.102.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -81,7 +81,7 @@ services:
|
||||
# vmselect collects results from configured `--storageNode` shards.
|
||||
vmselect-1:
|
||||
container_name: vmselect-1
|
||||
image: victoriametrics/vmselect:v1.101.0-cluster
|
||||
image: victoriametrics/vmselect:v1.102.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -94,7 +94,7 @@ services:
|
||||
restart: always
|
||||
vmselect-2:
|
||||
container_name: vmselect-2
|
||||
image: victoriametrics/vmselect:v1.101.0-cluster
|
||||
image: victoriametrics/vmselect:v1.102.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -112,7 +112,7 @@ services:
|
||||
# It can be used as an authentication proxy.
|
||||
vmauth:
|
||||
container_name: vmauth
|
||||
image: victoriametrics/vmauth:v1.101.0
|
||||
image: victoriametrics/vmauth:v1.102.0
|
||||
depends_on:
|
||||
- "vmselect-1"
|
||||
- "vmselect-2"
|
||||
@@ -127,7 +127,7 @@ services:
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:v1.101.0
|
||||
image: victoriametrics/vmalert:v1.102.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
|
||||
@@ -55,7 +55,7 @@ services:
|
||||
# scraping, storing metrics and serve read requests.
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.101.0
|
||||
image: victoriametrics/victoria-metrics:v1.102.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
|
||||
@@ -4,7 +4,7 @@ services:
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:v1.101.0
|
||||
image: victoriametrics/vmagent:v1.102.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -22,7 +22,7 @@ services:
|
||||
# storing metrics and serve read requests.
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.101.0
|
||||
image: victoriametrics/victoria-metrics:v1.102.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
- 8089:8089
|
||||
@@ -65,7 +65,7 @@ services:
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:v1.101.0
|
||||
image: victoriametrics/vmalert:v1.102.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
- "alertmanager"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
services:
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:v1.101.0
|
||||
image: victoriametrics/vmagent:v1.102.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.101.0
|
||||
image: victoriametrics/victoria-metrics:v1.102.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -51,7 +51,7 @@ services:
|
||||
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:v1.101.0
|
||||
image: victoriametrics/vmalert:v1.102.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -46,7 +46,7 @@ services:
|
||||
- '--config=/config.yml'
|
||||
|
||||
vmsingle:
|
||||
image: victoriametrics/victoria-metrics:v1.101.0
|
||||
image: victoriametrics/victoria-metrics:v1.102.0
|
||||
ports:
|
||||
- '8428:8428'
|
||||
command:
|
||||
|
||||
@@ -19,8 +19,8 @@ On the server:
|
||||
* VictoriaMetrics is running on ports: 8428, 8089, 4242, 2003 and they are bound to the local interface.
|
||||
|
||||
********************************************************************************
|
||||
# This image includes v1.101.0 release of VictoriaMetrics.
|
||||
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.101.0
|
||||
# This image includes v1.102.0 release of VictoriaMetrics.
|
||||
# See Release notes https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0
|
||||
|
||||
# Welcome to VictoriaMetrics droplet!
|
||||
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /Articles.html
|
||||
---
|
||||
|
||||
# Articles
|
||||
|
||||
See also [case studies](https://docs.victoriametrics.com/casestudies/).
|
||||
|
||||
## Third-party articles and slides about VictoriaMetrics
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /BestPractices.html
|
||||
---
|
||||
|
||||
# VictoriaMetrics best practices
|
||||
|
||||
## Install Recommendation
|
||||
|
||||
It is recommended running the latest available release of VictoriaMetrics from [this page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
|
||||
|
||||
@@ -9,8 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /CHANGELOG.html
|
||||
---
|
||||
# CHANGELOG
|
||||
|
||||
The following `tip` changes can be tested by building VictoriaMetrics components from the latest commits according to the following docs:
|
||||
|
||||
* [How to build single-node VictoriaMetrics](https://docs.victoriametrics.com/#how-to-build-from-sources)
|
||||
@@ -31,6 +29,18 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
|
||||
|
||||
## tip
|
||||
|
||||
**Update note 1: [vmauth](https://docs.victoriametrics.com/vmauth/) HTTP response code has changed from 503 to 502 for a case when all upstream backends were not available. This was changed to align [vmauth](https://docs.victoriametrics.com/vmauth/) behaviour with other well-known reverse-proxies behaviour. **
|
||||
|
||||
**Update note 2: release contains breaking change to inter-cluster communication. The `vmstorage` nodes with version lower than `tip` won't be able to communicate with `vmselect` and `vminsert` nodes with version lower than `tip`. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5438) issue for the details.**
|
||||
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.20.1 to 3.20.2. See [alpine 3.20.2 release notes](https://alpinelinux.org/posts/Alpine-3.20.2-released.html).
|
||||
|
||||
* FEATURE: [vmauth](./vmauth.md): add `keep_original_host` option, which can be used for proxying the original `Host` header from client request to the backend. By default the backend host is used as `Host` header when proxying requests to the configured backends. See [these docs](./vmauth.md#host-http-header).
|
||||
* FEATURE: [vmauth](./vmauth.md) now returns HTTP 502 status code when all upstream backends are not available. Previously, it returned HTTP 503 status code. This change aligns vmauth behavior with other well-known reverse-proxies behavior.
|
||||
* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): use a persistent `vmstorage` node ID for consistent hashing at data write path. This allows to keep the same data distribution after `vmstorage` changes its IP address. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5438).
|
||||
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): properly proxy requests to backend urls ending with `/` if the original request path equals to `/`. Previously the trailing `/` at the backend path was incorrectly removed. For example, if the request to `http://vmauth/` is configured to be proxied to `url_prefix=http://backend/foo/`, then it was proxied to `http://backend/foo`, while it should go to `http://backend/foo/`.
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): fix `cannot read data after closing the reader` error when proxying HTTP requests without body (aka `GET` requests). The issue has been introduced in [v1.102.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0) in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/7ee57974935a662896f2de40fdf613156630617d).
|
||||
|
||||
## [v1.102.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.0)
|
||||
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /CHANGELOG_2020.html
|
||||
---
|
||||
|
||||
# CHANGELOG for the year 2020
|
||||
|
||||
## [v1.51.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.51.0)
|
||||
|
||||
Released at 2020-12-27
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /CHANGELOG_2021.html
|
||||
---
|
||||
|
||||
# CHANGELOG for the year 2021
|
||||
|
||||
## [v1.71.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.71.0)
|
||||
|
||||
Released at 2021-12-20
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /CHANGELOG_2022.html
|
||||
---
|
||||
|
||||
# CHANGELOG for the year 2022
|
||||
|
||||
## [v1.85.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.3)
|
||||
|
||||
Released at 2022-12-20
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /CHANGELOG_2023.html
|
||||
---
|
||||
|
||||
# CHANGELOG for the year 2023
|
||||
|
||||
## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0)
|
||||
|
||||
Released at 2023-12-13
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /CONTRIBUTING.html
|
||||
---
|
||||
|
||||
# Contributing to VictoriaMetrics
|
||||
|
||||
If you like VictoriaMetrics and want to contribute, then it would be great:
|
||||
|
||||
- Joining VictoriaMetrics community Slack ([Slack inviter](https://slack.victoriametrics.com/) and [Slack channel](https://victoriametrics.slack.com/))
|
||||
@@ -72,7 +69,7 @@ Examples of good changelog messages:
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): add support for [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol) when [sending / receiving data to / from Kafka](https://docs.victoriametrics.com/vmagent/#kafka-integration). This protocol allows saving egress network bandwidth costs when sending data from `vmagent` to `Kafka` located in another datacenter or availability zone. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1225).
|
||||
|
||||
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmgent.html) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).
|
||||
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): suppress `series after dedup` error message in logs when `-remoteWrite.streamAggr.dedupInterval` command-line flag is set at [vmagent](https://docs.victoriametrics.com/vmagent/) or when `-streamAggr.dedupInterval` command-line flag is set at [single-node VictoriaMetrics](https://docs.victoriametrics.com/).
|
||||
|
||||
## KISS principle
|
||||
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /CaseStudies.html
|
||||
---
|
||||
|
||||
# Case studies and talks
|
||||
|
||||
Below please find public case studies and talks from VictoriaMetrics users. You can also join our [community Slack channel](https://slack.victoriametrics.com/)
|
||||
where you can chat with VictoriaMetrics users to get additional references, reviews and case studies.
|
||||
|
||||
|
||||
@@ -9,12 +9,10 @@ title: Cluster version
|
||||
aliases:
|
||||
- /Cluster-VictoriaMetrics.html
|
||||
---
|
||||
# Cluster version
|
||||
|
||||
<picture>
|
||||
<source srcset="logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
<source srcset="logo.webp" media="(prefers-color-scheme: light)">
|
||||
<img src="logo.webp" width="300" alt="VictoriaMetrics logo">
|
||||
<source srcset="/logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
<source srcset="/logo.webp" media="(prefers-color-scheme: light)">
|
||||
<img src="/logo.webp" width="300" alt="VictoriaMetrics logo">
|
||||
</picture>
|
||||
|
||||
VictoriaMetrics is a fast, cost-effective and scalable time series database. It can be used as a long-term remote storage for Prometheus.
|
||||
@@ -61,7 +59,7 @@ Each service may scale independently and may run on the most suitable hardware.
|
||||
This is a [shared nothing architecture](https://en.wikipedia.org/wiki/Shared-nothing_architecture).
|
||||
It increases cluster availability, and simplifies cluster maintenance as well as cluster scaling.
|
||||
|
||||
<img src="Cluster-VictoriaMetrics_cluster-scheme.webp">
|
||||

|
||||
|
||||
## Multitenancy
|
||||
|
||||
@@ -1303,6 +1301,8 @@ Below is the output for `/path/to/vminsert -help`:
|
||||
Show VictoriaMetrics version
|
||||
-vmstorageDialTimeout duration
|
||||
Timeout for establishing RPC connections from vminsert to vmstorage. See also -vmstorageUserTimeout (default 3s)
|
||||
-vmstorageUsePersistentID
|
||||
Whether to use persistent storage node ID for -storageNode instances. If set to false uses storage node address in order to generate an ID. Using persistent node ID is useful if vmstorage node address changes over time, e.g. due to dynamic IP addresses or DNS names.
|
||||
-vmstorageUserTimeout duration
|
||||
Network timeout for RPC connections from vminsert to vmstorage (Linux only). Lower values speed up re-rerouting recovery when some of vmstorage nodes become unavailable because of networking issues. Read more about TCP_USER_TIMEOUT at https://blog.cloudflare.com/when-tcp-sockets-refuse-to-die/ . See also -vmstorageDialTimeout (default 3s)
|
||||
```
|
||||
|
||||
@@ -10,9 +10,6 @@ aliases:
|
||||
- /FAQ.html
|
||||
- /faq.html
|
||||
---
|
||||
|
||||
# FAQ
|
||||
|
||||
## What is the main purpose of VictoriaMetrics?
|
||||
|
||||
To provide the best monitoring solution.
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /LTS-releases.html
|
||||
---
|
||||
|
||||
# Long-term support releases
|
||||
|
||||
[Enterprise version of VictoriaMetrics](https://docs.victoriametrics.com/enterprise/) provides long-term support lines of releases (aka LTS releases).
|
||||
Every LTS line receives bugfixes and [security fixes](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/SECURITY.md) for 12 months after
|
||||
the initial release. New LTS lines are published every 6 months, so the latest two LTS lines are supported at any given moment. This gives up to 6 months
|
||||
|
||||
@@ -14,3 +14,20 @@ docs-images-to-webp-by-extension:
|
||||
sh -c 'find /docs/ -type f ! -path "/docs/operator/*" ! -path "/docs/_site/*" -name "*.$(IMAGES_EXTENSION)" -print0 | \
|
||||
xargs -0 -P $(MAKE_CONCURRENCY) -I {} sh -c '"'"'cwebp -preset drawing -m 6 -o "$${1%.*}.webp" $$1'"'"' _ {}'
|
||||
find docs/ -type f ! -path 'docs/operator/*' ! -path 'docs/_site/*' -name '*.$(IMAGES_EXTENSION)' -print0 | xargs -0 rm -f
|
||||
|
||||
docs-debug:
|
||||
if [ ! -d vmdocs ]; then \
|
||||
git clone --depth 1 git@github.com:VictoriaMetrics/vmdocs vmdocs; \
|
||||
fi; \
|
||||
cd vmdocs && \
|
||||
git checkout main && \
|
||||
git pull origin main && \
|
||||
cd .. && \
|
||||
docker build \
|
||||
-t vmdocs-docker-package \
|
||||
vmdocs && \
|
||||
docker run \
|
||||
--rm \
|
||||
--name vmdocs-docker-container \
|
||||
-p 1313:1313 \
|
||||
-v ./docs:/opt/docs/content vmdocs-docker-package
|
||||
|
||||
@@ -10,9 +10,6 @@ aliases:
|
||||
- /ExtendedPromQL.html
|
||||
- /MetricsQL.html
|
||||
---
|
||||
|
||||
# MetricsQL
|
||||
|
||||
[VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
|
||||
query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
|
||||
MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
|
||||
|
||||
@@ -9,10 +9,7 @@ menu:
|
||||
aliases:
|
||||
- /PerTenantStatistic.html
|
||||
---
|
||||
|
||||
# VictoriaMetrics Cluster Per Tenant Statistic
|
||||
|
||||
<img alt="cluster-per-tenant-stat" src="PerTenantStatistic-stats.webp">
|
||||

|
||||
|
||||
***The per-tenant statistic is a part of [enterprise package](https://docs.victoriametrics.com/enterprise/). It is available for download and evaluation at [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
To get the license key you can request a [free trial license](https://victoriametrics.com/products/enterprise/trial/).***
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /Quick-Start.html
|
||||
---
|
||||
|
||||
# Quick start
|
||||
|
||||
## How to install
|
||||
|
||||
VictoriaMetrics is distributed in two forms:
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
---
|
||||
title: VictoriaMetrics
|
||||
---
|
||||
# VictoriaMetrics
|
||||
|
||||
[](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
|
||||
[](https://hub.docker.com/r/victoriametrics/victoria-metrics)
|
||||
[](https://slack.victoriametrics.com/)
|
||||
@@ -12,9 +7,9 @@ title: VictoriaMetrics
|
||||
[](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
|
||||
|
||||
<picture>
|
||||
<source srcset="logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
<source srcset="logo.webp" media="(prefers-color-scheme: light)">
|
||||
<img src="logo.webp" width="300" alt="VictoriaMetrics logo">
|
||||
<source srcset="/logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
<source srcset="/logo.webp" media="(prefers-color-scheme: light)">
|
||||
<img src="/logo.webp" width="300" alt="VictoriaMetrics logo">
|
||||
</picture>
|
||||
|
||||
VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.
|
||||
@@ -313,7 +308,7 @@ Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaM
|
||||
|
||||
In the "Type and version" section it is recommended to set the type to "Prometheus" and the version to at least "2.24.x":
|
||||
|
||||
<img src="grafana-datasource-prometheus.webp" alt="Grafana datasource" />
|
||||

|
||||
|
||||
This allows Grafana to use a more efficient API to get label values.
|
||||
|
||||
@@ -503,7 +498,7 @@ via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit
|
||||
DataDog agent allows configuring destinations for metrics sending via ENV variable `DD_DD_URL`
|
||||
or via [configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) in section `dd_url`.
|
||||
|
||||
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.webp">
|
||||

|
||||
|
||||
To configure DataDog agent via ENV variable add the following prefix:
|
||||
|
||||
@@ -530,7 +525,7 @@ pick [single-node or cluster URL](https://docs.victoriametrics.com/url-examples/
|
||||
DataDog allows configuring [Dual Shipping](https://docs.datadoghq.com/agent/guide/dual-shipping/) for metrics
|
||||
sending via ENV variable `DD_ADDITIONAL_ENDPOINTS` or via configuration file `additional_endpoints`.
|
||||
|
||||
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.webp">
|
||||

|
||||
|
||||
Run DataDog using the following ENV variable with VictoriaMetrics as additional metrics receiver:
|
||||
|
||||
@@ -617,13 +612,13 @@ VictoriaMetrics performs the following transformations to the ingested InfluxDB
|
||||
|
||||
For example, the following InfluxDB line:
|
||||
|
||||
```raw
|
||||
```influxtextmetric
|
||||
foo,tag1=value1,tag2=value2 field1=12,field2=40
|
||||
```
|
||||
|
||||
is converted into the following Prometheus data points:
|
||||
|
||||
```raw
|
||||
```promtextmetric
|
||||
foo_field1{tag1="value1", tag2="value2"} 12
|
||||
foo_field2{tag1="value1", tag2="value2"} 40
|
||||
```
|
||||
@@ -2646,24 +2641,30 @@ It is built from `*.md` files located in [docs](https://github.com/VictoriaMetri
|
||||
and gets automatically updated once changes are merged to [master](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master) branch.
|
||||
To update the documentation follow the steps below:
|
||||
- [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks)
|
||||
VictoriaMetrics repo and apply changes to the docs:
|
||||
VictoriaMetrics repo and apply changes to the docs:
|
||||
- To update [the main page](https://docs.victoriametrics.com/) modify [this file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md).
|
||||
- To update other pages, apply changes to the corresponding file in [docs folder](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs).
|
||||
- If your changes contain an image then see [images in documentation](https://docs.victoriametrics.com/#images-in-documentation).
|
||||
- Once changes are made, execute the command below to finalize and sync the changes:
|
||||
```sh
|
||||
make docs-sync
|
||||
```
|
||||
|
||||
```sh
|
||||
make docs-sync
|
||||
```
|
||||
|
||||
- Create [a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request)
|
||||
with proposed changes and wait for it to be merged.
|
||||
with proposed changes and wait for it to be merged.
|
||||
|
||||
Requirements for changes to docs:
|
||||
|
||||
- Keep backward compatibility of existing links. Avoid changing anchors or deleting pages as they could have been
|
||||
used or posted in other docs, GitHub issues, stackoverlow answers, etc.
|
||||
- Keep docs simple. Try using as simple wording as possible.
|
||||
used or posted in other docs, GitHub issues, stackoverlow answers, etc.
|
||||
- Keep docs clear, concise and simple. Try using as simple wording as possible, without loosing the clarity.
|
||||
- Keep docs consistent. When modifying existing docs, verify that other places referencing to this doc are still relevant.
|
||||
- Prefer improving the existing docs instead of adding new ones.
|
||||
- Use absolute links.
|
||||
- Use absolute links. This simplifies moving docs between different files.
|
||||
|
||||
Priodically run `make spellcheck` - this command detects spelling errors at `docs/` folder. Please fix the found spelling errors
|
||||
and commit the fixes in a separate commit.
|
||||
|
||||
### Images in documentation
|
||||
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /Release-Guide.html
|
||||
---
|
||||
|
||||
# Release process guidance
|
||||
|
||||
## Pre-reqs
|
||||
|
||||
1. Make sure you have enterprise remote configured
|
||||
@@ -154,37 +151,19 @@ Once updated, run the following commands:
|
||||
|
||||
1. Commit and push changes to `master`.
|
||||
1. Run "Release" action on Github:
|
||||
<img src="Release-Guide_helm-release.webp" alt="release helm charts">
|
||||

|
||||
1. Merge new PRs *"Automatic update CHANGELOGs and READMEs"* and *"Synchronize docs"* after pipelines are complete.
|
||||
|
||||
## Ansible Roles
|
||||
|
||||
### Bump the version of images
|
||||
|
||||
Repository [https://github.com/VictoriaMetrics/ansible-playbooks](https://github.com/VictoriaMetrics/ansible-playbooks)
|
||||
|
||||
1. Update `vmagent` version in [`main.yml`](https://github.com/VictoriaMetrics/ansible-playbooks/blob/master/roles/vmagent/defaults/main.yml#L4)
|
||||
1. Update `vmalert` version in [`main.yml`](https://github.com/VictoriaMetrics/ansible-playbooks/blob/master/roles/vmalert/defaults/main.yml#L4)
|
||||
1. Update `cluster` version in [`main.yml`](https://github.com/VictoriaMetrics/ansible-playbooks/blob/master/roles/cluster/defaults/main.yml#L3)
|
||||
1. Update `single` version in [`main.yml`](https://github.com/VictoriaMetrics/ansible-playbooks/blob/master/roles/single/defaults/main.yml#L6)
|
||||
1. Commit changes
|
||||
1. Create a new tag
|
||||
1. Create a new release. This automatically publishes the new versions to galaxy.ansible.com
|
||||
1. Update the version of VictoriaMetrics components at [https://github.com/VictoriaMetrics/ansible-playbooks](https://github.com/VictoriaMetrics/ansible-playbooks).
|
||||
1. Commit changes.
|
||||
1. Create a new tag with `git tag -sm <TAG> <TAG>`.
|
||||
1. Push the changes with the new tag. This automatically publishes the new versions to galaxy.ansible.com.
|
||||
|
||||
## RPM packages
|
||||
|
||||
### Bump the version of components
|
||||
|
||||
Repository [https://github.com/VictoriaMetrics/victoriametrics-lts-rpm](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm)
|
||||
|
||||
1. Update `vmagent` version in [`vmagent.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmagent.spec#L2)
|
||||
1. Update `vmalert` version in [`vmalert.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmalert.spec#L2)
|
||||
1. Update `vmauth` version in [`vmauth.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmauth.spec#L2)
|
||||
1. Update `vmbackup` version in [`vmbackup.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmbackup.spec#L2)
|
||||
1. Update `vmctl` version in [`vmctl.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmctl.spec#L2)
|
||||
1. Update `vmrestore` version in [`vmrestore.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmrestore.spec#L2)
|
||||
1. Update `vminsert` version in [`vminsert.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vminsert.spec#L2)
|
||||
1. Update `vmselect` version in [`vmselect.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmselect.spec#L2)
|
||||
1. Update `vmstorage` version in [`vmstorage.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmstorage.spec#L2)
|
||||
1. Update `vmsingle` version in [`vmsingle.spec`](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm/blob/master/vmsingle.spec#L2)
|
||||
1. Update the version of VictoriaMetrics components at [https://github.com/VictoriaMetrics/victoriametrics-lts-rpm](https://github.com/VictoriaMetrics/victoriametrics-lts-rpm).
|
||||
1. Commit and push changes to the repository. This will automatically build and publish new versions of RPM packages.
|
||||
|
||||
@@ -9,8 +9,6 @@ title: VictoriaMetrics
|
||||
aliases:
|
||||
- /Single-server-VictoriaMetrics.html
|
||||
---
|
||||
# VictoriaMetrics
|
||||
|
||||
[](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
|
||||
[](https://hub.docker.com/r/victoriametrics/victoria-metrics)
|
||||
[](https://slack.victoriametrics.com/)
|
||||
@@ -20,9 +18,9 @@ aliases:
|
||||
[](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
|
||||
|
||||
<picture>
|
||||
<source srcset="logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
<source srcset="logo.webp" media="(prefers-color-scheme: light)">
|
||||
<img src="logo.webp" width="300" alt="VictoriaMetrics logo">
|
||||
<source srcset="/logo_white.webp" media="(prefers-color-scheme: dark)">
|
||||
<source srcset="/logo.webp" media="(prefers-color-scheme: light)">
|
||||
<img src="/logo.webp" width="300" alt="VictoriaMetrics logo">
|
||||
</picture>
|
||||
|
||||
VictoriaMetrics is a fast, cost-effective and scalable monitoring solution and time series database.
|
||||
@@ -321,7 +319,7 @@ Substitute `<victoriametrics-addr>` with the hostname or IP address of VictoriaM
|
||||
|
||||
In the "Type and version" section it is recommended to set the type to "Prometheus" and the version to at least "2.24.x":
|
||||
|
||||
<img src="grafana-datasource-prometheus.webp" alt="Grafana datasource" />
|
||||

|
||||
|
||||
This allows Grafana to use a more efficient API to get label values.
|
||||
|
||||
@@ -511,7 +509,7 @@ via ["submit metrics" API](https://docs.datadoghq.com/api/latest/metrics/#submit
|
||||
DataDog agent allows configuring destinations for metrics sending via ENV variable `DD_DD_URL`
|
||||
or via [configuration file](https://docs.datadoghq.com/agent/guide/agent-configuration-files/) in section `dd_url`.
|
||||
|
||||
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM.webp">
|
||||

|
||||
|
||||
To configure DataDog agent via ENV variable add the following prefix:
|
||||
|
||||
@@ -538,7 +536,7 @@ pick [single-node or cluster URL](https://docs.victoriametrics.com/url-examples/
|
||||
DataDog allows configuring [Dual Shipping](https://docs.datadoghq.com/agent/guide/dual-shipping/) for metrics
|
||||
sending via ENV variable `DD_ADDITIONAL_ENDPOINTS` or via configuration file `additional_endpoints`.
|
||||
|
||||
<img src="Single-server-VictoriaMetrics-sending_DD_metrics_to_VM_and_DD.webp">
|
||||

|
||||
|
||||
Run DataDog using the following ENV variable with VictoriaMetrics as additional metrics receiver:
|
||||
|
||||
@@ -625,13 +623,13 @@ VictoriaMetrics performs the following transformations to the ingested InfluxDB
|
||||
|
||||
For example, the following InfluxDB line:
|
||||
|
||||
```raw
|
||||
```influxtextmetric
|
||||
foo,tag1=value1,tag2=value2 field1=12,field2=40
|
||||
```
|
||||
|
||||
is converted into the following Prometheus data points:
|
||||
|
||||
```raw
|
||||
```promtextmetric
|
||||
foo_field1{tag1="value1", tag2="value2"} 12
|
||||
foo_field2{tag1="value1", tag2="value2"} 40
|
||||
```
|
||||
@@ -2654,24 +2652,30 @@ It is built from `*.md` files located in [docs](https://github.com/VictoriaMetri
|
||||
and gets automatically updated once changes are merged to [master](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master) branch.
|
||||
To update the documentation follow the steps below:
|
||||
- [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks)
|
||||
VictoriaMetrics repo and apply changes to the docs:
|
||||
VictoriaMetrics repo and apply changes to the docs:
|
||||
- To update [the main page](https://docs.victoriametrics.com/) modify [this file](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md).
|
||||
- To update other pages, apply changes to the corresponding file in [docs folder](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs).
|
||||
- If your changes contain an image then see [images in documentation](https://docs.victoriametrics.com/#images-in-documentation).
|
||||
- Once changes are made, execute the command below to finalize and sync the changes:
|
||||
```sh
|
||||
make docs-sync
|
||||
```
|
||||
|
||||
```sh
|
||||
make docs-sync
|
||||
```
|
||||
|
||||
- Create [a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request)
|
||||
with proposed changes and wait for it to be merged.
|
||||
with proposed changes and wait for it to be merged.
|
||||
|
||||
Requirements for changes to docs:
|
||||
|
||||
- Keep backward compatibility of existing links. Avoid changing anchors or deleting pages as they could have been
|
||||
used or posted in other docs, GitHub issues, stackoverlow answers, etc.
|
||||
- Keep docs simple. Try using as simple wording as possible.
|
||||
used or posted in other docs, GitHub issues, stackoverlow answers, etc.
|
||||
- Keep docs clear, concise and simple. Try using as simple wording as possible, without loosing the clarity.
|
||||
- Keep docs consistent. When modifying existing docs, verify that other places referencing to this doc are still relevant.
|
||||
- Prefer improving the existing docs instead of adding new ones.
|
||||
- Use absolute links.
|
||||
- Use absolute links. This simplifies moving docs between different files.
|
||||
|
||||
Priodically run `make spellcheck` - this command detects spelling errors at `docs/` folder. Please fix the found spelling errors
|
||||
and commit the fixes in a separate commit.
|
||||
|
||||
### Images in documentation
|
||||
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /Troubleshooting.html
|
||||
---
|
||||
|
||||
# Troubleshooting
|
||||
|
||||
This document contains troubleshooting guides for most common issues when working with VictoriaMetrics:
|
||||
|
||||
- [General troubleshooting checklist](#general-troubleshooting-checklist)
|
||||
|
||||
@@ -11,9 +11,6 @@ menu:
|
||||
aliases:
|
||||
- /VictoriaLogs/CHANGELOG.html
|
||||
---
|
||||
|
||||
# VictoriaLogs changelog
|
||||
|
||||
The following `tip` changes can be tested by building VictoriaLogs from the latest commit of [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/) repository
|
||||
according to [these docs](https://docs.victoriametrics.com/victorialogs/quickstart/#building-from-source-code)
|
||||
|
||||
|
||||
@@ -12,9 +12,6 @@ aliases:
|
||||
- /VictoriaLogs/FAQ.html
|
||||
- /VictoriaLogs/faq.html
|
||||
---
|
||||
|
||||
# VictoriaLogs FAQ
|
||||
|
||||
## What is the difference between VictoriaLogs and Elasticsearch (OpenSearch)?
|
||||
|
||||
Both Elasticsearch and VictoriaLogs allow ingesting structured and unstructured logs
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /VictoriaLogs/LogsQL.html
|
||||
---
|
||||
|
||||
# LogsQL
|
||||
|
||||
LogsQL is a simple yet powerful query language for [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/).
|
||||
See [examples](https://docs.victoriametrics.com/victorialogs/logsql-examples/) and [tutorial](#logsql-tutorial)
|
||||
in order to feel the language.
|
||||
|
||||
@@ -12,9 +12,6 @@ aliases:
|
||||
- /victorialogs/quick-start.html
|
||||
- /victorialogs/quick-start/
|
||||
---
|
||||
|
||||
# VictoriaLogs Quick Start
|
||||
|
||||
It is recommended to read [README](https://docs.victoriametrics.com/victorialogs/)
|
||||
and [Key Concepts](https://docs.victoriametrics.com/victorialogs/keyconcepts/)
|
||||
before you start working with VictoriaLogs.
|
||||
|
||||
@@ -1,12 +1,3 @@
|
||||
---
|
||||
title: VictoriaLogs
|
||||
weight: 0
|
||||
aliases:
|
||||
- /VictoriaLogs/
|
||||
- /VictoriaLogs/index.html
|
||||
---
|
||||
# VictoriaLogs
|
||||
|
||||
VictoriaLogs is [open source](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/app/victoria-logs) user-friendly database for logs
|
||||
from [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/).
|
||||
|
||||
|
||||
@@ -11,9 +11,6 @@ menu:
|
||||
aliases:
|
||||
- /VictoriaLogs/Roadmap.html
|
||||
---
|
||||
|
||||
# VictoriaLogs roadmap
|
||||
|
||||
The following functionality is available in [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
|
||||
|
||||
- [Data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/).
|
||||
|
||||
8
docs/VictoriaLogs/_index.md
Normal file
8
docs/VictoriaLogs/_index.md
Normal file
@@ -0,0 +1,8 @@
|
||||
---
|
||||
title: VictoriaLogs
|
||||
weight: 0
|
||||
aliases:
|
||||
- /VictoriaLogs/
|
||||
- /VictoriaLogs/index.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -11,9 +11,6 @@ aliases:
|
||||
- /victorialogs/data-ingestion/Filebeat.html
|
||||
- /victorialogs/data-ingestion/filebeat.html
|
||||
---
|
||||
|
||||
# Filebeat setup
|
||||
|
||||
Specify [`output.elasticsearch`](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) section in the `filebeat.yml`
|
||||
for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ aliases:
|
||||
Specify [http output](https://docs.fluentbit.io/manual/pipeline/outputs/http) section in the `fluentbit.conf`
|
||||
for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
|
||||
|
||||
```conf
|
||||
```fluentbit
|
||||
[Output]
|
||||
Name http
|
||||
Match *
|
||||
@@ -37,7 +37,7 @@ and uses the correct [stream fields](https://docs.victoriametrics.com/victorialo
|
||||
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) in the `uri`
|
||||
and inspecting VictoriaLogs logs then:
|
||||
|
||||
```conf
|
||||
```fluentbit
|
||||
[Output]
|
||||
Name http
|
||||
Match *
|
||||
@@ -52,7 +52,7 @@ If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#
|
||||
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
||||
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
||||
|
||||
```conf
|
||||
```fluentbit
|
||||
[Output]
|
||||
Name http
|
||||
Match *
|
||||
@@ -66,7 +66,7 @@ For example, the following config instructs VictoriaLogs to ignore `log.offset`
|
||||
If the Fluentbit sends logs to VictoriaLogs in another datacenter, then it may be useful enabling data compression via `compress gzip` option.
|
||||
This usually allows saving network bandwidth and costs by up to 5 times:
|
||||
|
||||
```conf
|
||||
```fluentbit
|
||||
[Output]
|
||||
Name http
|
||||
Match *
|
||||
@@ -82,7 +82,7 @@ By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [te
|
||||
If you need storing logs in other tenant, then specify the needed tenant via `header` options.
|
||||
For example, the following `fluentbit.conf` config instructs Fluentbit to store the data to `(AccountID=12, ProjectID=34)` tenant:
|
||||
|
||||
```conf
|
||||
```fluentbit
|
||||
[Output]
|
||||
Name http
|
||||
Match *
|
||||
|
||||
@@ -11,13 +11,10 @@ aliases:
|
||||
- /victorialogs/data-ingestion/logstash.html
|
||||
- /victorialogs/data-ingestion/Logstash.html
|
||||
---
|
||||
|
||||
# Logstash setup
|
||||
|
||||
Specify [`output.elasticsearch`](https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html) section in the `logstash.conf` file
|
||||
for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
|
||||
|
||||
```conf
|
||||
```logstash
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => ["http://localhost:9428/insert/elasticsearch/"]
|
||||
@@ -39,7 +36,7 @@ and uses the correct [stream fields](https://docs.victoriametrics.com/victorialo
|
||||
This can be done by specifying `debug` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters)
|
||||
and inspecting VictoriaLogs logs then:
|
||||
|
||||
```conf
|
||||
```logstash
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => ["http://localhost:9428/insert/elasticsearch/"]
|
||||
@@ -57,7 +54,7 @@ If some [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#
|
||||
during data ingestion, then they can be put into `ignore_fields` [parameter](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters).
|
||||
For example, the following config instructs VictoriaLogs to ignore `log.offset` and `event.original` fields in the ingested logs:
|
||||
|
||||
```conf
|
||||
```logstash
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => ["http://localhost:9428/insert/elasticsearch/"]
|
||||
@@ -74,7 +71,7 @@ output {
|
||||
If the Logstash sends logs to VictoriaLogs in another datacenter, then it may be useful enabling data compression via `http_compression: true` option.
|
||||
This usually allows saving network bandwidth and costs by up to 5 times:
|
||||
|
||||
```conf
|
||||
```logstash
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => ["http://localhost:9428/insert/elasticsearch/"]
|
||||
@@ -92,7 +89,7 @@ By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [te
|
||||
If you need storing logs in other tenant, then specify the needed tenant via `custom_headers` at `output.elasticsearch` section.
|
||||
For example, the following `logstash.conf` config instructs Logstash to store the data to `(AccountID=12, ProjectID=34)` tenant:
|
||||
|
||||
```conf
|
||||
```logstash
|
||||
output {
|
||||
elasticsearch {
|
||||
hosts => ["http://localhost:9428/insert/elasticsearch/"]
|
||||
|
||||
@@ -11,8 +11,6 @@ aliases:
|
||||
- /victorialogs/data-ingestion/Promtail.html
|
||||
- /victorialogs/data-ingestion/promtail.html
|
||||
---
|
||||
# Promtail setup
|
||||
|
||||
[Promtail](https://grafana.com/docs/loki/latest/clients/promtail/) is a default log shipper for Grafana Loki.
|
||||
Promtail can be configured to send the collected logs to VictoriaLogs according to the following docs.
|
||||
|
||||
|
||||
@@ -1,19 +1,3 @@
|
||||
---
|
||||
sort: 3
|
||||
title: Data ingestion
|
||||
weight: 3
|
||||
menu:
|
||||
docs:
|
||||
identifier: victorialogs-data-ingestion
|
||||
parent: "victorialogs"
|
||||
weight: 3
|
||||
aliases:
|
||||
- /VictoriaLogs/data-ingestion/
|
||||
- /VictoriaLogs/data-ingestion/index.html
|
||||
---
|
||||
|
||||
# Data ingestion
|
||||
|
||||
[VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) can accept logs from the following log collectors:
|
||||
|
||||
- Syslog, Rsyslog and Syslog-ng - see [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/).
|
||||
|
||||
@@ -11,9 +11,6 @@ aliases:
|
||||
- /victorialogs/data-ingestion/Vector.html
|
||||
- /victorialogs/data-ingestion/vector.html
|
||||
---
|
||||
# Vector setup
|
||||
|
||||
|
||||
## Elasticsearch sink
|
||||
|
||||
Specify [Elasticsearch sink type](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/) in the `vector.toml`
|
||||
|
||||
14
docs/VictoriaLogs/data-ingestion/_index.md
Normal file
14
docs/VictoriaLogs/data-ingestion/_index.md
Normal file
@@ -0,0 +1,14 @@
|
||||
---
|
||||
sort: 3
|
||||
title: Data ingestion
|
||||
weight: 3
|
||||
menu:
|
||||
docs:
|
||||
identifier: victorialogs-data-ingestion
|
||||
parent: "victorialogs"
|
||||
weight: 3
|
||||
aliases:
|
||||
- /VictoriaLogs/data-ingestion/
|
||||
- /VictoriaLogs/data-ingestion/index.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -7,9 +7,6 @@ menu:
|
||||
parent: "victorialogs-data-ingestion"
|
||||
weight: 10
|
||||
---
|
||||
|
||||
# Syslog setup
|
||||
|
||||
[VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) can accept logs in [Syslog formats](https://en.wikipedia.org/wiki/Syslog) at the specified TCP and UDP addresses
|
||||
via `-syslog.listenAddr.tcp` and `-syslog.listenAddr.udp` command-line flags. The following syslog formats are supported:
|
||||
|
||||
|
||||
@@ -10,9 +10,6 @@ menu:
|
||||
aliases:
|
||||
- /VictoriaLogs/keyConcepts.html
|
||||
---
|
||||
|
||||
# VictoriaLogs key concepts
|
||||
|
||||
## Data model
|
||||
|
||||
[VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) works with both structured and unstructured logs.
|
||||
|
||||
@@ -7,9 +7,6 @@ menu:
|
||||
parent: "victorialogs"
|
||||
weight: 100
|
||||
---
|
||||
|
||||
# LogsQL examples
|
||||
|
||||
## How to select recently ingested logs?
|
||||
|
||||
[Run](https://docs.victoriametrics.com/victorialogs/querying/) the following query:
|
||||
|
||||
@@ -1,19 +1,3 @@
|
||||
---
|
||||
sort: 4
|
||||
title: Querying
|
||||
weight: 4
|
||||
menu:
|
||||
docs:
|
||||
identifier: victorialogs-querying
|
||||
parent: "victorialogs"
|
||||
weight: 4
|
||||
aliases:
|
||||
- /VictoriaLogs/querying/
|
||||
- /VictoriaLogs/querying/index.html
|
||||
---
|
||||
|
||||
# Querying
|
||||
|
||||
[VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) can be queried with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/)
|
||||
via the following ways:
|
||||
|
||||
@@ -222,14 +206,14 @@ The `<offset>` can contain values in [the format specified here](https://docs.vi
|
||||
For example, the following command returns per-day number of logs with `error` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word)
|
||||
over the last week in New York time zone (`-4h`):
|
||||
|
||||
```logsql
|
||||
```sh
|
||||
curl http://localhost:9428/select/logsql/hits -d 'query=error' -d 'start=1w' -d 'step=1d' -d 'offset=-4h'
|
||||
```
|
||||
|
||||
Additionally, any number of `field=<field_name>` args can be passed to `/select/logsql/hits` for grouping hits buckets by the mentioned `<field_name>` fields.
|
||||
For example, the following query groups hits by `level` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) additionally to the provided `step`:
|
||||
|
||||
```logsql
|
||||
```sh
|
||||
curl http://localhost:9428/select/logsql/hits -d 'query=*' -d 'start=3h' -d 'step=1h' -d 'field=level'
|
||||
```
|
||||
|
||||
|
||||
14
docs/VictoriaLogs/querying/_index.md
Normal file
14
docs/VictoriaLogs/querying/_index.md
Normal file
@@ -0,0 +1,14 @@
|
||||
---
|
||||
sort: 4
|
||||
title: Querying
|
||||
weight: 4
|
||||
menu:
|
||||
docs:
|
||||
identifier: victorialogs-querying
|
||||
parent: "victorialogs"
|
||||
weight: 4
|
||||
aliases:
|
||||
- /VictoriaLogs/querying/
|
||||
- /VictoriaLogs/querying/index.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -1,16 +1,17 @@
|
||||
---
|
||||
weight: 9
|
||||
title: VictoriaLogs datasource for Grafana
|
||||
title: Grafana datasource
|
||||
editLink: https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/README.md
|
||||
menu:
|
||||
docs:
|
||||
identifier: victorialogs-grafana-datasource
|
||||
parent: victorialogs
|
||||
weight: 9
|
||||
aliases:
|
||||
- /victorialogs/victorialogs-datasource.html
|
||||
---
|
||||
|
||||
# VictoriaLogs datasource for Grafana
|
||||
|
||||
|
||||
|
||||
The VictoriaLogs datasource plugin allows you to query and visualize
|
||||
@@ -310,4 +311,4 @@ For more information on the fields in `plugin.json`, please refer to the [Grafan
|
||||
## License
|
||||
|
||||
This project is licensed under
|
||||
the [AGPL-3.0-only](https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/LICENSE).
|
||||
the [Apache 2.0 license](https://github.com/VictoriaMetrics/victorialogs-datasource/blob/main/LICENSE).
|
||||
|
||||
4
docs/_index.md
Normal file
4
docs/_index.md
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
title: VictoriaMetrics
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -10,73 +10,70 @@ menu:
|
||||
aliases:
|
||||
- /anomaly-detection/CHANGELOG.html
|
||||
---
|
||||
|
||||
# CHANGELOG
|
||||
|
||||
Please find the changelog for VictoriaMetrics Anomaly Detection below.
|
||||
|
||||
> **Important note: Users are strongly encouraged to upgrade to `vmanomaly` [v1.9.2](https://hub.docker.com/repository/docker/victoriametrics/vmanomaly/tags?page=1&ordering=name) or newer for optimal performance and accuracy. <br><br> This recommendation is crucial for configurations with a low `infer_every` parameter [in your scheduler](/anomaly-detection/components/scheduler/#parameters-1), and in scenarios where data exhibits significant high-order seasonality patterns (such as hourly or daily cycles). Previous versions from v1.5.1 to v1.8.0 were identified to contain a critical issue impacting model training, where models were inadvertently trained on limited data subsets, leading to suboptimal fits, affecting the accuracy of anomaly detection. <br><br> Upgrading to v1.9.2 addresses this issue, ensuring proper model training and enhanced reliability. For users utilizing Helm charts, it is recommended to upgrade to version [1.0.0](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/CHANGELOG.md#100) or newer.**
|
||||
> **Important note: Users are strongly encouraged to upgrade to `vmanomaly` [v1.9.2](https://hub.docker.com/repository/docker/victoriametrics/vmanomaly/tags?page=1&ordering=name) or newer for optimal performance and accuracy. <br><br> This recommendation is crucial for configurations with a low `infer_every` parameter [in your scheduler](./components/scheduler.md#parameters-1), and in scenarios where data exhibits significant high-order seasonality patterns (such as hourly or daily cycles). Previous versions from v1.5.1 to v1.8.0 were identified to contain a critical issue impacting model training, where models were inadvertently trained on limited data subsets, leading to suboptimal fits, affecting the accuracy of anomaly detection. <br><br> Upgrading to v1.9.2 addresses this issue, ensuring proper model training and enhanced reliability. For users utilizing Helm charts, it is recommended to upgrade to version [1.0.0](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/CHANGELOG.md#100) or newer.**
|
||||
|
||||
## v1.13.3
|
||||
Released: 2024-07-17
|
||||
- FIX: now validation of `args` argument for [`HoltWinters`](/anomaly-detection/components/models#holt-winters) model works properly.
|
||||
- FIX: now validation of `args` argument for [`HoltWinters`](./components/models.md#holt-winters) model works properly.
|
||||
|
||||
## v1.13.2
|
||||
Released: 2024-07-15
|
||||
- IMPROVEMENT: update `node-exporter` [preset](/anomaly-detection/presets/#node-exporter) to reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/index.html#false-positive)
|
||||
- FIX: add `verify_tls` arg for [`push`](/anomaly-detection/components/monitoring/#push-config-parameters) monitoring section. Also, `verify_tls` is now correctly used in [VmWriter](/anomaly-detection/components/writer/#vm-writer).
|
||||
- FIX: now [`AutoTuned`](/anomaly-detection/components/models/#autotuned) model wrapper works correctly in [on-disk model storage mode](/anomaly-detection/faq/#resource-consumption-of-vmanomaly).
|
||||
- FIX: now [rolling models](/anomaly-detection/components/models/#rolling-models), like [`RollingQuantile`](/anomaly-detection/components/models/#rolling-quantile) are properly handled in [One-off scheduler](/anomaly-detection/components/scheduler/#oneoff-scheduler), when wrapped in [`AutoTuned`](/anomaly-detection/components/models/#autotuned)
|
||||
- IMPROVEMENT: update `node-exporter` [preset](./Presets.md#node-exporter) to reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/index.html#false-positive)
|
||||
- FIX: add `verify_tls` arg for [`push`](./components/monitoring.md#push-config-parameters) monitoring section. Also, `verify_tls` is now correctly used in [VmWriter](./components/writer.md#vm-writer).
|
||||
- FIX: now [`AutoTuned`](./components/models.md#autotuned) model wrapper works correctly in [on-disk model storage mode](./FAQ.md#resource-consumption-of-vmanomaly).
|
||||
- FIX: now [rolling models](./components/models.md#rolling-models), like [`RollingQuantile`](./components/models.md#rolling-quantile) are properly handled in [One-off scheduler](./components/scheduler.md#oneoff-scheduler), when wrapped in [`AutoTuned`](./components/models.md#autotuned)
|
||||
|
||||
## v1.13.0
|
||||
Released: 2024-06-11
|
||||
- FEATURE: Introduced `preset` [mode to run vmanomaly service](/anomaly-detection/presets) with minimal user input and on widely-known metrics, like those produced by [`node_exporter`](/anomaly-detection/presets#node-exporter).
|
||||
- FEATURE: Introduced `min_dev_from_expected` [model common arg](/anomaly-detection/components/models/#minimal-deviation-from-expected), aimed at **reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** in scenarios where deviations between the real value `y` and the expected value `yhat` are **relatively** high and may cause models to generate high [anomaly scores](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score). However, these deviations are not significant enough in **absolute values** to be considered anomalies based on domain knowledge.
|
||||
- FEATURE: Introduced `detection_direction` [model common arg](/anomaly-detection/components/models/#detection-direction), enabling domain-driven anomaly detection strategies. Configure models to identify anomalies occurring *above, below, or in both directions* relative to the expected values.
|
||||
- FEATURE: add `n_jobs` arg to [`BacktestingScheduler`](/anomaly-detection/components/scheduler/#backtesting-scheduler) to allow *proportionally faster (yet more resource-intensive)* evaluations of a config on historical data. Default value is 1, that implies *sequential* execution.
|
||||
- FEATURE: allow anomaly detection models to be dumped to a host filesystem after `fit` stage (instead of in-memory). Resource-intensive setups (many models, many metrics, bigger [`fit_window` arg](/anomaly-detection/components/scheduler/#periodic-scheduler-config-example)) and/or 3rd-party models that store fit data (like [ProphetModel](/anomaly-detection/components/models/index.html#prophet) or [HoltWinters](/anomaly-detection/components/models/index.html#holt-winters)) will have RAM consumption greatly reduced at a cost of slightly slower `infer` stage. Please find how to enable it [here](/anomaly-detection/faq/#resource-consumption-of-vmanomaly)
|
||||
- IMPROVEMENT: Reduced the resource used for each fitted [`ProphetModel`](/anomaly-detection/components/models/index.html#prophet) by up to 6 times. This includes both RAM for in-memory models and disk space for on-disk models storage. For more details, refer to [this discussion on Facebook's Prophet](https://github.com/facebook/prophet/issues/1159#issuecomment-537415637).
|
||||
- IMPROVEMENT: now config [components](/anomaly-detection/components/index.html) class can be referenced by a short alias instead of a full class path - i.e. `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, `scheduler.periodic.PeriodicScheduler` becomes `periodic`, etc.
|
||||
- FIX: if using multi-scheduler setup (introduced in [v1.11.0](/anomaly-detection/changelog/#v1110)), prevent schedulers (and correspondent services) that are not attached to any model (so neither found in ['schedulers' arg](/anomaly-detection/components/models/index.html#schedulers) nor left blank in `model` section) from being spawn, causing resource overhead and slight interference with existing ones.
|
||||
- FIX: set random seed for [ProphetModel](/anomaly-detection/components/models#prophet) to assure uncertainty estimates (like `yhat_lower`, `yhat_upper`) and dependant series (like `anomaly_score`), produced during `.infer()` calls are always deterministic given the same input. See [initial issue](https://github.com/facebook/prophet/issues/1124) for the details.
|
||||
- FIX: prevent *orphan* queries (that are not attached to any model or scheduler) found in `queries` arg of [Reader config section](/anomaly-detection/components/reader/index.html#vm-reader) to be fetched from VictoriaMetrics TSDB, avoiding redundant data processing. A warning will be logged, if such queries exist in a parsed config.
|
||||
- FEATURE: Introduced `preset` [mode to run vmanomaly service](./Presets.md) with minimal user input and on widely-known metrics, like those produced by [`node_exporter`](./Presets.md#node-exporter).
|
||||
- FEATURE: Introduced `min_dev_from_expected` [model common arg](./components/models.md#minimal-deviation-from-expected), aimed at **reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** in scenarios where deviations between the real value `y` and the expected value `yhat` are **relatively** high and may cause models to generate high [anomaly scores](./FAQ.md#what-is-anomaly-score). However, these deviations are not significant enough in **absolute values** to be considered anomalies based on domain knowledge.
|
||||
- FEATURE: Introduced `detection_direction` [model common arg](./components/models.md#detection-direction), enabling domain-driven anomaly detection strategies. Configure models to identify anomalies occurring *above, below, or in both directions* relative to the expected values.
|
||||
- FEATURE: add `n_jobs` arg to [`BacktestingScheduler`](./components/scheduler.md#backtesting-scheduler) to allow *proportionally faster (yet more resource-intensive)* evaluations of a config on historical data. Default value is 1, that implies *sequential* execution.
|
||||
- FEATURE: allow anomaly detection models to be dumped to a host filesystem after `fit` stage (instead of in-memory). Resource-intensive setups (many models, many metrics, bigger [`fit_window` arg](./components/scheduler.md#periodic-scheduler-config-example)) and/or 3rd-party models that store fit data (like [ProphetModel](./components/models.md#prophet) or [HoltWinters](./components/models.md#holt-winters)) will have RAM consumption greatly reduced at a cost of slightly slower `infer` stage. Please find how to enable it [here](./FAQ.md#resource-consumption-of-vmanomaly)
|
||||
- IMPROVEMENT: Reduced the resource used for each fitted [`ProphetModel`](./components/models.md#prophet) by up to 6 times. This includes both RAM for in-memory models and disk space for on-disk models storage. For more details, refer to [this discussion on Facebook's Prophet](https://github.com/facebook/prophet/issues/1159#issuecomment-537415637).
|
||||
- IMPROVEMENT: now config [components](./components/README.md) class can be referenced by a short alias instead of a full class path - i.e. `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, `scheduler.periodic.PeriodicScheduler` becomes `periodic`, etc.
|
||||
- FIX: if using multi-scheduler setup (introduced in [v1.11.0](./CHANGELOG.md#v1110)), prevent schedulers (and correspondent services) that are not attached to any model (so neither found in ['schedulers' arg](./components/models.md#schedulers) nor left blank in `model` section) from being spawn, causing resource overhead and slight interference with existing ones.
|
||||
- FIX: set random seed for [ProphetModel](./components/models.md#prophet) to assure uncertainty estimates (like `yhat_lower`, `yhat_upper`) and dependant series (like `anomaly_score`), produced during `.infer()` calls are always deterministic given the same input. See [initial issue](https://github.com/facebook/prophet/issues/1124) for the details.
|
||||
- FIX: prevent *orphan* queries (that are not attached to any model or scheduler) found in `queries` arg of [Reader config section](./components/reader.md#vm-reader) to be fetched from VictoriaMetrics TSDB, avoiding redundant data processing. A warning will be logged, if such queries exist in a parsed config.
|
||||
|
||||
## v1.12.0
|
||||
Released: 2024-03-31
|
||||
- FEATURE: Introduction of `AutoTunedModel` model class to optimize any [built-in model](/anomaly-detection/components/models/#built-in-models) on data during `fit` phase. Specify as little as `anomaly_percentage` param from `(0, 0.5)` interval and `tuned_model_class` (i.e. [`model.zscore.ZscoreModel`](/anomaly-detection/components/models/#z-score)) to get it working with best settings that match your data. See details [here](/anomaly-detection/components/models/#autotuned).
|
||||
- FEATURE: Introduction of `AutoTunedModel` model class to optimize any [built-in model](./components/models.md#built-in-models) on data during `fit` phase. Specify as little as `anomaly_percentage` param from `(0, 0.5)` interval and `tuned_model_class` (i.e. [`model.zscore.ZscoreModel`](./components/models.md#z-score)) to get it working with best settings that match your data. See details [here](./components/models.md#autotuned).
|
||||
<!--
|
||||
- FEATURE: Preset support enablement. From now users will be able to specify only a few parameters (like `datasource_url`) + a new (backward-compatible) `preset: preset_name` field in a config file and get a service run with **predefined queries, scheduling and models**. Also, now preset assets (guide, configs, dashboards) will be available at `:8490/presets` endpoint.
|
||||
-->
|
||||
- IMPROVEMENT: Better logging of model lifecycle (fit/infer stages).
|
||||
- IMPROVEMENT: Introduce `provide_series` arg to all the [built-in models](/anomaly-detection/components/models/#built-in-models) to define what output fields to generate for writing (i.e. `provide_series: ['anomaly_score']` means only scores are being produced)
|
||||
- FIX: [Self-monitoring metrics](anomaly-detection/components/monitoring/#models-behaviour-metrics) are now aggregated to `queries` aliases level (not to label sets of individual timeseries) and aligned with [reader, writer and model sections](/anomaly-detection/components/monitoring/#metrics-generated-by-vmanomaly) description , so `/metrics` endpoint holds only necessary information for scraping.
|
||||
- FIX: Self-monitoring metric `vmanomaly_models_active` now has additional labels `model_alias`, `scheduler_alias`, `preset` to align with model-centric [self-monitoring](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#models-behaviour-metrics).
|
||||
- IMPROVEMENT: Add possibility to use temporal information in [IsolationForest models](/anomaly-detection/components/models/#isolation-forest-multivariate) via [cyclical encoding](https://towardsdatascience.com/cyclical-features-encoding-its-about-time-ce23581845ca). This is particularly helpful to detect multivariate [seasonality](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality)-dependant anomalies.
|
||||
- BREAKING CHANGE: **ARIMA** model is removed from [built-in models](/anomaly-detection/components/models/#built-in-models). For affected users, it is suggested to replace ARIMA by [Prophet](/anomaly-detection/components/models/#prophet) or [Holt-Winters](/anomaly-detection/components/models/#holt-winters).
|
||||
- IMPROVEMENT: Introduce `provide_series` arg to all the [built-in models](./components/models.md#built-in-models) to define what output fields to generate for writing (i.e. `provide_series: ['anomaly_score']` means only scores are being produced)
|
||||
- FIX: [Self-monitoring metrics](./components/monitoring.md#models-behaviour-metrics) are now aggregated to `queries` aliases level (not to label sets of individual timeseries) and aligned with [reader, writer and model sections](./components/monitoring.md#metrics-generated-by-vmanomaly) description , so `/metrics` endpoint holds only necessary information for scraping.
|
||||
- FIX: Self-monitoring metric `vmanomaly_models_active` now has additional labels `model_alias`, `scheduler_alias`, `preset` to align with model-centric [self-monitoring](./components/monitoring.md#models-behaviour-metrics).
|
||||
- IMPROVEMENT: Add possibility to use temporal information in [IsolationForest models](./components/models.md#isolation-forest-multivariate) via [cyclical encoding](https://towardsdatascience.com/cyclical-features-encoding-its-about-time-ce23581845ca). This is particularly helpful to detect multivariate [seasonality](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality)-dependant anomalies.
|
||||
- BREAKING CHANGE: **ARIMA** model is removed from [built-in models](./components/models.md#built-in-models). For affected users, it is suggested to replace ARIMA by [Prophet](./components/models.md#prophet) or [Holt-Winters](./components/models.md#holt-winters).
|
||||
|
||||
## v1.11.0
|
||||
Released: 2024-02-22
|
||||
- FEATURE: Multi-scheduler support. Now users can use multiple [model specs](https://docs.victoriametrics.com/anomaly-detection/components/models/) in a single config (via aliasing), each spec can be run with its own (even multiple) [schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/).
|
||||
- FEATURE: Multi-scheduler support. Now users can use multiple [model specs](./components/models.md) in a single config (via aliasing), each spec can be run with its own (even multiple) [schedulers](./components/scheduler.md).
|
||||
- Introduction of `schedulers` arg in model spec:
|
||||
- It allows each model to be managed by 1 (or more) schedulers, so overall resource usage is optimized and flexibility is preserved.
|
||||
- Passing an empty list or not specifying this param implies that each model is run in **all** the schedulers, which is a backward-compatible behavior.
|
||||
- Please find more details in docs on [Model section](https://docs.victoriametrics.com/anomaly-detection/components/models/#schedulers)
|
||||
- Please find more details in docs on [Model section](./components/models.md#schedulers)
|
||||
- DEPRECATION: slight refactor of a scheduler config section
|
||||
- Now schedulers are passed as a mapping of `scheduler_alias: scheduler_spec` under [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) sections. Using old format (< [1.11.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1110)) will produce warnings for now and will be removed in future versions.
|
||||
- Now schedulers are passed as a mapping of `scheduler_alias: scheduler_spec` under [scheduler](./components/scheduler.md) sections. Using old format (< [1.11.0](./CHANGELOG.md#v1110)) will produce warnings for now and will be removed in future versions.
|
||||
- DEPRECATION: The `--watch` CLI option for config file reloads is deprecated and will be ignored in the future.
|
||||
|
||||
## v1.10.0
|
||||
Released: 2024-02-15
|
||||
- FEATURE: Multi-model support. Now users can specify multiple [model specs](https://docs.victoriametrics.com/anomaly-detection/components/models/) in a single config (via aliasing), as well as to reference what [queries from VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/?highlight=queries#config-parameters) it should be run on.
|
||||
- FEATURE: Multi-model support. Now users can specify multiple [model specs](./components/models.md) in a single config (via aliasing), as well as to reference what [queries from VmReader](./components/reader.md#config-parameters) it should be run on.
|
||||
- Introduction of `queries` arg in model spec:
|
||||
- It allows the model to be executed only on a particular query subset from `reader` section.
|
||||
- Passing an empty list or not specifying this param implies that each model is run on results from **all** queries, which is a backward-compatible behavior.
|
||||
- Please find more details in docs on [Model section](https://docs.victoriametrics.com/anomaly-detection/components/models/#queries)
|
||||
- Please find more details in docs on [Model section](./components/models.md#queries)
|
||||
|
||||
- DEPRECATION: slight refactor of a model config section
|
||||
- Now models are passed as a mapping of `model_alias: model_spec` under [model](https://docs.victoriametrics.com/anomaly-detection/components/models/) sections. Using old format (<= [1.9.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v192)) will produce warnings for now and will be removed in future versions.
|
||||
- Please find more details in docs on [Model section](https://docs.victoriametrics.com/anomaly-detection/components/models/)
|
||||
- IMPROVEMENT: now logs from [`monitoring.pull`](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#monitoring-section-config-example) GET requests to `/metrics` endpoint are shown only in DEBUG mode
|
||||
- Now models are passed as a mapping of `model_alias: model_spec` under [model](./components/models.md) sections. Using old format (<= [1.9.2](./CHANGELOG.md#v192)) will produce warnings for now and will be removed in future versions.
|
||||
- Please find more details in docs on [Model section](./components/models.md)
|
||||
- IMPROVEMENT: now logs from [`monitoring.pull`](./components/monitoring.md#monitoring-section-config-example) GET requests to `/metrics` endpoint are shown only in DEBUG mode
|
||||
- IMPROVEMENT: labelset for multivariate models is deduplicated and cleaned, resulting in better UX
|
||||
|
||||
> **Note**: These updates support more flexible setup and effective resource management in service, as now it's not longer needed to spawn several instances of `vmanomaly` to split queries/models context across.
|
||||
@@ -84,7 +81,7 @@ Released: 2024-02-15
|
||||
|
||||
## v1.9.2
|
||||
Released: 2024-01-29
|
||||
- BUGFIX: now multivariate models (like [`IsolationForestMultivariateModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-foresthttpsenwikipediaorgwikiisolation_forest-multivariate)) are properly handled throughout fit/infer phases.
|
||||
- BUGFIX: now multivariate models (like [`IsolationForestMultivariateModel`](./components/models.md#isolation-foresthttpsenwikipediaorgwikiisolation_forest-multivariate)) are properly handled throughout fit/infer phases.
|
||||
|
||||
|
||||
## v1.9.1
|
||||
@@ -95,17 +92,17 @@ Released: 2024-01-27
|
||||
|
||||
## v1.9.0
|
||||
Released: 2024-01-26
|
||||
- BUGFIX: The `query_from_last_seen_timestamp` internal logic in [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader.html#vm-reader), first introduced in [v1.5.1](https://docs.victoriametrics.com/anomaly-detection/CHANGELOG.html#v151), now functions correctly. This fix ensures that the input data shape remains consistent for subsequent `fit`-based model calls in the service.
|
||||
- BREAKING CHANGE: The `sampling_period` parameter is now mandatory in [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader.html#vm-reader). This change aims to clarify and standardize the frequency of input/output in `vmanomaly`, thereby reducing uncertainty and aligning with user expectations.
|
||||
- BUGFIX: The `query_from_last_seen_timestamp` internal logic in [VmReader](./components/reader.md#vm-reader), first introduced in [v1.5.1](#v151), now functions correctly. This fix ensures that the input data shape remains consistent for subsequent `fit`-based model calls in the service.
|
||||
- BREAKING CHANGE: The `sampling_period` parameter is now mandatory in [VmReader](./components/reader.md#vm-reader). This change aims to clarify and standardize the frequency of input/output in `vmanomaly`, thereby reducing uncertainty and aligning with user expectations.
|
||||
> **Note**: The majority of users, who have been proactively specifying the `sampling_period` parameter in their configurations, will experience no disruption from this update. This transition formalizes a practice that was already prevalent and expected among our user base.
|
||||
|
||||
|
||||
## v1.8.0
|
||||
Released: 2024-01-15
|
||||
- FEATURE: Added Univariate [MAD (median absolute deviation)](/anomaly-detection/components/models.html#mad-median-absolute-deviation) model support.
|
||||
- FEATURE: Added Univariate [MAD (median absolute deviation)](./components/models.md#mad-median-absolute-deviation) model support.
|
||||
- IMPROVEMENT: Update Python to 3.12.1 and all the dependencies.
|
||||
- IMPROVEMENT: Don't check /health endpoint, check the real /query_range or /import endpoints directly. Users kept getting problems with /health.
|
||||
- DEPRECATION: "health_path" param is deprecated and doesn't do anything in config ([reader](/anomaly-detection/components/reader.html#vm-reader), [writer](/anomaly-detection/components/writer.html#vm-writer), [monitoring.push](/anomaly-detection/components/monitoring.html#push-config-parameters)).
|
||||
- DEPRECATION: "health_path" param is deprecated and doesn't do anything in config ([reader](./components/reader.md#vm-reader), [writer](./components/writer.md#vm-writer), [monitoring.push](./components/monitoring.md#push-config-parameters)).
|
||||
|
||||
|
||||
## v1.7.2
|
||||
@@ -203,4 +200,4 @@ Released: 2023-01-06
|
||||
|
||||
## v1.0.0-beta
|
||||
Released: 2022-12-08
|
||||
- First public release is available
|
||||
- First public release is available
|
||||
|
||||
@@ -10,68 +10,65 @@ menu:
|
||||
aliases:
|
||||
- /anomaly-detection/FAQ.html
|
||||
---
|
||||
|
||||
# FAQ - VictoriaMetrics Anomaly Detection
|
||||
|
||||
## What is VictoriaMetrics Anomaly Detection (vmanomaly)?
|
||||
VictoriaMetrics Anomaly Detection, also known as `vmanomaly`, is a service for detecting unexpected changes in time series data. Utilizing machine learning models, it computes and pushes back an ["anomaly score"](/anomaly-detection/components/models.html#vmanomaly-output) for user-specified metrics. This hands-off approach to anomaly detection reduces the need for manual alert setup and can adapt to various metrics, improving your observability experience.
|
||||
VictoriaMetrics Anomaly Detection, also known as `vmanomaly`, is a service for detecting unexpected changes in time series data. Utilizing machine learning models, it computes and pushes back an ["anomaly score"](./components/models.md#vmanomaly-output) for user-specified metrics. This hands-off approach to anomaly detection reduces the need for manual alert setup and can adapt to various metrics, improving your observability experience.
|
||||
|
||||
Please refer to [our guide section](/anomaly-detection/#practical-guides-and-installation) to find out more.
|
||||
Please refer to [our guide section](./README.md#practical-guides-and-installation) to find out more.
|
||||
|
||||
> **Note: `vmanomaly` is a part of [enterprise package](https://docs.victoriametrics.com/enterprise/). You need to get a [free trial license](https://victoriametrics.com/products/enterprise/trial/) for evaluation.**
|
||||
> **Note: `vmanomaly` is a part of [enterprise package](../enterprise.md). You need to get a [free trial license](https://victoriametrics.com/products/enterprise/trial/) for evaluation.**
|
||||
|
||||
## What is anomaly score?
|
||||
Among the metrics produced by `vmanomaly` (as detailed in [vmanomaly output metrics](/anomaly-detection/components/models.html#vmanomaly-output)), `anomaly_score` is a pivotal one. It is **a continuous score > 0**, calculated in such a way that **scores ranging from 0.0 to 1.0 usually represent normal data**, while **scores exceeding 1.0 are typically classified as anomalous**. However, it's important to note that the threshold for anomaly detection can be customized in the alert configuration settings.
|
||||
Among the metrics produced by `vmanomaly` (as detailed in [vmanomaly output metrics](./components/models.md#vmanomaly-output)), `anomaly_score` is a pivotal one. It is **a continuous score > 0**, calculated in such a way that **scores ranging from 0.0 to 1.0 usually represent normal data**, while **scores exceeding 1.0 are typically classified as anomalous**. However, it's important to note that the threshold for anomaly detection can be customized in the alert configuration settings.
|
||||
|
||||
The decision to set the changepoint at `1.0` is made to ensure consistency across various models and alerting configurations, such that a score above `1.0` consistently signifies an anomaly, thus, alerting rules are maintained more easily.
|
||||
|
||||
> Note: `anomaly_score` is a metric itself, which preserves all labels found in input data and (optionally) appends [custom labels, specified in writer](/anomaly-detection/components/writer.html#metrics-formatting) - follow the link for detailed output example.
|
||||
> Note: `anomaly_score` is a metric itself, which preserves all labels found in input data and (optionally) appends [custom labels, specified in writer](./components/writer.md#metrics-formatting) - follow the link for detailed output example.
|
||||
|
||||
## How is anomaly score calculated?
|
||||
For most of the [univariate models](/anomaly-detection/components/models/#univariate-models) that can generate `yhat`, `yhat_lower`, and `yhat_upper` time series in [their output](/anomaly-detection/components/models/#vmanomaly-output) (such as [Prophet](/anomaly-detection/components/models/#prophet) or [Z-score](/anomaly-detection/components/models/#z-score)), the anomaly score is calculated as follows:
|
||||
For most of the [univariate models](./components/models.md#univariate-models) that can generate `yhat`, `yhat_lower`, and `yhat_upper` time series in [their output](./components/models.md#vmanomaly-output) (such as [Prophet](./components/models.md#prophet) or [Z-score](./components/models.md#z-score)), the anomaly score is calculated as follows:
|
||||
- If `yhat` (expected series behavior) equals `y` (actual value observed), then the anomaly score is 0.
|
||||
- If `y` (actual value observed) falls within the `[yhat_lower, yhat_upper]` confidence interval, the anomaly score will gradually approach 1, the closer `y` is to the boundary.
|
||||
- If `y` (actual value observed) strictly exceeds the `[yhat_lower, yhat_upper]` interval, the anomaly score will be greater than 1, increasing as the margin between the actual value and the expected range grows.
|
||||
|
||||
Please see example graph illustrating this logic below:
|
||||
|
||||
<img alt="anomaly-score-calculation-example" src="vmanomaly-prophet-example.webp">
|
||||

|
||||
|
||||
|
||||
## How does vmanomaly work?
|
||||
`vmanomaly` applies built-in (or custom) [anomaly detection algorithms](/anomaly-detection/components/models.html), specified in a config file. Although a single config file supports one model, running multiple instances of `vmanomaly` with different configs is possible and encouraged for parallel processing or better support for your use case (i.e. simpler model for simple metrics, more sophisticated one for metrics with trends and seasonalities).
|
||||
`vmanomaly` applies built-in (or custom) [anomaly detection algorithms](./components/models.md), specified in a config file. Although a single config file supports one model, running multiple instances of `vmanomaly` with different configs is possible and encouraged for parallel processing or better support for your use case (i.e. simpler model for simple metrics, more sophisticated one for metrics with trends and seasonalities).
|
||||
|
||||
1. For more detailed information, please visit the [overview section](/anomaly-detection/Overview.html#about).
|
||||
2. To view a diagram illustrating the interaction of components, please explore the [components section](/anomaly-detection/components/).
|
||||
1. For more detailed information, please visit the [overview section](./Overview.md#about).
|
||||
2. To view a diagram illustrating the interaction of components, please explore the [components section](./components/README.md).
|
||||
|
||||
## What data does vmanomaly operate on?
|
||||
`vmanomaly` operates on data fetched from VictoriaMetrics, where you can leverage full power of [MetricsQL](https://docs.victoriametrics.com/metricsql/) for data selection, sampling, and processing. Users can also [apply global filters](https://docs.victoriametrics.com/#prometheus-querying-api-enhancements) for more targeted data analysis, enhancing scope limitation and tenant visibility.
|
||||
`vmanomaly` operates on data fetched from VictoriaMetrics, where you can leverage full power of [MetricsQL](../MetricsQL.md) for data selection, sampling, and processing. Users can also [apply global filters](../README.md#prometheus-querying-api-enhancements) for more targeted data analysis, enhancing scope limitation and tenant visibility.
|
||||
|
||||
Respective config is defined in a [`reader`](/anomaly-detection/components/reader.html#vm-reader) section.
|
||||
Respective config is defined in a [`reader`](./components/reader.md#vm-reader) section.
|
||||
|
||||
## Handling noisy input data
|
||||
`vmanomaly` operates on data fetched from VictoriaMetrics using [MetricsQL](https://docs.victoriametrics.com/metricsql/) queries, so the initial data quality can be fine-tuned with aggregation, grouping, and filtering to reduce noise and improve anomaly detection accuracy.
|
||||
`vmanomaly` operates on data fetched from VictoriaMetrics using [MetricsQL](../MetricsQL.md) queries, so the initial data quality can be fine-tuned with aggregation, grouping, and filtering to reduce noise and improve anomaly detection accuracy.
|
||||
|
||||
## Output produced by vmanomaly
|
||||
`vmanomaly` models generate [metrics](/anomaly-detection/components/models.html#vmanomaly-output) like `anomaly_score`, `yhat`, `yhat_lower`, `yhat_upper`, and `y`. These metrics provide a comprehensive view of the detected anomalies. The service also produces [health check metrics](/anomaly-detection/components/monitoring.html#metrics-generated-by-vmanomaly) for monitoring its performance.
|
||||
`vmanomaly` models generate [metrics](./components/models.md#vmanomaly-output) like `anomaly_score`, `yhat`, `yhat_lower`, `yhat_upper`, and `y`. These metrics provide a comprehensive view of the detected anomalies. The service also produces [health check metrics](./components/monitoring.md#metrics-generated-by-vmanomaly) for monitoring its performance.
|
||||
|
||||
## Choosing the right model for vmanomaly
|
||||
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](anomaly-detection/components/models.html#z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](anomaly-detection/components/models.html#prophet).
|
||||
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](./components/models.md#z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](./components/models.md#prophet).
|
||||
|
||||
Also, starting from [v1.12.0](/anomaly-detection/changelog/#v1120) it's possible to auto-tune the most important params of selected model class, find [the details here](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned).
|
||||
Also, starting from [v1.12.0](./CHANGELOG.md#v1120) it's possible to auto-tune the most important params of selected model class, find [the details here](./components/models.md#autotuned).
|
||||
|
||||
Please refer to [respective blogpost on anomaly types and alerting heuristics](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/) for more details.
|
||||
|
||||
Still not 100% sure what to use? We are [here to help](/anomaly-detection/#get-in-touch).
|
||||
Still not 100% sure what to use? We are [here to help](./README.md#get-in-touch).
|
||||
|
||||
## Alert generation in vmanomaly
|
||||
While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where an external alerting tool, like [`vmalert`](/vmalert.html), can be used to create alerts based on these scores for integrating it with your alerting management system.
|
||||
While `vmanomaly` detects anomalies and produces scores, it *does not directly generate alerts*. The anomaly scores are written back to VictoriaMetrics, where an external alerting tool, like [`vmalert`](../vmalert.md), can be used to create alerts based on these scores for integrating it with your alerting management system.
|
||||
|
||||
## Preventing alert fatigue
|
||||
Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](/vmalert.html) can use [MetricsQL](https://docs.victoriametrics.com/metricsql/) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
|
||||
Produced anomaly scores are designed in such a way that values from 0.0 to 1.0 indicate non-anomalous data, while a value greater than 1.0 is generally classified as an anomaly. However, there are no perfect models for anomaly detection, that's why reasonable defaults expressions like `anomaly_score > 1` may not work 100% of the time. However, anomaly scores, produced by `vmanomaly` are written back as metrics to VictoriaMetrics, where tools like [`vmalert`](../vmalert.md) can use [MetricsQL](../MetricsQL.md) expressions to fine-tune alerting thresholds and conditions, balancing between avoiding [false negatives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative) and reducing [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive).
|
||||
|
||||
## How to backtest particular configuration on historical data?
|
||||
Starting from [v1.7.2](/anomaly-detection/changelog/#v172) you can produce (and write back to VictoriaMetrics TSDB) anomaly scores for historical (backtesting) period, using `BacktestingScheduler` [component](/anomaly-detection/components/scheduler/#backtesting-scheduler) to imitate consecutive "production runs" of `PeriodicScheduler` [component](/anomaly-detection/components/scheduler/#periodic-scheduler). Please find an example config below:
|
||||
Starting from [v1.7.2](./CHANGELOG.md#v172) you can produce (and write back to VictoriaMetrics TSDB) anomaly scores for historical (backtesting) period, using `BacktestingScheduler` [component](./components/scheduler.md#backtesting-scheduler) to imitate consecutive "production runs" of `PeriodicScheduler` [component](./components/scheduler.md#periodic-scheduler). Please find an example config below:
|
||||
|
||||
```yaml
|
||||
schedulers:
|
||||
@@ -111,12 +108,12 @@ writer:
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/monitoring/
|
||||
```
|
||||
|
||||
Configuration above will produce N intervals of full length (`fit_window`=14d + `fit_every`=1h) until `to_iso` timestamp is reached to run N consecutive `fit` calls to train models; Then these models will be used to produce `M = [fit_every / sampling_frequency]` infer datapoints for `fit_every` range at the end of each such interval, imitating M consecutive calls of `infer_every` in `PeriodicScheduler` [config](/anomaly-detection/components/scheduler/#periodic-scheduler). These datapoints then will be written back to VictoriaMetrics TSDB, defined in `writer` [section](/anomaly-detection/components/writer/#vm-writer) for further visualization (i.e. in VMUI or Grafana)
|
||||
Configuration above will produce N intervals of full length (`fit_window`=14d + `fit_every`=1h) until `to_iso` timestamp is reached to run N consecutive `fit` calls to train models; Then these models will be used to produce `M = [fit_every / sampling_frequency]` infer datapoints for `fit_every` range at the end of each such interval, imitating M consecutive calls of `infer_every` in `PeriodicScheduler` [config](./components/scheduler.md#periodic-scheduler). These datapoints then will be written back to VictoriaMetrics TSDB, defined in `writer` [section](./components/writer.md#vm-writer) for further visualization (i.e. in VMUI or Grafana)
|
||||
|
||||
## Resource consumption of vmanomaly
|
||||
`vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](/anomaly-detection/components/scheduler.html) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](/anomaly-detection/components/reader.html#vm-reader), and the complexity of the employed [models](anomaly-detection/components/models.html). Its resource usage is directly related to these factors, making it adaptable to various operational scales.
|
||||
`vmanomaly` itself is a lightweight service, resource usage is primarily dependent on [scheduling](./components/scheduler.md) (how often and on what data to fit/infer your models), [# and size of timeseries returned by your queries](./components/reader.md#vm-reader), and the complexity of the employed [models](./components/models.md). Its resource usage is directly related to these factors, making it adaptable to various operational scales.
|
||||
|
||||
> **Note**: Starting from [v1.13.0](/anomaly-detection/changelog/#v1130), there is a mode to save anomaly detection models on host filesystem after `fit` stage (instead of keeping them in-memory by default). **Resource-intensive setups** (many models, many metrics, bigger [`fit_window` arg](/anomaly-detection/components/scheduler/#periodic-scheduler-config-example)) and/or 3rd-party models that store fit data (like [ProphetModel](/anomaly-detection/components/models/index.html#prophet) or [HoltWinters](/anomaly-detection/components/models/index.html#holt-winters)) will have RAM consumption greatly reduced at a cost of slightly slower `infer` stage. To enable it, you need to set environment variable `VMANOMALY_MODEL_DUMPS_DIR` to desired location. [Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md) are being updated accordingly ([`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) for persistent storage starting from chart version `1.3.0`).
|
||||
> **Note**: Starting from [v1.13.0](./CHANGELOG.md#v1130), there is a mode to save anomaly detection models on host filesystem after `fit` stage (instead of keeping them in-memory by default). **Resource-intensive setups** (many models, many metrics, bigger [`fit_window` arg](./components/scheduler.md#periodic-scheduler-config-example)) and/or 3rd-party models that store fit data (like [ProphetModel](./components/models.md#prophet) or [HoltWinters](./components/models.md#holt-winters)) will have RAM consumption greatly reduced at a cost of slightly slower `infer` stage. To enable it, you need to set environment variable `VMANOMALY_MODEL_DUMPS_DIR` to desired location. [Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md) are being updated accordingly ([`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) for persistent storage starting from chart version `1.3.0`).
|
||||
|
||||
Here's an example of how to set it up in docker-compose using volumes:
|
||||
```yaml
|
||||
@@ -150,13 +147,13 @@ volumes:
|
||||
## Scaling vmanomaly
|
||||
> **Note:** As of latest release we don't support cluster or auto-scaled version yet (though, it's in our roadmap for - better backends, more parallelization, etc.), so proposed workarounds should be addressed manually.
|
||||
|
||||
`vmanomaly` can be scaled horizontally by launching multiple independent instances, each with its own [MetricsQL](https://docs.victoriametrics.com/metricsql/) queries and [configurations](/anomaly-detection/components/):
|
||||
`vmanomaly` can be scaled horizontally by launching multiple independent instances, each with its own [MetricsQL](../MetricsQL.md) queries and [configurations](./components/README.md):
|
||||
|
||||
- By splitting **queries**, [defined in reader section](/anomaly-detection/components/reader/?highlight=queries#vm-reader) and spawn separate service around it. Also in case you have *only 1 query returning huge amount of timeseries*, you can further split it by applying MetricsQL filters, i.e. using "extra_filters" [param in reader](/anomaly-detection/components/reader/?highlight=extra_filters#vm-reader)
|
||||
- By splitting **queries**, [defined in reader section](./components/reader.md#vm-reader) and spawn separate service around it. Also in case you have *only 1 query returning huge amount of timeseries*, you can further split it by applying MetricsQL filters, i.e. using "extra_filters" [param in reader](./components/reader.md#vm-reader)
|
||||
|
||||
- or **models** (in case you decide to run several models for each timeseries received i.e. for averaging anomaly scores in your alerting rules of `vmalert` or using a vote approach to reduce false positives) - see `queries` arg in [model config](/anomaly-detection/components/models/#queries)
|
||||
- or **models** (in case you decide to run several models for each timeseries received i.e. for averaging anomaly scores in your alerting rules of `vmalert` or using a vote approach to reduce false positives) - see `queries` arg in [model config](./components/models.md#queries)
|
||||
|
||||
- or **schedulers** (in case you want the same models to be trained under several schedules) - see `schedulers` arg [model section](/anomaly-detection/components/models/#schedulers) and `scheduler` [component itself](/anomaly-detection/components/scheduler/)
|
||||
- or **schedulers** (in case you want the same models to be trained under several schedules) - see `schedulers` arg [model section](./components/models.md#schedulers) and `scheduler` [component itself](./components/scheduler.md)
|
||||
|
||||
|
||||
Here's an example of how to split on `extra_filters` param
|
||||
|
||||
@@ -11,10 +11,6 @@ aliases:
|
||||
- /anomaly-detection.html
|
||||
- /anomaly-detection/overview.html
|
||||
---
|
||||
|
||||
|
||||
# Overview
|
||||
|
||||
## About
|
||||
|
||||
**VictoriaMetrics Anomaly Detection** (or shortly, `vmanomaly`) is a service that continuously scans VictoriaMetrics time
|
||||
@@ -37,7 +33,7 @@ metrics.
|
||||
|
||||
`vmanomaly` can be used as a helper to set up your own alerting. You can rely on the spikes you see in anomaly scores to form the metric queries for alerting rules.
|
||||
|
||||
> **Note: `vmanomaly` is a part of [enterprise package](https://docs.victoriametrics.com/enterprise/). You need to get a [free trial license](https://victoriametrics.com/products/enterprise/trial/) for evaluation.**
|
||||
> **Note: `vmanomaly` is a part of [enterprise package](../enterprise.md). You need to get a [free trial license](https://victoriametrics.com/products/enterprise/trial/) for evaluation.**
|
||||
|
||||
## How?
|
||||
|
||||
@@ -53,9 +49,9 @@ processes in parallel, each using its own config.
|
||||
## Models
|
||||
|
||||
Currently, vmanomaly ships with a set of built-in models:
|
||||
> For a detailed overview, see [model section](/anomaly-detection/components/models.html)
|
||||
> For a detailed overview, see [model section](./components/models.md)
|
||||
|
||||
1. [**ZScore**](/anomaly-detection/components/models.html#z-score)
|
||||
1. [**ZScore**](./components/models.md#z-score)
|
||||
|
||||
_(useful for testing)_
|
||||
|
||||
@@ -63,7 +59,7 @@ Currently, vmanomaly ships with a set of built-in models:
|
||||
from time-series mean (straight line). Keeps only two model parameters internally:
|
||||
`mean` and `std` (standard deviation).
|
||||
|
||||
1. [**Prophet**](/anomaly-detection/components/models.html#prophet)
|
||||
1. [**Prophet**](./components/models.md#prophet)
|
||||
|
||||
_(simplest in configuration, recommended for getting started)_
|
||||
|
||||
@@ -77,32 +73,32 @@ Currently, vmanomaly ships with a set of built-in models:
|
||||
|
||||
See [Prophet documentation](https://facebook.github.io/prophet/)
|
||||
|
||||
1. [**Holt-Winters**](/anomaly-detection/components/models.html#holt-winters)
|
||||
1. [**Holt-Winters**](./components/models.md#holt-winters)
|
||||
|
||||
Very popular forecasting algorithm. See [statsmodels.org documentation](
|
||||
https://www.statsmodels.org/stable/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing.html)
|
||||
for Holt-Winters exponential smoothing.
|
||||
|
||||
1. [**Seasonal-Trend Decomposition**](/anomaly-detection/components/models.html#seasonal-trend-decomposition)
|
||||
1. [**Seasonal-Trend Decomposition**](./components/models.md#seasonal-trend-decomposition)
|
||||
|
||||
Extracts three components: season, trend, and residual, that can be plotted individually for
|
||||
easier debugging. Uses LOESS (locally estimated scatterplot smoothing).
|
||||
See [statsmodels.org documentation](https://www.statsmodels.org/dev/examples/notebooks/generated/stl_decomposition.html)
|
||||
for LOESS STD.
|
||||
|
||||
1. [**Rolling Quantile**](/anomaly-detection/components/models.html#rolling-quantile)
|
||||
1. [**Rolling Quantile**](./components/models.md#rolling-quantile)
|
||||
|
||||
A simple moving window of quantiles. Easy to use, easy to understand, but not as powerful as
|
||||
other models.
|
||||
|
||||
1. [**Isolation Forest**](/anomaly-detection/components/models.html#isolation-forest-multivariate)
|
||||
1. [**Isolation Forest**](./components/models.md#isolation-forest-multivariate)
|
||||
|
||||
Detects anomalies using binary trees. It works for both univariate and multivariate data. Be aware of [the curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality) in the case of multivariate data - we advise against using a single model when handling multiple time series *if the number of these series significantly exceeds their average length (# of data points)*.
|
||||
|
||||
The algorithm has a linear time complexity and a low memory requirement, which works well with high-volume data. See [scikit-learn.org documentation](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html) for Isolation Forest.
|
||||
|
||||
|
||||
1. [**MAD (Median Absolute Deviation)**](anomaly-detection/components/models.html#mad-median-absolute-deviation)
|
||||
1. [**MAD (Median Absolute Deviation)**](./components/models.md#mad-median-absolute-deviation)
|
||||
|
||||
A robust method for anomaly detection that is less sensitive to outliers in data compared to standard deviation-based models. It considers a point as an anomaly if the absolute deviation from the median is significantly large.
|
||||
|
||||
@@ -111,14 +107,14 @@ Currently, vmanomaly ships with a set of built-in models:
|
||||
For example, here’s how Prophet predictions could look like on a real-data example
|
||||
(Prophet auto-detected seasonality interval):
|
||||
|
||||
<img alt="propher-example" src="vmanomaly-prophet-example.webp">
|
||||

|
||||
|
||||
And here’s what Holt-Winters predictions real-world data could look like (seasonality manually
|
||||
set to 1 week). Notice that it predicts anomalies in
|
||||
different places than Prophet because the model noticed there are usually spikes on Friday
|
||||
morning, so it accounted for that:
|
||||
|
||||
<img alt="holtwinters-example" src="vmanomaly-holtwinters-example.webp">
|
||||

|
||||
|
||||
## Process
|
||||
Upon starting, vmanomaly queries the initial range of data, and trains its model (“fit” by convention).
|
||||
@@ -132,26 +128,26 @@ optionally preserving labels).
|
||||
|
||||
|
||||
## Usage
|
||||
> Starting from [v1.5.0](/anomaly-detection/CHANGELOG.html#v150), vmanomaly requires a license key to run. You can obtain a trial license key [here](https://victoriametrics.com/products/enterprise/trial/).
|
||||
> Starting from [v1.5.0](./CHANGELOG.md#v150), vmanomaly requires a license key to run. You can obtain a trial license key [here](https://victoriametrics.com/products/enterprise/trial/).
|
||||
|
||||
> See [Quickstart](/anomaly-detection/QuickStart.html).
|
||||
> See [Quickstart](./QuickStart.md).
|
||||
|
||||
> See [Integration guide: vmanomaly and vmalert](/anomaly-detection/guides/guide-vmanomaly-vmalert.html).
|
||||
> See [Integration guide: vmanomaly and vmalert](./guides/guide-vmanomaly-vmalert/README.md).
|
||||
|
||||
### Config file
|
||||
There are 4 required sections in config file:
|
||||
|
||||
* [`schedulers`](/anomaly-detection/components/scheduler.html) - defines how often to run and make inferences, as well as what timerange to use to train the model.
|
||||
* [`models`](/anomaly-detection/components/models.html) - specific model parameters and configurations.
|
||||
* [`reader`](/anomaly-detection/components/reader.html) - how to read data and where it is located
|
||||
* [`writer`](/anomaly-detection/components/writer.html) - where and how to write the generated output.
|
||||
* [`schedulers`](./components/scheduler.md) - defines how often to run and make inferences, as well as what timerange to use to train the model.
|
||||
* [`models`](./components/models.md) - specific model parameters and configurations.
|
||||
* [`reader`](./components/reader.md) - how to read data and where it is located
|
||||
* [`writer`](./components/writer.md) - where and how to write the generated output.
|
||||
|
||||
[`monitoring`](#monitoring) - defines how to monitor work of *vmanomaly* service. This config section is *optional*.
|
||||
|
||||
> For a detailed description, see [config sections](/anomaly-detection/components)
|
||||
> For a detailed description, see [config sections](./components/README.md)
|
||||
|
||||
#### Config example
|
||||
Here is an example of config file that will run [Facebook's Prophet model](/anomaly-detection/components/models.html#prophet), that will be retrained every 2 hours on 14 days of previous data. It will generate inference results (including `anomaly_score` metric) every 1 minute.
|
||||
Here is an example of config file that will run [Facebook's Prophet model](./components/models.md#prophet), that will be retrained every 2 hours on 14 days of previous data. It will generate inference results (including `anomaly_score` metric) every 1 minute.
|
||||
|
||||
|
||||
You need to specify your datasource urls to use it:
|
||||
@@ -183,7 +179,7 @@ writer:
|
||||
*vmanomaly* can be monitored by using push or pull approach.
|
||||
It can push metrics to VictoriaMetrics or expose metrics in Prometheus exposition format.
|
||||
|
||||
> For a detailed description, see [monitoring section](/anomaly-detection/components/monitoring.html)
|
||||
> For a detailed description, see [monitoring section](./components/monitoring.md)
|
||||
|
||||
#### Push approach
|
||||
|
||||
@@ -201,7 +197,7 @@ monitoring:
|
||||
#### Pull approach
|
||||
|
||||
*vmanomaly* can export internal metrics in Prometheus exposition format at `/metrics` page.
|
||||
These metrics can be scraped via [vmagent](https://docs.victoriametrics.com/vmagent/) or Prometheus.
|
||||
These metrics can be scraped via [vmagent](../vmagent.md) or Prometheus.
|
||||
|
||||
In order to enable pull approach, specify `pull` section in config file:
|
||||
|
||||
@@ -222,9 +218,9 @@ To use *vmanomaly* you need to pull docker image:
|
||||
docker pull victoriametrics/vmanomaly:latest
|
||||
```
|
||||
|
||||
> Note: please check what is latest release in [CHANGELOG](/anomaly-detection/CHANGELOG.html)
|
||||
> Note: please check what is latest release in [CHANGELOG](./CHANGELOG.md)
|
||||
|
||||
> Note: `us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial` is deprecated since [v1.6.0](/anomaly-detection/CHANGELOG.html#v160). Use [DockerHub repo](https://hub.docker.com/r/victoriametrics/vmanomaly/tags) instead
|
||||
> Note: `us-docker.pkg.dev/victoriametrics-test/public/vmanomaly-trial` is deprecated since [v1.6.0](./CHANGELOG.md#v160). Use [DockerHub repo](https://hub.docker.com/r/victoriametrics/vmanomaly/tags) instead
|
||||
|
||||
You can put a tag on it for your convenience:
|
||||
|
||||
@@ -263,7 +259,7 @@ The license key can be passed via the following command-line flags:
|
||||
In order to make it easier to monitor the license expiration date, the following metrics are exposed(see
|
||||
[Monitoring](#monitoring) section for details on how to scrape them):
|
||||
|
||||
```
|
||||
```promtextmetric
|
||||
# HELP vm_license_expires_at When the license expires as a Unix timestamp in seconds
|
||||
# TYPE vm_license_expires_at gauge
|
||||
vm_license_expires_at 1.6963776e+09
|
||||
@@ -272,7 +268,7 @@ vm_license_expires_at 1.6963776e+09
|
||||
vm_license_expires_in_seconds 4.886608e+06
|
||||
```
|
||||
|
||||
Example alerts for [vmalert](https://docs.victoriametrics.com/vmalert/):
|
||||
Example alerts for [vmalert](../vmalert.md):
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
|
||||
@@ -8,17 +8,16 @@ menu:
|
||||
weight: 1
|
||||
title: Presets
|
||||
---
|
||||
# Anomaly detection presets
|
||||
> Please check the [Quick Start Guide](/anomaly-detection/quickstart/) to install and run `vmanomaly`
|
||||
> Please check the [Quick Start Guide](./QuickStart.md) to install and run `vmanomaly`
|
||||
|
||||
> Presets are available starting from [v1.13.0](/anomaly-detection/CHANGELOG/#v1130)
|
||||
> Presets are available starting from [v1.13.0](./CHANGELOG.md#v1130)
|
||||
|
||||
|
||||
**Preset** mode allows for simpler configuration and anomaly detection with `vmanomaly` on widely-recognized metrics, such as those generated by [node_exporter](https://github.com/prometheus/node_exporter), which are typically challenging to monitor using static threshold-based alerting rules.
|
||||
|
||||
This approach represents a paradigm shift from traditional [static threshold-based alerting rules](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#rule-based-alerting), focused on *raw metric values*, to *static* rules based on [`anomaly_scores`](/anomaly-detection/faq/#what-is-anomaly-score). These scores offer a consistent, default threshold that remains stable over time, being adjusted for trends, seasonality, data scale, thus, reducing the engineering effort required for maintenance. Anomaly scores are produced by [machine learning models](/anomaly-detection/components/models), which are regularly retrained on varying time frames, ensuring alerts remain current and responsive to evolving data patterns.
|
||||
This approach represents a paradigm shift from traditional [static threshold-based alerting rules](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#rule-based-alerting), focused on *raw metric values*, to *static* rules based on [`anomaly_scores`](./FAQ.md#what-is-anomaly-score). These scores offer a consistent, default threshold that remains stable over time, being adjusted for trends, seasonality, data scale, thus, reducing the engineering effort required for maintenance. Anomaly scores are produced by [machine learning models](./components/models.md), which are regularly retrained on varying time frames, ensuring alerts remain current and responsive to evolving data patterns.
|
||||
|
||||
Additionally, **preset mode** minimizes user input needed to run the service. You can configure `vmanomaly` by specifying only the preset name and data sources in the [`reader`](/anomaly-detection/components/reader/) and [`writer`](/anomaly-detection/components/writer/) sections of the configuration file. All other parameters are already preconfigured.
|
||||
Additionally, **preset mode** minimizes user input needed to run the service. You can configure `vmanomaly` by specifying only the preset name and data sources in the [`reader`](./components/reader.md) and [`writer`](./components/writer.md) sections of the configuration file. All other parameters are already preconfigured.
|
||||
|
||||
|
||||
Available presets:
|
||||
@@ -35,11 +34,12 @@ writer:
|
||||
datasource_url: "http://victoriametrics:8428/" # your datasource url
|
||||
# tenant_id: '0:0' # specify for cluster version
|
||||
```
|
||||
Run a service using config file with one of the [available options](/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly).
|
||||
Run a service using config file with one of the [available options](./QuickStart.md#how-to-install-and-run-vmanomaly).
|
||||
|
||||
After you run `vmanomaly` with `preset` arg specified, available assets can be viewed, copied and downloaded at `http://localhost:8490/presets/` endpoint.
|
||||
|
||||
<img alt="preset-localhost" src="presets-localhost.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
## Node-Exporter
|
||||
|
||||
@@ -51,12 +51,12 @@ preset: "node-exporter"
|
||||
```
|
||||
|
||||
### Generated anomaly scores
|
||||
Machine learning models will be fit for each timeseries, returned by underlying [MetricsQL](https://docs.victoriametrics.com/metricsql/) queries.
|
||||
Anomaly score metric labels will also contain [model classes](/anomaly-detection/components/models/) and [schedulers](/anomaly-detection/components/scheduler/) for labelset uniqueness.
|
||||
Machine learning models will be fit for each timeseries, returned by underlying [MetricsQL](../MetricsQL.md) queries.
|
||||
Anomaly score metric labels will also contain [model classes](./components/models.md) and [schedulers](./components/scheduler.md) for labelset uniqueness.
|
||||
|
||||
Here's an example of produced metrics:
|
||||
|
||||
```shell
|
||||
```promtextmetric
|
||||
anomaly_score{for="cpu_seconds_total", instance="node-exporter:9100", preset="node-exporter", mode="system", model_alias="prophet", scheduler_alias="1d_1m"} 0.23451242720277776
|
||||
anomaly_score{for="cpu_seconds_total", instance="node-exporter:9100", preset="node-exporter", mode="user", model_alias="prophet", scheduler_alias="1d_1m"} 0.2637952255694444
|
||||
anomaly_score{for="page_faults", instance="node-exporter:9100", job="node-exporter", preset="node-exporter", model_alias="prophet", scheduler_alias="1d_1m"} 0.00593712535
|
||||
@@ -101,7 +101,7 @@ Grafana dashboard `.json` file can be found [here](https://github.com/VictoriaMe
|
||||
|
||||
The produced anomaly scores will have a label `for` containing the name of corresponding indicator.
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Indicator</th>
|
||||
@@ -111,34 +111,94 @@ The produced anomaly scores will have a label `for` containing the name of corre
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>page_faults</code></td>
|
||||
<td><code>node_vmstat_pgmajfault</code></td>
|
||||
<td>Number of major faults that have occurred since the last update. Major faults occur when a process tries to access a page in memory that is not currently mapped in the process's address space, and it requires loading data from the disk.</td>
|
||||
<td>
|
||||
|
||||
`page_faults`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`node_vmstat_pgmajfault`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Number of major faults that have occurred since the last update. Major faults occur when a process tries to access a page in memory that is not currently mapped in the process's address space, and it requires loading data from the disk.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>context_switch</code></td>
|
||||
<td><code>node_context_switches_total</code></td>
|
||||
<td>This metric represents the total number of context switches across all CPUs.</td>
|
||||
<td>
|
||||
|
||||
`context_switch`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`node_context_switches_total`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
This metric represents the total number of context switches across all CPUs.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>cpu_seconds_total</code></td>
|
||||
<td><code>node_cpu_seconds_total</code></td>
|
||||
<td>Total amount of CPU time consumed by the system in seconds by CPU processing mode (e.g., user, system, idle).</td>
|
||||
<td>
|
||||
|
||||
`cpu_seconds_total`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`node_cpu_seconds_total`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Total amount of CPU time consumed by the system in seconds by CPU processing mode (e.g., user, system, idle).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>host_network_receive_errors</code> & <code>host_network_transmit_errors</code></td>
|
||||
<td><code>node_network_receive_errs_total</code>, <code>node_network_receive_packets_total</code>, <code>node_network_transmit_errs_total</code>, <code>node_network_transmit_packets_total</code>
|
||||
<td>Total number of errors encountered while receiving/transmitting packets on the network interfaces of a node.</td>
|
||||
<td>
|
||||
|
||||
`host_network_receive_errors` & `host_network_transmit_errors`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`node_network_receive_errs_total`,
|
||||
`node_network_receive_packets_total`,
|
||||
`node_network_transmit_errs_total`,
|
||||
`node_network_transmit_packets_total`
|
||||
<td>
|
||||
|
||||
Total number of errors encountered while receiving/transmitting packets on the network interfaces of a node.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>receive_bytes</code> & <code>transmit_bytes</code></td>
|
||||
<td><code>node_network_receive_bytes_total</code>, <code>node_network_transmit_bytes_total</code></td>
|
||||
<td>Total number of bytes received/transmitted on network interfaces of a node.</td>
|
||||
<td>
|
||||
|
||||
`receive_bytes` & `transmit_bytes`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`node_network_receive_bytes_total`,
|
||||
`node_network_transmit_bytes_total`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Total number of bytes received/transmitted on network interfaces of a node.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>read_latency</code> & <code>write_latency</code></td>
|
||||
<td><code>node_disk_read_time_seconds_total</code>, <code>node_disk_reads_completed_total</code>, <code>node_disk_write_time_seconds_total</code>, <code>node_disk_writes_completed_total</code></td>
|
||||
<td>Disk latency. The total read/write time spent in seconds. / The total number of reads/writes completed successfully.</td>
|
||||
<td>
|
||||
|
||||
`read_latency` & `write_latency`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`node_disk_read_time_seconds_total`,
|
||||
`node_disk_reads_completed_total`,
|
||||
`node_disk_write_time_seconds_total`,
|
||||
`node_disk_writes_completed_total`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Disk latency. The total read/write time spent in seconds. / The total number of reads/writes completed successfully.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -149,21 +209,26 @@ Here's how attached [Grafana dashboard](https://github.com/VictoriaMetrics/Victo
|
||||
|
||||
On the (global) graph **'Percentage of Anomalies'**, you can see a spike 8.75% of anomalies at the timestamp '2024-06-03 10:35:00'. The (global) graph **'Anomalies per Indicator'** shows the indicators that were anomalous at the corresponding time.
|
||||
|
||||
<img alt="global" src="presets_global_percentage.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
At this timestamp on the **'Number of Anomalous Indicators by Node'** graph we can identify the node that had the most anomalies: `10.142.0.27`
|
||||
|
||||
<img alt="by_node" src="presets_anomalies_by_node.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
Now you can select anomalous node to drill down further (local):
|
||||
|
||||
<img alt="anomalous_node_selection" src="presets_anomalous_node_selection.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
For this node from the timestamp `2024-06-03 10:35:00` CPU time spent handling software interrupts started to grow.
|
||||
(`cpu_seconds_total{mode="softirq"}`)
|
||||
|
||||
<img alt="irq" src="presets_cpu_seconds_softirq.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
At the same time `cpu_seconds_total` for `steal` mode started to grow as well.
|
||||
|
||||
<img alt="steal" src="presets_cpu_seconds_steal.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
@@ -10,11 +10,8 @@ menu:
|
||||
aliases:
|
||||
- /anomaly-detection/QuickStart.html
|
||||
---
|
||||
|
||||
# VictoriaMetrics Anomaly Detection Quick Start
|
||||
|
||||
For service introduction visit [README](/anomaly-detection/) page
|
||||
and [Overview](/anomaly-detection/overview.html) of how `vmanomaly` works.
|
||||
For service introduction visit [README](./README.md) page
|
||||
and [Overview](./Overview.md) of how `vmanomaly` works.
|
||||
|
||||
## How to install and run vmanomaly
|
||||
|
||||
@@ -25,7 +22,7 @@ The following options are available:
|
||||
- [To run Docker image](#docker)
|
||||
- [To run in Kubernetes with Helm charts](#kubernetes-with-helm-charts)
|
||||
|
||||
> **Note**: Starting from [v1.13.0](/anomaly-detection/changelog/#v1130) there is a mode to keep anomaly detection models on host filesystem after `fit` stage (instead of keeping them in-memory by default); This may lead to **noticeable reduction of RAM used** on bigger setups. See instructions [here](/anomaly-detection/faq/#resource-consumption-of-vmanomaly).
|
||||
> **Note**: Starting from [v1.13.0](./CHANGELOG.md#v1130) there is a mode to keep anomaly detection models on host filesystem after `fit` stage (instead of keeping them in-memory by default); This may lead to **noticeable reduction of RAM used** on bigger setups. See instructions [here](./FAQ.md#resource-consumption-of-vmanomaly).
|
||||
|
||||
### Docker
|
||||
|
||||
@@ -84,13 +81,13 @@ services:
|
||||
# ...
|
||||
```
|
||||
|
||||
For a complete docker-compose example please refer to [our alerting guide](/anomaly-detection/guides/guide-vmanomaly-vmalert/), chapter [docker-compose](/anomaly-detection/guides/guide-vmanomaly-vmalert/#docker-compose)
|
||||
For a complete docker-compose example please refer to [our alerting guide](./guides/guide-vmanomaly-vmalert/README.md), chapter [docker-compose](./guides/guide-vmanomaly-vmalert/README.md#docker-compose)
|
||||
|
||||
|
||||
|
||||
See also:
|
||||
|
||||
- Verify the license online OR offline. See the details [here](/anomaly-detection/overview/#licensing).
|
||||
- Verify the license online OR offline. See the details [here](./Overview.md#licensing).
|
||||
- [How to configure `vmanomaly`](#how-to-configure-vmanomaly)
|
||||
|
||||
### Kubernetes with Helm charts
|
||||
@@ -138,19 +135,19 @@ writer:
|
||||
|
||||
|
||||
Next steps:
|
||||
- Define how often to run and make inferences in the [scheduler](/anomaly-detection/components/scheduler/) section of a config file.
|
||||
- Setup the datasource to read data from in the [reader](/anomaly-detection/components/reader/) section.
|
||||
- Specify where and how to store anomaly detection metrics in the [writer](/anomaly-detection/components/writer/) section.
|
||||
- Configure built-in models parameters according to your needs in the [models](/anomaly-detection/components/models/) section.
|
||||
- Integrate your [custom models](/anomaly-detection/components/models/#custom-model-guide) with `vmanomaly`.
|
||||
- Define queries for input data using [MetricsQL](https://docs.victoriametrics.com/metricsql/).
|
||||
- Define how often to run and make inferences in the [scheduler](./components/scheduler.md) section of a config file.
|
||||
- Setup the datasource to read data from in the [reader](./components/reader.md) section.
|
||||
- Specify where and how to store anomaly detection metrics in the [writer](./components/writer.md) section.
|
||||
- Configure built-in models parameters according to your needs in the [models](./components/models.md) section.
|
||||
- Integrate your [custom models](./components/models.md#custom-model-guide) with `vmanomaly`.
|
||||
- Define queries for input data using [MetricsQL](../../MetricsQL.md).
|
||||
|
||||
|
||||
## Check also
|
||||
|
||||
Here are other materials that you might find useful:
|
||||
|
||||
- [Guide: Anomaly Detection and Alerting Setup](/anomaly-detection/guides/guide-vmanomaly-vmalert/)
|
||||
- [FAQ](/anomaly-detection/faq/)
|
||||
- [Changelog](/anomaly-detection/changelog/)
|
||||
- [Anomaly Detection Blog](https://victoriametrics.com/blog/tags/anomaly-detection/)
|
||||
- [Guide: Anomaly Detection and Alerting Setup](./guides/guide-vmanomaly-vmalert/README.md)
|
||||
- [FAQ](./FAQ.md)
|
||||
- [Changelog](./CHANGELOG.md)
|
||||
- [Anomaly Detection Blog](https://victoriametrics.com/blog/tags/anomaly-detection/)
|
||||
|
||||
@@ -1,40 +1,28 @@
|
||||
---
|
||||
# sort: 14
|
||||
title: VictoriaMetrics Anomaly Detection
|
||||
weight: 0
|
||||
aliases:
|
||||
- /anomaly-detection.html
|
||||
- /vmanomaly.html
|
||||
- /vmanomaly/
|
||||
---
|
||||
|
||||
# VictoriaMetrics Anomaly Detection
|
||||
|
||||
In the dynamic and complex world of system monitoring, VictoriaMetrics Anomaly Detection, being a part of our [Enterprise offering](https://victoriametrics.com/products/enterprise/), stands as a pivotal tool for achieving advanced observability. It empowers SREs and DevOps teams by automating the intricate task of identifying abnormal behavior in time-series data. It goes beyond traditional threshold-based alerting, utilizing machine learning techniques to not only detect anomalies but also minimize false positives, thus reducing alert fatigue. By providing simplified alerting mechanisms atop of [unified anomaly scores](/anomaly-detection/components/models.html#vmanomaly-output), it enables teams to spot and address potential issues faster, ensuring system reliability and operational efficiency.
|
||||
In the dynamic and complex world of system monitoring, VictoriaMetrics Anomaly Detection, being a part of our [Enterprise offering](https://victoriametrics.com/products/enterprise/), stands as a pivotal tool for achieving advanced observability. It empowers SREs and DevOps teams by automating the intricate task of identifying abnormal behavior in time-series data. It goes beyond traditional threshold-based alerting, utilizing machine learning techniques to not only detect anomalies but also minimize false positives, thus reducing alert fatigue. By providing simplified alerting mechanisms atop of [unified anomaly scores](./components/models.md#vmanomaly-output), it enables teams to spot and address potential issues faster, ensuring system reliability and operational efficiency.
|
||||
|
||||
## Practical Guides and Installation
|
||||
Begin your VictoriaMetrics Anomaly Detection journey with ease using our guides and installation instructions:
|
||||
|
||||
- **Quickstart**: Check out how to get `vmanomaly` up and running [here](/anomaly-detection/QuickStart.html).
|
||||
- **Overview**: Find out how `vmanomaly` service operates [here](/anomaly-detection/Overview.html)
|
||||
- **Integration**: Integrate anomaly detection into your observability ecosystem. Get started [here](/anomaly-detection/guides/guide-vmanomaly-vmalert.html).
|
||||
- **Anomaly Detection Presets**: Enable anomaly detection on predefined set of indicators, that require frequently changing static thresholds for alerting. Find more information [here](/anomaly-detection/presets/).
|
||||
- **Quickstart**: Check out how to get `vmanomaly` up and running [here](./QuickStart.md).
|
||||
- **Overview**: Find out how `vmanomaly` service operates [here](./Overview.md)
|
||||
- **Integration**: Integrate anomaly detection into your observability ecosystem. Get started [here](./guides/guide-vmanomaly-vmalert/README.md).
|
||||
- **Anomaly Detection Presets**: Enable anomaly detection on predefined set of indicators, that require frequently changing static thresholds for alerting. Find more information [here](./Presets.md).
|
||||
|
||||
- **Installation Options**: Select the method that aligns with your technical requirements:
|
||||
- **Docker Installation**: Suitable for containerized environments. See [Docker guide](/anomaly-detection/Overview.html#run-vmanomaly-docker-container).
|
||||
- **Docker Installation**: Suitable for containerized environments. See [Docker guide](./Overview.md#run-vmanomaly-docker-container).
|
||||
- **Helm Chart Installation**: Appropriate for those using Kubernetes. See our [Helm charts](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-anomaly).
|
||||
|
||||
|
||||
> **Note**: starting from [v1.5.0](./CHANGELOG.md#v150) `vmanomaly` requires a [license key](/anomaly-detection/Overview.html#licensing) to run. You can obtain a trial license key [**here**](https://victoriametrics.com/products/enterprise/trial/).
|
||||
> **Note**: starting from [v1.5.0](./CHANGELOG.md#v150) `vmanomaly` requires a [license key](./Overview.md#licensing) to run. You can obtain a trial license key [**here**](https://victoriametrics.com/products/enterprise/trial/).
|
||||
|
||||
## Key Components
|
||||
Explore the integral components that configure VictoriaMetrics Anomaly Detection:
|
||||
* [Explore components and their interation](/anomaly-detection/components)
|
||||
- [Models](/anomaly-detection/components/models)
|
||||
- [Reader](/anomaly-detection/components/reader)
|
||||
- [Scheduler](/anomaly-detection/components/scheduler)
|
||||
- [Writer](/anomaly-detection/components/writer)
|
||||
- [Monitoring](/anomaly-detection/components/monitoring)
|
||||
* [Explore components and their interation](./components/README.md)
|
||||
- [Models](./components/models.md)
|
||||
- [Reader](./components/reader.md)
|
||||
- [Scheduler](./components/scheduler.md)
|
||||
- [Writer](./components/writer.md)
|
||||
- [Monitoring](./components/monitoring.md)
|
||||
|
||||
## Deep Dive into Anomaly Detection
|
||||
Enhance your knowledge with our handbook on Anomaly Detection & Root Cause Analysis and stay updated:
|
||||
@@ -47,7 +35,7 @@ Enhance your knowledge with our handbook on Anomaly Detection & Root Cause Analy
|
||||
## Frequently Asked Questions (FAQ)
|
||||
Got questions about VictoriaMetrics Anomaly Detection? Chances are, we've got the answers ready for you.
|
||||
|
||||
Dive into [our FAQ section](/anomaly-detection/FAQ) to find responses to common questions.
|
||||
Dive into [our FAQ section](./FAQ.md) to find responses to common questions.
|
||||
|
||||
## Get in Touch
|
||||
We are eager to connect with you and adapt our solutions to your specific needs. Here's how you can engage with us:
|
||||
@@ -55,4 +43,4 @@ We are eager to connect with you and adapt our solutions to your specific needs.
|
||||
* Interested in exploring our [Enterprise features](https://victoriametrics.com/products/enterprise), including [Anomaly Detection](https://victoriametrics.com/products/enterprise/anomaly-detection)? [Request your trial license](https://victoriametrics.com/products/enterprise/trial/) today and take the first step towards advanced system observability.
|
||||
|
||||
---
|
||||
Our [CHANGELOG is just a click away](./CHANGELOG.md), keeping you informed about the latest updates and enhancements.
|
||||
Our [CHANGELOG is just a click away](./CHANGELOG.md), keeping you informed about the latest updates and enhancements.
|
||||
|
||||
10
docs/anomaly-detection/_index.md
Normal file
10
docs/anomaly-detection/_index.md
Normal file
@@ -0,0 +1,10 @@
|
||||
---
|
||||
# sort: 14
|
||||
title: VictoriaMetrics Anomaly Detection
|
||||
weight: 0
|
||||
aliases:
|
||||
- /anomaly-detection.html
|
||||
- /vmanomaly.html
|
||||
- /vmanomaly/
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -1,39 +1,24 @@
|
||||
---
|
||||
title: Components
|
||||
weight: 2
|
||||
menu:
|
||||
docs:
|
||||
identifier: "vmanomaly-components"
|
||||
parent: "anomaly-detection"
|
||||
weight: 2
|
||||
aliases:
|
||||
- /anomaly-detection/components/
|
||||
- /anomaly-detection/components/index.html
|
||||
---
|
||||
This chapter describes different components, that correspond to respective sections of a config to launch VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](../Overview.md)) service:
|
||||
|
||||
# Components
|
||||
- [Model(s) section](./models.md) - Required
|
||||
- [Reader section](./reader.md) - Required
|
||||
- [Scheduler(s) section](./scheduler.md) - Required
|
||||
- [Writer section](./writer.md) - Required
|
||||
- [Monitoring section](./monitoring.md) - Optional
|
||||
|
||||
This chapter describes different components, that correspond to respective sections of a config to launch VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](/anomaly-detection/overview.html)) service:
|
||||
> **Note**: starting from [v1.7.0](../CHANGELOG.md#v172), once the service starts, automated config validation is performed. Please see container logs for errors that need to be fixed to create fully valid config, visiting sections above for examples and documentation.
|
||||
|
||||
- [Model(s) section](models.html) - Required
|
||||
- [Reader section](reader.html) - Required
|
||||
- [Scheduler(s) section](scheduler.html) - Required
|
||||
- [Writer section](writer.html) - Required
|
||||
- [Monitoring section](monitoring.html) - Optional
|
||||
> **Note**: starting from [v1.13.0](../CHANGELOG.md#v1130), components' class can be referenced by a short alias instead of a full class path - i.e. `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, `scheduler.periodic.PeriodicScheduler` becomes `periodic`, etc. Please see according sections for the details.
|
||||
|
||||
> **Note**: starting from [v1.7.0](/anomaly-detection/CHANGELOG#v172), once the service starts, automated config validation is performed. Please see container logs for errors that need to be fixed to create fully valid config, visiting sections above for examples and documentation.
|
||||
|
||||
> **Note**: starting from [v1.13.0](/anomaly-detection/CHANGELOG#v1130), components' class can be referenced by a short alias instead of a full class path - i.e. `model.zscore.ZscoreModel` becomes `zscore`, `reader.vm.VmReader` becomes `vm`, `scheduler.periodic.PeriodicScheduler` becomes `periodic`, etc. Please see according sections for the details.
|
||||
|
||||
> **Note:** Starting from [v1.13.0](/anomaly-detection/CHANGELOG#v1130) `preset` modes are available for `vmanomaly`. Please find the guide [here](/anomaly-detection/presets/).
|
||||
> **Note:** Starting from [v1.13.0](../CHANGELOG.md#v1130) `preset` modes are available for `vmanomaly`. Please find the guide [here](../Presets.md).
|
||||
|
||||
Below, you will find an example illustrating how the components of `vmanomaly` interact with each other and with a single-node VictoriaMetrics setup.
|
||||
|
||||
> **Note**: [Reader](/anomaly-detection/components/reader.html#vm-reader) and [Writer](/anomaly-detection/components/writer.html#vm-writer) also support [multitenancy](/Cluster-VictoriaMetrics.html#multitenancy), so you can read/write from/to different locations - see `tenant_id` param description.
|
||||
> **Note**: [Reader](./reader.md#vm-reader) and [Writer](./writer.md#vm-writer) also support [multitenancy](../../Cluster-VictoriaMetrics.md#multitenancy), so you can read/write from/to different locations - see `tenant_id` param description.
|
||||
|
||||
<img alt="vmanomaly-components" src="vmanomaly-components.webp" width="800px"/>
|
||||

|
||||
|
||||
Here's a minimalistic full config example, demonstrating many-to-many configuration (actual for [latest version](/anomaly-detection/CHANGELOG/)):
|
||||
Here's a minimalistic full config example, demonstrating many-to-many configuration (actual for [latest version](../CHANGELOG.md)):
|
||||
|
||||
```yaml
|
||||
# how and when to run the models is defined by schedulers
|
||||
@@ -89,4 +74,4 @@ monitoring:
|
||||
pull: # Enable /metrics endpoint.
|
||||
addr: "0.0.0.0"
|
||||
port: 8490
|
||||
```
|
||||
```
|
||||
|
||||
13
docs/anomaly-detection/components/_index.md
Normal file
13
docs/anomaly-detection/components/_index.md
Normal file
@@ -0,0 +1,13 @@
|
||||
---
|
||||
title: Components
|
||||
weight: 2
|
||||
menu:
|
||||
docs:
|
||||
identifier: "vmanomaly-components"
|
||||
parent: "anomaly-detection"
|
||||
weight: 2
|
||||
aliases:
|
||||
- /anomaly-detection/components/
|
||||
- /anomaly-detection/components/index.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -13,15 +13,12 @@ aliases:
|
||||
- /anomaly-detection/components/models/custom_model.html
|
||||
- /anomaly-detection/components/models/models.html
|
||||
---
|
||||
|
||||
# Models
|
||||
|
||||
This section describes `Models` component of VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](/anomaly-detection/overview/)) and the guide of how to define a respective section of a config to launch the service.
|
||||
This section describes `Models` component of VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](../Overview.md)) and the guide of how to define a respective section of a config to launch the service.
|
||||
- `vmanomaly` includes various [built-in models](#built-in-models).
|
||||
- you can also integrate your custom model - see [custom model](#custom-model-guide).
|
||||
|
||||
|
||||
> **Note: Starting from [v1.10.0](/anomaly-detection/changelog#v1100) model section in config supports multiple models via aliasing. <br>Also, `vmanomaly` expects model section to be named `models`. Using old (flat) format with `model` key is deprecated and will be removed in future versions. Having `model` and `models` sections simultaneously in a config will result in only `models` being used:**
|
||||
> **Note: Starting from [v1.10.0](../CHANGELOG.md#v1100) model section in config supports multiple models via aliasing. <br>Also, `vmanomaly` expects model section to be named `models`. Using old (flat) format with `model` key is deprecated and will be removed in future versions. Having `model` and `models` sections simultaneously in a config will result in only `models` being used:**
|
||||
|
||||
```yaml
|
||||
models:
|
||||
@@ -40,7 +37,7 @@ models:
|
||||
# ...
|
||||
```
|
||||
|
||||
Old-style configs (< [1.10.0](/anomaly-detection/changelog#v1100))
|
||||
Old-style configs (< [1.10.0](../CHANGELOG.md#v1100))
|
||||
|
||||
```yaml
|
||||
model:
|
||||
@@ -65,11 +62,11 @@ models:
|
||||
|
||||
## Common args
|
||||
|
||||
From [1.10.0](/anomaly-detection/changelog#1100), **common args**, supported by *every model (and model type)* were introduced.
|
||||
From [1.10.0](../CHANGELOG.md#1100), **common args**, supported by *every model (and model type)* were introduced.
|
||||
|
||||
### Queries
|
||||
|
||||
Introduced in [1.10.0](/anomaly-detection/changelog#1100), as a part to support multi-model configs, `queries` arg is meant to define [queries from VmReader](/anomaly-detection/components/reader/?highlight=queries#config-parameters) particular model should be run on (meaning, all the series returned by each of these queries will be used in such model for fitting and inferencing).
|
||||
Introduced in [1.10.0](../CHANGELOG.md#1100), as a part to support multi-model configs, `queries` arg is meant to define [queries from VmReader](./reader.md#config-parameters) particular model should be run on (meaning, all the series returned by each of these queries will be used in such model for fitting and inferencing).
|
||||
|
||||
`queries` arg is supported for all [the built-in](#built-in-models) (as well as for [custom](#custom-model-guide)) models.
|
||||
|
||||
@@ -94,7 +91,7 @@ models:
|
||||
|
||||
### Schedulers
|
||||
|
||||
Introduced in [1.11.0](/anomaly-detection/changelog#1110), as a part to support multi-scheduler configs, `schedulers` arg is meant to define [schedulers](/anomaly-detection/components/scheduler) particular model should be attached to.
|
||||
Introduced in [1.11.0](../CHANGELOG.md#1110), as a part to support multi-scheduler configs, `schedulers` arg is meant to define [schedulers](./scheduler.md) particular model should be attached to.
|
||||
|
||||
`schedulers` arg is supported for all [the built-in](#built-in-models) (as well as for [custom](#custom-model-guide)) models.
|
||||
|
||||
@@ -119,7 +116,7 @@ models:
|
||||
|
||||
### Provide series
|
||||
|
||||
Introduced in [1.12.0](/anomaly-detection/changelog#1120), `provide_series` arg limit the [output generated](#vmanomaly-output) by `vmanomaly` for writing. I.e. if the model produces default output series `['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']` by specifying `provide_series` section as below, you limit the data being written to only `['anomaly_score']` for each metric received as a subject to anomaly detection.
|
||||
Introduced in [1.12.0](../CHANGELOG.md#1120), `provide_series` arg limit the [output generated](#vmanomaly-output) by `vmanomaly` for writing. I.e. if the model produces default output series `['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']` by specifying `provide_series` section as below, you limit the data being written to only `['anomaly_score']` for each metric received as a subject to anomaly detection.
|
||||
|
||||
```yaml
|
||||
models:
|
||||
@@ -131,23 +128,26 @@ models:
|
||||
**Note** If `provide_series` is not specified in model config, the model will produce its default [model-dependent output](#vmanomaly-output). The output can't be less than `['anomaly_score']`. Even if `timestamp` column is omitted, it will be implicitly added to `provide_series` list, as it's required for metrics to be properly written.
|
||||
|
||||
### Detection direction
|
||||
Introduced in [1.13.0](/anomaly-detection/CHANGELOG/#1130), `detection_direction` arg can help in reducing the number of [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/index.html#false-positive) and increasing the accuracy, when domain knowledge suggest to identify anomalies occurring when actual values (`y`) are *above, below, or in both directions* relative to the expected values (`yhat`). Available choices are: `both`, `above_expected`, `below_expected`.
|
||||
Introduced in [1.13.0](../CHANGELOG.md#1130), `detection_direction` arg can help in reducing the number of [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/index.html#false-positive) and increasing the accuracy, when domain knowledge suggest to identify anomalies occurring when actual values (`y`) are *above, below, or in both directions* relative to the expected values (`yhat`). Available choices are: `both`, `above_expected`, `below_expected`.
|
||||
|
||||
Here's how default (backward-compatible) behavior looks like - anomalies will be tracked in `both` directions (`y > yhat` or `y < yhat`). This is useful when there is no domain expertise to filter the required direction.
|
||||
|
||||
<img src="/anomaly-detection/components/schema_detection_direction=both.webp" width="800px" alt="schema_detection_direction=both"/>
|
||||

|
||||
|
||||
|
||||
When set to `above_expected`, anomalies are tracked only when `y > yhat`.
|
||||
|
||||
*Example metrics*: Error rate, response time, page load time, number of failed transactions - metrics where *lower values are better*, so **higher** values are typically tracked.
|
||||
|
||||
<img src="/anomaly-detection/components/schema_detection_direction=above_expected.webp" width="800px" alt="schema_detection_direction=above_expected"/>
|
||||

|
||||
|
||||
|
||||
When set to `below_expected`, anomalies are tracked only when `y < yhat`.
|
||||
|
||||
*Example metrics*: Service Level Agreement (SLA) compliance, conversion rate, Customer Satisfaction Score (CSAT) - metrics where *higher values are better*, so **lower** values are typically tracked.
|
||||
|
||||
<img src="/anomaly-detection/components/schema_detection_direction=below_expected.webp" width="800px" alt="schema_detection_direction=below_expected"/>
|
||||

|
||||
|
||||
|
||||
Config with a split example:
|
||||
|
||||
@@ -185,7 +185,7 @@ reader:
|
||||
|
||||
### Minimal deviation from expected
|
||||
|
||||
Introduced in [v1.13.0](/anomaly-detection/CHANGELOG/#1130), the `min_dev_from_expected` argument is designed to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** in scenarios where deviations between the actual value (`y`) and the expected value (`yhat`) are **relatively** high. Such deviations can cause models to generate high [anomaly scores](/anomaly-detection/faq/#what-is-anomaly-score). However, these deviations may not be significant enough in **absolute values** from a business perspective to be considered anomalies. This parameter ensures that anomaly scores for data points where `|y - yhat| < min_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it behaves as `min_dev_from_expected=0` to maintain backward compatibility.
|
||||
Introduced in [v1.13.0](../CHANGELOG.md#1130), the `min_dev_from_expected` argument is designed to **reduce [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive)** in scenarios where deviations between the actual value (`y`) and the expected value (`yhat`) are **relatively** high. Such deviations can cause models to generate high [anomaly scores](../FAQ.md#what-is-anomaly-score). However, these deviations may not be significant enough in **absolute values** from a business perspective to be considered anomalies. This parameter ensures that anomaly scores for data points where `|y - yhat| < min_dev_from_expected` are explicitly set to 0. By default, if this parameter is not set, it behaves as `min_dev_from_expected=0` to maintain backward compatibility.
|
||||
|
||||
> **Note**: `min_dev_from_expected` must be >= 0. The higher the value of `min_dev_from_expected`, the fewer data points will be available for anomaly detection, and vice versa.
|
||||
|
||||
@@ -193,11 +193,13 @@ Introduced in [v1.13.0](/anomaly-detection/CHANGELOG/#1130), the `min_dev_from_e
|
||||
|
||||
Visualizations below demonstrate this concept; the green zone defined as the `[yhat - min_dev_from_expected, yhat + min_dev_from_expected]` range excludes actual data points (`y`) from generating anomaly scores if they fall within that range.
|
||||
|
||||
<img src="/anomaly-detection/components/schema_min_dev_from_expected=0.webp" width="800px" alt="min_dev_from_expected-default"/>
|
||||

|
||||
|
||||
<img src="/anomaly-detection/components/schema_min_dev_from_expected=1.0.webp" width="800px" alt="min_dev_from_expected-small"/>
|
||||
|
||||
<img src="/anomaly-detection/components/schema_min_dev_from_expected=5.0.webp" width="800px" alt="min_dev_from_expected-big"/>
|
||||

|
||||
|
||||
|
||||

|
||||
|
||||
Example config of how to use this param based on query results:
|
||||
|
||||
@@ -238,7 +240,7 @@ Each of these models can also be
|
||||
|
||||
For a univariate type, **one separate model** is fit/used for inference per **each time series**, defined in its [queries](#queries) arg.
|
||||
|
||||
For example, if you have some **univariate** model, defined to use 3 [MetricQL queries](https://docs.victoriametrics.com/metricsql/), each returning 5 time series, there will be 3*5=15 models created in total. Each such model produce **individual [output](#vmanomaly-output)** for each of time series.
|
||||
For example, if you have some **univariate** model, defined to use 3 [MetricQL queries](../../MetricsQL.md), each returning 5 time series, there will be 3*5=15 models created in total. Each such model produce **individual [output](#vmanomaly-output)** for each of time series.
|
||||
|
||||
If during an inference, you got a series having **new labelset** (not present in any of fitted models), the inference will be skipped until you get a model, trained particularly for such labelset during forthcoming re-fit step.
|
||||
|
||||
@@ -246,14 +248,14 @@ If during an inference, you got a series having **new labelset** (not present in
|
||||
|
||||
**Examples:** [Prophet](#prophet), [Holt-Winters](#holt-winters)
|
||||
|
||||
<p></p>
|
||||
<img alt="vmanomaly-model-type-univariate" src="/anomaly-detection/components/model-lifecycle-univariate.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
### Multivariate Models
|
||||
|
||||
For a multivariate type, **one shared model** is fit/used for inference on **all time series** simultaneously, defined in its [queries](#queries) arg.
|
||||
|
||||
For example, if you have some **multivariate** model to use 3 [MetricQL queries](https://docs.victoriametrics.com/metricsql/), each returning 5 time series, there will be one shared model created in total. Once fit, this model will expect **exactly 15 time series with exact same labelsets as an input**. This model will produce **one shared [output](#vmanomaly-output)**.
|
||||
For example, if you have some **multivariate** model to use 3 [MetricQL queries](../../MetricsQL.md), each returning 5 time series, there will be one shared model created in total. Once fit, this model will expect **exactly 15 time series with exact same labelsets as an input**. This model will produce **one shared [output](#vmanomaly-output)**.
|
||||
|
||||
If during an inference, you got a **different amount of series** or some series having a **new labelset** (not present in any of fitted models), the inference will be skipped until you get a model, trained particularly for such labelset during forthcoming re-fit step.
|
||||
|
||||
@@ -261,8 +263,8 @@ If during an inference, you got a **different amount of series** or some series
|
||||
|
||||
**Examples:** [IsolationForest](#isolation-forest-multivariate)
|
||||
|
||||
<p></p>
|
||||
<img alt="vmanomaly-model-type-multivariate" src="/anomaly-detection/components/model-lifecycle-multivariate.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
### Rolling Models
|
||||
|
||||
@@ -270,16 +272,16 @@ A rolling model is a model that, once trained, **cannot be (naturally) used to m
|
||||
|
||||
An instance of rolling model is **simultaneously fit and used for inference** during its `infer` method call.
|
||||
|
||||
As a result, such model instances are **not stored** between consecutive re-fit calls (defined by `fit_every` [arg](/anomaly-detection/components/scheduler/?highlight=fit_every#periodic-scheduler) in `PeriodicScheduler`), leading to **lower RAM** consumption.
|
||||
As a result, such model instances are **not stored** between consecutive re-fit calls (defined by `fit_every` [arg](./scheduler.md#periodic-scheduler) in `PeriodicScheduler`), leading to **lower RAM** consumption.
|
||||
|
||||
Such models put **more pressure** on your reader's source, i.e. if your model should be fit on large amount of data (say, 14 days with 1-minute resolution) and at the same time you have **frequent inference** (say, once per minute) on new chunks of data - that's because such models require (fit + infer) window of data to be fit first to be used later in each inference call.
|
||||
|
||||
> **Note**: Rolling models require `fit_every` to be set equal to `infer_every` in your [PeriodicScheduler](/anomaly-detection/components/scheduler/?highlight=fit_every#periodic-scheduler).
|
||||
> **Note**: Rolling models require `fit_every` to be set equal to `infer_every` in your [PeriodicScheduler](./scheduler.md#periodic-scheduler).
|
||||
|
||||
**Examples:** [RollingQuantile](#rolling-quantile)
|
||||
|
||||
<p></p>
|
||||
<img alt="vmanomaly-model-type-rolling" src="/anomaly-detection/components/model-type-rolling.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
### Non-Rolling Models
|
||||
|
||||
@@ -291,12 +293,12 @@ Such models put **less pressure** on your reader's source, i.e. if you fit on la
|
||||
|
||||
> **Note**: However, it's still highly recommended, to keep your model up-to-date with tendencies found in your data as it evolves in time.
|
||||
|
||||
Produced model instances are **stored in-memory** between consecutive re-fit calls (defined by `fit_every` [arg](/anomaly-detection/components/scheduler/?highlight=fit_every#periodic-scheduler) in `PeriodicScheduler`), leading to **higher RAM** consumption.
|
||||
Produced model instances are **stored in-memory** between consecutive re-fit calls (defined by `fit_every` [arg](./scheduler#periodic-scheduler) in `PeriodicScheduler`), leading to **higher RAM** consumption.
|
||||
|
||||
**Examples:** [Prophet](#prophet)
|
||||
|
||||
<p></p>
|
||||
<img alt="vmanomaly-model-type-non-rolling" src="/anomaly-detection/components/model-type-non-rolling.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
## Built-in Models
|
||||
|
||||
@@ -322,12 +324,12 @@ VM Anomaly Detection (`vmanomaly` hereinafter) models support 2 groups of parame
|
||||
|
||||
|
||||
### AutoTuned
|
||||
Tuning hyperparameters of a model can be tricky and often requires in-depth knowledge of Machine Learning. `AutoTunedModel` is designed specifically to take the cognitive load off the user - specify as little as `anomaly_percentage` param from `(0, 0.5)` interval and `tuned_model_class` (i.e. [`model.zscore.ZscoreModel`](/anomaly-detection/components/models/#z-score)) to get it working with best settings that match your data.
|
||||
Tuning hyperparameters of a model can be tricky and often requires in-depth knowledge of Machine Learning. `AutoTunedModel` is designed specifically to take the cognitive load off the user - specify as little as `anomaly_percentage` param from `(0, 0.5)` interval and `tuned_model_class` (i.e. [`model.zscore.ZscoreModel`](./models.md#z-score)) to get it working with best settings that match your data.
|
||||
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.auto.AutoTunedModel"` (or `auto` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `tuned_class_name` (string) - Built-in model class to tune, i.e. `model.zscore.ZscoreModel` (or `zscore` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support).
|
||||
* `class` (string) - model class name `"model.auto.AutoTunedModel"` (or `auto` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
* `tuned_class_name` (string) - Built-in model class to tune, i.e. `model.zscore.ZscoreModel` (or `zscore` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support).
|
||||
* `optimization_params` (dict) - Optimization parameters for unsupervised model tuning. Control % of found anomalies, as well as a tradeoff between time spent and the accuracy. The more `timeout` and `n_trials` are, the better model configuration can be found for `tuned_class_name`, but the longer it takes and vice versa. Set `n_jobs` to `-1` to use all the CPUs available, it makes sense if only you have a big dataset to train on during `fit` calls, otherwise overhead isn't worth it.
|
||||
- `anomaly_percentage` (float) - expected percentage of anomalies that can be seen in training data, from (0, 0.5) interval.
|
||||
- `seed` (int) - Random seed for reproducibility and deterministic nature of underlying optimizations.
|
||||
@@ -335,7 +337,8 @@ Tuning hyperparameters of a model can be tricky and often requires in-depth know
|
||||
- `n_trials` (int) - How many trials to sample from hyperparameter search space. The higher, the longer it takes but the better the results can be. Defaults to 128.
|
||||
- `timeout` (float) - How many seconds in total can be spent on each model to tune hyperparameters. The higher, the longer it takes, allowing to test more trials out of defined `n_trials`, but the better the results can be.
|
||||
|
||||
<img alt="vmanomaly-autotune-schema" src="/anomaly-detection/components/autotune.webp" width="800px"/>
|
||||

|
||||
|
||||
|
||||
```yaml
|
||||
# ...
|
||||
@@ -356,7 +359,7 @@ models:
|
||||
> **Note**: There are some expected limitations of Autotune mode:
|
||||
> - It can't be made on your [custom model](#custom-model-guide).
|
||||
> - It can't be applied to itself (like `tuned_class_name: 'model.auto.AutoTunedModel'`)
|
||||
> - `AutoTunedModel` can't be used on [rolling models](/anomaly-detection/components/models/#rolling-models) like [`RollingQuantile`](/anomaly-detection/components/models/#rolling-quantile) in combination with [on-disk model storage mode](/anomaly-detection/faq/#resource-consumption-of-vmanomaly), as the rolling models exists only during `infer` calls and aren't persisted neither in RAM, nor on disk.
|
||||
> - `AutoTunedModel` can't be used on [rolling models](./models.md#rolling-models) like [`RollingQuantile`](./models.md#rolling-quantile) in combination with [on-disk model storage mode](../FAQ.md#resource-consumption-of-vmanomaly), as the rolling models exists only during `infer` calls and aren't persisted neither in RAM, nor on disk.
|
||||
|
||||
|
||||
### [Prophet](https://facebook.github.io/prophet/)
|
||||
@@ -364,7 +367,7 @@ Here we utilize the Facebook Prophet implementation, as detailed in their [libra
|
||||
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.prophet.ProphetModel"` (or `prophet` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `class` (string) - model class name `"model.prophet.ProphetModel"` (or `prophet` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
* `seasonalities` (list[dict], optional) - Extra seasonalities to pass to Prophet. See [`add_seasonality()`](https://facebook.github.io/prophet/docs/seasonality,_holiday_effects,_and_regressors.html#modeling-holidays-and-special-events:~:text=modeling%20the%20cycle-,Specifying,-Custom%20Seasonalities) Prophet param.
|
||||
|
||||
**Note**: Apart from standard `vmanomaly` output, Prophet model can provide [additional metrics](#additional-output-metrics-produced-by-fb-prophet).
|
||||
@@ -404,7 +407,7 @@ Resulting metrics of the model are described [here](#vmanomaly-output)
|
||||
### [Z-score](https://en.wikipedia.org/wiki/Standard_score)
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.zscore.ZscoreModel"` (or `zscore` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `class` (string) - model class name `"model.zscore.ZscoreModel"` (or `zscore` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
* `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculation boundaries and anomaly score. Defaults to `2.5`.
|
||||
|
||||
*Config Example*
|
||||
@@ -425,7 +428,7 @@ Here we use Holt-Winters Exponential Smoothing implementation from `statsmodels`
|
||||
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.holtwinters.HoltWinters"` (or `holtwinters` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `class` (string) - model class name `"model.holtwinters.HoltWinters"` (or `holtwinters` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
|
||||
* `frequency` (string) - Must be set equal to sampling_period. Model needs to know expected data-points frequency (e.g. '10m'). If omitted, frequency is guessed during fitting as **the median of intervals between fitting data timestamps**. During inference, if incoming data doesn't have the same frequency, then it will be interpolated. E.g. data comes at 15 seconds resolution, and our resample_freq is '1m'. Then fitting data will be downsampled to '1m' and internal model is trained at '1m' intervals. So, during inference, prediction data would be produced at '1m' intervals, but interpolated to "15s" to match with expected output, as output data must have the same timestamps. As accepted by pandas.Timedelta (e.g. '5m').
|
||||
|
||||
@@ -467,7 +470,7 @@ The MAD model is a robust method for anomaly detection that is *less sensitive*
|
||||
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.mad.MADModel"` (or `mad` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `class` (string) - model class name `"model.mad.MADModel"` (or `mad` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
* `threshold` (float, optional) - The threshold multiplier for the MAD to determine anomalies. Defaults to `2.5`. Higher values will identify fewer points as anomalies.
|
||||
|
||||
*Config Example*
|
||||
@@ -487,7 +490,7 @@ Resulting metrics of the model are described [here](#vmanomaly-output).
|
||||
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.rolling_quantile.RollingQuantileModel"` (or `rolling_quantile` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `class` (string) - model class name `"model.rolling_quantile.RollingQuantileModel"` (or `rolling_quantile` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
* `quantile` (float) - quantile value, from 0.5 to 1.0. This constraint is implied by 2-sided confidence interval.
|
||||
* `window_steps` (integer) - size of the moving window. (see 'sampling_period')
|
||||
|
||||
@@ -508,7 +511,7 @@ Here we use Seasonal Decompose implementation from `statsmodels` [library](https
|
||||
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.std.StdModel"` (or `std` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `class` (string) - model class name `"model.std.StdModel"` (or `std` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
* `period` (integer) - Number of datapoints in one season.
|
||||
* `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculating boundaries to define anomaly score. Defaults to `2.5`.
|
||||
|
||||
@@ -541,11 +544,11 @@ Here we use Isolation Forest implementation from `scikit-learn` [library](https:
|
||||
|
||||
*Parameters specific for vmanomaly*:
|
||||
|
||||
* `class` (string) - model class name `"model.isolation_forest.IsolationForestMultivariateModel"` (or `isolation_forest_multivariate` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support)
|
||||
* `class` (string) - model class name `"model.isolation_forest.IsolationForestMultivariateModel"` (or `isolation_forest_multivariate` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support)
|
||||
|
||||
* `contamination` (float or string, optional) - The amount of contamination of the data set, i.e. the proportion of outliers in the data set. Used when fitting to define the threshold on the scores of the samples. Default value - "auto". Should be either `"auto"` or be in the range (0.0, 0.5].
|
||||
|
||||
* `seasonal_features` (list of string) - List of seasonality to encode through [cyclical encoding](https://towardsdatascience.com/cyclical-features-encoding-its-about-time-ce23581845ca), i.e. `dow` (day of week). **Introduced in [1.12.0](/anomaly-detection/CHANGELOG/#v1120)**.
|
||||
* `seasonal_features` (list of string) - List of seasonality to encode through [cyclical encoding](https://towardsdatascience.com/cyclical-features-encoding-its-about-time-ce23581845ca), i.e. `dow` (day of week). **Introduced in [1.12.0](../CHANGELOG.md#v1120)**.
|
||||
- Empty by default for backward compatibility.
|
||||
- Example: `seasonal_features: ['dow', 'hod']`.
|
||||
- Supported seasonalities:
|
||||
@@ -601,12 +604,12 @@ The default metrics produced by `vmanomaly` include:
|
||||
|
||||
## vmanomaly monitoring metrics
|
||||
|
||||
Each model exposes [several monitoring metrics](/anomaly-detection/components/monitoring/#models-behaviour-metrics) to its `health_path` endpoint:
|
||||
Each model exposes [several monitoring metrics](./monitoring.md#models-behaviour-metrics) to its `health_path` endpoint:
|
||||
|
||||
|
||||
## Custom Model Guide
|
||||
|
||||
Apart from `vmanomaly` [built-in models](/anomaly-detection/components/models/#built-in-models), users can create their own custom models for anomaly detection.
|
||||
Apart from `vmanomaly` [built-in models](./models.md#built-in-models), users can create their own custom models for anomaly detection.
|
||||
|
||||
Here in this guide, we will
|
||||
- Make a file containing our custom model definition
|
||||
@@ -698,8 +701,8 @@ class CustomModel(Model):
|
||||
### 2. Configuration file
|
||||
|
||||
Next, we need to create `config.yaml` file with `vmanomaly` configuration and model input parameters.
|
||||
In the config file's `models` section we need to set our model class to `model.custom.CustomModel` (or `custom` starting from [v1.13.0](/anomaly-detection/CHANGELOG/#1130) with class alias support) and define all parameters used in `__init__` method.
|
||||
You can find out more about configuration parameters in `vmanomaly` [config docs](/anomaly-detection/components/).
|
||||
In the config file's `models` section we need to set our model class to `model.custom.CustomModel` (or `custom` starting from [v1.13.0](../CHANGELOG.md#1130) with class alias support) and define all parameters used in `__init__` method.
|
||||
You can find out more about configuration parameters in `vmanomaly` [config docs](./README.md).
|
||||
|
||||
```yaml
|
||||
schedulers:
|
||||
@@ -762,7 +765,7 @@ victoriametrics/vmanomaly:latest /config.yaml \
|
||||
--license-file=/license
|
||||
```
|
||||
|
||||
Please find more detailed instructions (license, etc.) [here](/anomaly-detection/overview.html#run-vmanomaly-docker-container)
|
||||
Please find more detailed instructions (license, etc.) [here](../Overview.md#run-vmanomaly-docker-container)
|
||||
|
||||
|
||||
### Output
|
||||
|
||||
@@ -8,16 +8,13 @@ menu:
|
||||
weight: 5
|
||||
identifier: "vmanomaly-monitoring"
|
||||
aliases:
|
||||
- /anomaly-detection/components/monitoring.html
|
||||
- ./monitoring.html
|
||||
---
|
||||
|
||||
# Monitoring
|
||||
|
||||
There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push](https://docs.victoriametrics.com/keyconcepts/#push-model) and [pull](https://docs.victoriametrics.com/keyconcepts/#pull-model). Parameters for each of them should be specified in the config file, `monitoring` section.
|
||||
There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push](../../keyConcepts.md#push-model) and [pull](../../keyConcepts.md#pull-model). Parameters for each of them should be specified in the config file, `monitoring` section.
|
||||
|
||||
## Pull Model Config parameters
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
@@ -27,13 +24,25 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>addr</code></td>
|
||||
<td><code>"0.0.0.0"</code></td>
|
||||
<td>
|
||||
|
||||
`addr`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`"0.0.0.0"`
|
||||
</td>
|
||||
<td>Server IP Address</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>port</code></td>
|
||||
<td><code>8080</code></td>
|
||||
<td>
|
||||
|
||||
`port`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`8080`
|
||||
</td>
|
||||
<td>Port</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
@@ -41,7 +50,7 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
|
||||
|
||||
## Push Config parameters
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
@@ -51,42 +60,84 @@ There are 2 models to monitor VictoriaMetrics Anomaly Detection behavior - [push
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>url</code></td>
|
||||
<td>
|
||||
|
||||
`url`
|
||||
</td>
|
||||
<td></td>
|
||||
<td>Link where to push metrics to. Example: <code>"http://localhost:8480/"</code></td>
|
||||
<td>
|
||||
|
||||
Link where to push metrics to. Example: `"http://localhost:8480/"`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>tenant_id</code></td>
|
||||
<td>
|
||||
|
||||
`tenant_id`
|
||||
</td>
|
||||
<td></td>
|
||||
<td>Tenant ID for cluster version. Example: <code>"0:0"</code></td>
|
||||
<td>
|
||||
|
||||
Tenant ID for cluster version. Example: `"0:0"`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>health_path</code></td>
|
||||
<td><code>"health"</code></td>
|
||||
<td>Deprecated since <a href="https://docs.victoriametrics.com/anomaly-detection/CHANGELOG.html#v180">v1.8.0</a>. Absolute, to override <code>/health</code> path</td>
|
||||
<td>
|
||||
|
||||
`health_path`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`"health"`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Deprecated since [v1.8.0](../CHANGELOG.md#v180). Absolute, to override `/health` path
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>user</code></td>
|
||||
<td>
|
||||
|
||||
`user`
|
||||
</td>
|
||||
<td></td>
|
||||
<td>BasicAuth username</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>password</code></td>
|
||||
<td>
|
||||
|
||||
`password`
|
||||
</td>
|
||||
<td></td>
|
||||
<td>BasicAuth password</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>verify_tls</code></td>
|
||||
<td><code>False</code></td>
|
||||
<td>
|
||||
|
||||
`verify_tls`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`False`
|
||||
</td>
|
||||
<td>Allows disabling TLS verification of the remote certificate.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>timeout</code></td>
|
||||
<td><code>"5s"</code></td>
|
||||
<td>
|
||||
|
||||
`timeout`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`"5s"`
|
||||
</td>
|
||||
<td>Stop waiting for a response after a given number of seconds.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>extra_labels</code></td>
|
||||
<td>
|
||||
|
||||
`extra_labels`
|
||||
</td>
|
||||
<td></td>
|
||||
<td>Section for custom labels specified by user.</td>
|
||||
</tr>
|
||||
@@ -114,7 +165,7 @@ monitoring:
|
||||
|
||||
## Metrics generated by vmanomaly
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Metric</th>
|
||||
@@ -124,7 +175,10 @@ monitoring:
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>vmanomaly_start_time_seconds</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_start_time_seconds`
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>vmanomaly start time in UNIX time</td>
|
||||
</tr>
|
||||
@@ -134,9 +188,9 @@ monitoring:
|
||||
### Models Behaviour Metrics
|
||||
Label names [description](#labelnames)
|
||||
|
||||
> **Note**: There is a new label key `model_alias` introduced in multi-model support [v1.10.0](/anomaly-detection/changelog/#v1100). This label key adjustment was made to preserve unique label set production during writing produced metrics back to VictoriaMetrics.
|
||||
> **Note**: There is a new label key `model_alias` introduced in multi-model support [v1.10.0](../CHANGELOG.md#v1100). This label key adjustment was made to preserve unique label set production during writing produced metrics back to VictoriaMetrics.
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Metric</th>
|
||||
@@ -147,40 +201,76 @@ Label names [description](#labelnames)
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>vmanomaly_model_runs</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_model_runs`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many times models ran (per model)</td>
|
||||
<td><code>stage, query_key, model_alias, scheduler_alias, preset</code></td>
|
||||
<td>
|
||||
|
||||
`stage, query_key, model_alias, scheduler_alias, preset`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_model_run_duration_seconds</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_model_run_duration_seconds`
|
||||
</td>
|
||||
<td>Summary</td>
|
||||
<td>How much time (in seconds) model invocations took</td>
|
||||
<td><code>stage, query_key, model_alias, scheduler_alias, preset</code></td>
|
||||
<td>
|
||||
|
||||
`stage, query_key, model_alias, scheduler_alias, preset`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_model_datapoints_accepted</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_model_datapoints_accepted`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many datapoints did models accept</td>
|
||||
<td><code>stage, query_key, model_alias, scheduler_alias, preset</code></td>
|
||||
<td>
|
||||
|
||||
`stage, query_key, model_alias, scheduler_alias, preset`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_model_datapoints_produced</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_model_datapoints_produced`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many datapoints were generated by models</td>
|
||||
<td><code>stage, query_key, model_alias, scheduler_alias, preset</code></td>
|
||||
<td>
|
||||
|
||||
`stage, query_key, model_alias, scheduler_alias, preset`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_models_active</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_models_active`
|
||||
</td>
|
||||
<td>Gauge</td>
|
||||
<td>How many models are currently inferring</td>
|
||||
<td><code>query_key, model_alias, scheduler_alias, preset</code></td>
|
||||
<td>
|
||||
|
||||
`query_key, model_alias, scheduler_alias, preset`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_model_runs_skipped</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_model_runs_skipped`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many times a run was skipped (per model)</td>
|
||||
<td><code>stage, query_key, model_alias, scheduler_alias, preset</code></td>
|
||||
<td>
|
||||
|
||||
`stage, query_key, model_alias, scheduler_alias, preset`
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -188,7 +278,7 @@ Label names [description](#labelnames)
|
||||
### Writer Behaviour Metrics
|
||||
Label names [description](#labelnames)
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Metric</th>
|
||||
@@ -199,40 +289,76 @@ Label names [description](#labelnames)
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>vmanomaly_writer_request_duration_seconds</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_writer_request_duration_seconds`
|
||||
</td>
|
||||
<td>Summary</td>
|
||||
<td>How much time (in seconds) did requests to VictoriaMetrics take</td>
|
||||
<td><code>url, query_key</code></td>
|
||||
<td>
|
||||
|
||||
`url, query_key`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_writer_response_count</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_writer_response_count`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>Response code counts we got from VictoriaMetrics</td>
|
||||
<td><code>url, query_key, code</code></td>
|
||||
<td>
|
||||
|
||||
`url, query_key, code`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_writer_sent_bytes</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_writer_sent_bytes`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How much bytes were sent to VictoriaMetrics</td>
|
||||
<td><code>url, query_key</code></td>
|
||||
<td>
|
||||
|
||||
`url, query_key`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_writer_request_serialize_seconds</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_writer_request_serialize_seconds`
|
||||
</td>
|
||||
<td>Summary</td>
|
||||
<td>How much time (in seconds) did serializing take</td>
|
||||
<td><code>query_key</code></td>
|
||||
<td>
|
||||
|
||||
`query_key`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_writer_datapoints_sent</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_writer_datapoints_sent`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many datapoints were sent to VictoriaMetrics</td>
|
||||
<td><code>query_key</code></td>
|
||||
<td>
|
||||
|
||||
`query_key`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_writer_timeseries_sent</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_writer_timeseries_sent`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many timeseries were sent to VictoriaMetrics</td>
|
||||
<td><code>query_key</code></td>
|
||||
<td>
|
||||
|
||||
`query_key`
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -240,7 +366,7 @@ Label names [description](#labelnames)
|
||||
### Reader Behaviour Metrics
|
||||
Label names [description](#labelnames)
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Metric</th>
|
||||
@@ -251,57 +377,87 @@ Label names [description](#labelnames)
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>vmanomaly_reader_request_duration_seconds</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_reader_request_duration_seconds`
|
||||
</td>
|
||||
<td>Summary</td>
|
||||
<td>How much time (in seconds) did queries to VictoriaMetrics take</td>
|
||||
<td><code>url, query_key</code></td>
|
||||
<td>
|
||||
|
||||
`url, query_key`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_reader_response_count</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_reader_response_count`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>Response code counts we got from VictoriaMetrics</td>
|
||||
<td><code>url, query_key, code</code></td>
|
||||
<td>
|
||||
|
||||
`url, query_key, code`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_reader_received_bytes</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_reader_received_bytes`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How much bytes were received in responses</td>
|
||||
<td><code>query_key</code></td>
|
||||
<td>
|
||||
|
||||
`query_key`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_reader_response_parsing_seconds</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_reader_response_parsing_seconds`
|
||||
</td>
|
||||
<td>Summary</td>
|
||||
<td>How much time (in seconds) did parsing take for each step</td>
|
||||
<td><code>step</code></td>
|
||||
<td>
|
||||
|
||||
`step`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_reader_timeseries_received</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_reader_timeseries_received`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many timeseries were received from VictoriaMetrics</td>
|
||||
<td><code>query_key</code></td>
|
||||
<td>
|
||||
|
||||
`query_key`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>vmanomaly_reader_datapoints_received</code></td>
|
||||
<td>
|
||||
|
||||
`vmanomaly_reader_datapoints_received`
|
||||
</td>
|
||||
<td>Counter</td>
|
||||
<td>How many rows were received from VictoriaMetrics</td>
|
||||
<td><code>query_key</code></td>
|
||||
<td>
|
||||
|
||||
`query_key`
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
### Labelnames
|
||||
<code>stage</code> - stage of model - 'fit', 'infer' or 'fit_infer' for models that do it simultaneously, see [model types](/anomaly-detection/components/models/#model-types).
|
||||
|
||||
<code>query_key</code> - query alias from [`reader`](/anomaly-detection/components/reader.html) config section.
|
||||
|
||||
<code>model_alias</code> - model alias from [`models`](/anomaly-detection/components/models.html) config section. **Introduced in [v1.10.0](/anomaly-detection/changelog/#v1100).**
|
||||
|
||||
<code>scheduler_alias</code> - scheduler alias from [`schedulers`](anomaly-detection/components/scheduler/) config section. **Introduced in [v1.11.0](/anomaly-detection/changelog/#v1110).**
|
||||
|
||||
<code>preset</code> - preset alias for forthcoming `preset` section compatibility. **Introduced in [v1.12.0](/anomaly-detection/changelog/#v1120).**
|
||||
|
||||
<code>url</code> - writer or reader url endpoint.
|
||||
|
||||
<code>code</code> - response status code or `connection_error`, `timeout`.
|
||||
|
||||
<code>step</code> - json or dataframe reading step.
|
||||
* `stage` - stage of model - 'fit', 'infer' or 'fit_infer' for models that do it simultaneously, see [model types](./models/#model-types).
|
||||
* `query_key` - query alias from [`reader`](./reader.md) config section.
|
||||
* `model_alias` - model alias from [`models`](./models.md) config section. **Introduced in [v1.10.0](../CHANGELOG.md#v1100).**
|
||||
* `scheduler_alias` - scheduler alias from [`schedulers`](./scheduler.md) config section. **Introduced in [v1.11.0](../CHANGELOG.md#v1110).**
|
||||
* `preset` - preset alias for forthcoming `preset` section compatibility. **Introduced in [v1.12.0](../CHANGELOG.md#v1120).**
|
||||
* `url` - writer or reader url endpoint.
|
||||
* `code` - response status code or `connection_error`, `timeout`.
|
||||
* `step` - json or dataframe reading step.
|
||||
|
||||
@@ -9,14 +9,11 @@ menu:
|
||||
aliases:
|
||||
- /anomaly-detection/components/reader.html
|
||||
---
|
||||
|
||||
# Reader
|
||||
|
||||
<!--
|
||||
There are 4 sources available to read data into VM Anomaly Detection from: VictoriaMetrics, (ND)JSON file, QueryRange, or CSV file. Depending on the data source, different parameters should be specified in the config file in the `reader` section.
|
||||
-->
|
||||
|
||||
VictoriaMetrics Anomaly Detection (`vmanomaly`) primarily uses [VmReader](#vm-reader) to ingest data. This reader focuses on fetching time-series data directly from VictoriaMetrics with the help of powerful [MetricsQL](https://docs.victoriametrics.com/metricsql/) expressions for aggregating, filtering and grouping your data, ensuring seamless integration and efficient data handling.
|
||||
VictoriaMetrics Anomaly Detection (`vmanomaly`) primarily uses [VmReader](#vm-reader) to ingest data. This reader focuses on fetching time-series data directly from VictoriaMetrics with the help of powerful [MetricsQL](../../MetricsQL.md) expressions for aggregating, filtering and grouping your data, ensuring seamless integration and efficient data handling.
|
||||
|
||||
Future updates will introduce additional readers, expanding the range of data sources `vmanomaly` can work with.
|
||||
|
||||
@@ -25,7 +22,7 @@ Future updates will introduce additional readers, expanding the range of data so
|
||||
|
||||
### Config parameters
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
@@ -35,69 +32,186 @@ Future updates will introduce additional readers, expanding the range of data so
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>class</code></td>
|
||||
<td><code>"reader.vm.VmReader" (or "vm" starting from <a href="https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130">v1.13.0</a>)</code></td>
|
||||
<td>Name of the class needed to enable reading from VictoriaMetrics or Prometheus. VmReader is the default option, if not specified.</td>
|
||||
<td>
|
||||
|
||||
`class`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`reader.vm.VmReader` (or `vm` starting from [v1.13.0](../CHANGELOG.md#v1130))
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Name of the class needed to enable reading from VictoriaMetrics or Prometheus. VmReader is the default option, if not specified.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>queries</code></td>
|
||||
<td><code>"ingestion_rate: 'sum(rate(vm_rows_inserted_total[5m])) by (type) > 0'"</code></td>
|
||||
<td>PromQL/MetricsQL query to select data in format: <code>QUERY_ALIAS: "QUERY"</code>. As accepted by <code>"/query_range?query=%s"</code>.</td>
|
||||
<td>
|
||||
|
||||
`queries`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`ingestion_rate: 'sum(rate(vm_rows_inserted_total[5m])) by (type) > 0'`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
PromQL/MetricsQL query to select data in format: `QUERY_ALIAS: "QUERY"`. As accepted by `/query_range?query=%s`.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>datasource_url</code></td>
|
||||
<td><code>"http://localhost:8481/"</code></td>
|
||||
<td>Datasource URL address</td>
|
||||
<td>
|
||||
|
||||
`datasource_url`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`http://localhost:8481/`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Datasource URL address
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>tenant_id</code></td>
|
||||
<td><code>"0:0"</code></td>
|
||||
<td>For VictoriaMetrics Cluster version only, tenants are identified by accountID or accountID:projectID. See VictoriaMetrics Cluster <a href="https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy">multitenancy docs</a></td>
|
||||
<td>
|
||||
|
||||
`tenant_id`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`0:0`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
For VictoriaMetrics Cluster version only, tenants are identified by accountID or accountID:projectID. See VictoriaMetrics Cluster [multitenancy docs](../../Cluster-VictoriaMetrics.md#multitenancy)
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>sampling_period</code></td>
|
||||
<td><code>"1h"</code></td>
|
||||
<td>Frequency of the points returned. Will be converted to <code>"/query_range?step=%s"</code> param (in seconds). <b>Required</b> since <a href="https://docs.victoriametrics.com/anomaly-detection/changelog/#v190">v1.9.0</a>.</td>
|
||||
<td>
|
||||
|
||||
`sampling_period`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`1h`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Frequency of the points returned. Will be converted to `/query_range?step=%s` param (in seconds). **Required** since [v1.9.0](../CHANGELOG.md#v190).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>query_range_path</code></td>
|
||||
<td><code>"api/v1/query_range"</code></td>
|
||||
<td>Performs PromQL/MetricsQL range query. Default <code>"api/v1/query_range"</code></td>
|
||||
<td>
|
||||
|
||||
`query_range_path`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`/api/v1/query_range`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Performs PromQL/MetricsQL range query
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>health_path</code></td>
|
||||
<td><code>"health"</code></td>
|
||||
<td>Absolute or relative URL address where to check availability of the datasource. Default is <code>"health"</code>.</td>
|
||||
<td>
|
||||
|
||||
`health_path`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`health`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Absolute or relative URL address where to check availability of the datasource.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>user</code></td>
|
||||
<td><code>"USERNAME"</code></td>
|
||||
<td>BasicAuth username</td>
|
||||
<td>
|
||||
|
||||
`user`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`USERNAME`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
BasicAuth username
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>password</code></td>
|
||||
<td><code>"PASSWORD"</code></td>
|
||||
<td>BasicAuth password</td>
|
||||
<td>
|
||||
|
||||
`password`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`PASSWORD`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
BasicAuth password
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>timeout</code></td>
|
||||
<td><code>"30s"</code></td>
|
||||
<td>Timeout for the requests, passed as a string. Defaults to "30s"</td>
|
||||
<td>
|
||||
|
||||
`timeout`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`30s`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Timeout for the requests, passed as a string
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>verify_tls</code></td>
|
||||
<td><code>"false"</code></td>
|
||||
<td>Allows disabling TLS verification of the remote certificate.</td>
|
||||
<td>
|
||||
|
||||
`verify_tls`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`false`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Allows disabling TLS verification of the remote certificate.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>bearer_token</code></td>
|
||||
<td><code>"token"</code></td>
|
||||
<td>Token is passed in the standard format with header: "Authorization: bearer {token}"</td>
|
||||
<td>
|
||||
|
||||
`bearer_token`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`token`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Token is passed in the standard format with header: `Authorization: bearer {token}`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>extra_filters</code></td>
|
||||
<td><code>"[]"</code></td>
|
||||
<td>List of strings with series selector. See: <a href="https://docs.victoriametrics.com/#prometheus-querying-api-enhancements">Prometheus querying API enhancements</a></td>
|
||||
<td>
|
||||
|
||||
`extra_filters`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`[]`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
List of strings with series selector. See: [Prometheus querying API enhancements](../../README.md##prometheus-querying-api-enhancements)
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -116,4 +230,4 @@ reader:
|
||||
|
||||
### Healthcheck metrics
|
||||
|
||||
`VmReader` exposes [several healthchecks metrics](./monitoring.html#reader-behaviour-metrics).
|
||||
`VmReader` exposes [several healthchecks metrics](./monitoring.md#reader-behaviour-metrics).
|
||||
|
||||
@@ -9,13 +9,10 @@ menu:
|
||||
aliases:
|
||||
- /anomaly-detection/components/scheduler.html
|
||||
---
|
||||
|
||||
# Scheduler
|
||||
|
||||
Scheduler defines how often to run and make inferences, as well as what timerange to use to train the model.
|
||||
Is specified in `scheduler` section of a config for VictoriaMetrics Anomaly Detection.
|
||||
|
||||
> **Note: Starting from [v1.11.0](/anomaly-detection/changelog#v1110) scheduler section in config supports multiple schedulers via aliasing. <br>Also, `vmanomaly` expects scheduler section to be named `schedulers`. Using old (flat) format with `scheduler` key is deprecated and will be removed in future versions.**
|
||||
> **Note: Starting from [v1.11.0](../CHANGELOG.md#v1110) scheduler section in config supports multiple schedulers via aliasing. <br>Also, `vmanomaly` expects scheduler section to be named `schedulers`. Using old (flat) format with `scheduler` key is deprecated and will be removed in future versions.**
|
||||
|
||||
```yaml
|
||||
schedulers:
|
||||
@@ -32,7 +29,7 @@ schedulers:
|
||||
...
|
||||
```
|
||||
|
||||
Old-style configs (< [1.11.0](/anomaly-detection/changelog#v1110))
|
||||
Old-style configs (< [1.11.0](../CHANGELOG.md#v1110))
|
||||
|
||||
```yaml
|
||||
scheduler:
|
||||
@@ -64,7 +61,7 @@ options={`"scheduler.periodic.PeriodicScheduler"`, `"scheduler.oneoff.OneoffSche
|
||||
- `"scheduler.oneoff.OneoffScheduler"`: runs the process once and exits. Useful for testing.
|
||||
- `"scheduler.backtesting.BacktestingScheduler"`: imitates consecutive backtesting runs of OneoffScheduler. Runs the process once and exits. Use to get more granular control over testing on historical data.
|
||||
|
||||
> **Note**: starting from [v.1.13.0](/anomaly-detection/CHANGELOG/#v1130), class aliases are supported, so `"scheduler.periodic.PeriodicScheduler"` can be substituted to `"periodic"`, `"scheduler.oneoff.OneoffScheduler"` - to `"oneoff"`, `"scheduler.backtesting.BacktestingScheduler"` - to `"backtesting"`
|
||||
> **Note**: starting from [v1.13.0](../CHANGELOG.md#v1130), class aliases are supported, so `"scheduler.periodic.PeriodicScheduler"` can be substituted to `"periodic"`, `"scheduler.oneoff.OneoffScheduler"` - to `"oneoff"`, `"scheduler.backtesting.BacktestingScheduler"` - to `"backtesting"`
|
||||
|
||||
**Depending on selected class, different parameters should be used**
|
||||
|
||||
@@ -76,7 +73,7 @@ For periodic scheduler parameters are defined as differences in times, expressed
|
||||
|
||||
Examples: `"50s"`, `"4m"`, `"3h"`, `"2d"`, `"1w"`.
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
@@ -107,7 +104,7 @@ Examples: `"50s"`, `"4m"`, `"3h"`, `"2d"`, `"1w"`.
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
@@ -118,22 +115,43 @@ Examples: `"50s"`, `"4m"`, `"3h"`, `"2d"`, `"1w"`.
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>fit_window</code></td>
|
||||
<td>
|
||||
|
||||
`fit_window`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"14d"</code></td>
|
||||
<td>
|
||||
|
||||
`"14d"`
|
||||
</td>
|
||||
<td>What time range to use for training the models. Must be at least 1 second.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>infer_every</code></td>
|
||||
<td>
|
||||
|
||||
`infer_every`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"1m"</code></td>
|
||||
<td>
|
||||
|
||||
`"1m"`
|
||||
</td>
|
||||
<td>How often a model will write its conclusions on newly added data. Must be at least 1 second.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>fit_every</code></td>
|
||||
<td>
|
||||
|
||||
`fit_every`
|
||||
</td>
|
||||
<td>str, Optional</td>
|
||||
<td><code>"1h"</code></td>
|
||||
<td>How often to completely retrain the models. If missing value of <code>infer_every</code> is used and retrain on every inference run.</td>
|
||||
<td>
|
||||
|
||||
`"1h"`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
How often to completely retrain the models. If missing value of `infer_every` is used and retrain on every inference run.
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -165,7 +183,7 @@ ISO format supported time zone offset formats are:
|
||||
If a time zone is omitted, a timezone-naive datetime is used.
|
||||
|
||||
### Defining fitting timeframe
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Format</th>
|
||||
@@ -178,27 +196,48 @@ If a time zone is omitted, a timezone-naive datetime is used.
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td><code>fit_start_iso</code></td>
|
||||
<td>
|
||||
|
||||
`fit_start_iso`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"2022-04-01T00:00:00Z", "2022-04-01T00:00:00+01:00", "2022-04-01T00:00:00+0100", "2022-04-01T00:00:00+01"</code></td>
|
||||
<td>
|
||||
|
||||
`"2022-04-01T00:00:00Z", "2022-04-01T00:00:00+01:00", "2022-04-01T00:00:00+0100", "2022-04-01T00:00:00+01"`
|
||||
</td>
|
||||
<td rowspan=2>Start datetime to use for training a model. ISO string or UNIX time in seconds.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>UNIX time</td>
|
||||
<td><code>fit_start_s</code></td>
|
||||
<td>
|
||||
|
||||
`fit_start_s`
|
||||
</td>
|
||||
<td>float</td>
|
||||
<td>1648771200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td><code>fit_end_iso</code></td>
|
||||
<td>
|
||||
|
||||
`fit_end_iso`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"2022-04-10T00:00:00Z", "2022-04-10T00:00:00+01:00", "2022-04-10T00:00:00+0100", "2022-04-10T00:00:00+01"</code></td>
|
||||
<td rowspan=2>End datetime to use for training a model. Must be greater than <code>fit_start_*</code>. ISO string or UNIX time in seconds.</td>
|
||||
<td>
|
||||
|
||||
`"2022-04-10T00:00:00Z", "2022-04-10T00:00:00+01:00", "2022-04-10T00:00:00+0100", "2022-04-10T00:00:00+01"`
|
||||
</td>
|
||||
<td rowspan=2>End datetime to use for training a model. Must be greater than
|
||||
|
||||
`fit_start_*`
|
||||
. ISO string or UNIX time in seconds.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>UNIX time</td>
|
||||
<td><code>fit_end_s</code></td>
|
||||
<td>
|
||||
|
||||
`fit_end_s`
|
||||
</td>
|
||||
<td>float</td>
|
||||
<td>1649548800</td>
|
||||
</tr>
|
||||
@@ -206,7 +245,7 @@ If a time zone is omitted, a timezone-naive datetime is used.
|
||||
</table>
|
||||
|
||||
### Defining inference timeframe
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Format</th>
|
||||
@@ -219,27 +258,48 @@ If a time zone is omitted, a timezone-naive datetime is used.
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td><code>infer_start_iso</code></td>
|
||||
<td>
|
||||
|
||||
`infer_start_iso`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"2022-04-11T00:00:00Z", "2022-04-11T00:00:00+01:00", "2022-04-11T00:00:00+0100", "2022-04-11T00:00:00+01"</code></td>
|
||||
<td>
|
||||
|
||||
`"2022-04-11T00:00:00Z", "2022-04-11T00:00:00+01:00", "2022-04-11T00:00:00+0100", "2022-04-11T00:00:00+01"`
|
||||
</td>
|
||||
<td rowspan=2>Start datetime to use for a model inference. ISO string or UNIX time in seconds.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>UNIX time</td>
|
||||
<td><code>infer_start_s</code></td>
|
||||
<td>
|
||||
|
||||
`infer_start_s`
|
||||
</td>
|
||||
<td>float</td>
|
||||
<td>1649635200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td><code>infer_end_iso</code></td>
|
||||
<td>
|
||||
|
||||
`infer_end_iso`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"2022-04-14T00:00:00Z", "2022-04-14T00:00:00+01:00", "2022-04-14T00:00:00+0100", "2022-04-14T00:00:00+01"</code></td>
|
||||
<td rowspan=2>End datetime to use for a model inference. Must be greater than <code>infer_start_*</code>. ISO string or UNIX time in seconds.</td>
|
||||
<td>
|
||||
|
||||
`"2022-04-14T00:00:00Z", "2022-04-14T00:00:00+01:00", "2022-04-14T00:00:00+0100", "2022-04-14T00:00:00+01"`
|
||||
</td>
|
||||
<td rowspan=2>End datetime to use for a model inference. Must be greater than
|
||||
|
||||
`infer_start_*`
|
||||
. ISO string or UNIX time in seconds.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>UNIX time</td>
|
||||
<td><code>infer_end_s</code></td>
|
||||
<td>
|
||||
|
||||
`infer_end_s`
|
||||
</td>
|
||||
<td>float</td>
|
||||
<td>1649894400</td>
|
||||
</tr>
|
||||
@@ -284,7 +344,7 @@ ISO format supported time zone offset formats are:
|
||||
If a time zone is omitted, a timezone-naive datetime is used.
|
||||
|
||||
### Parallelization
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
@@ -295,10 +355,19 @@ If a time zone is omitted, a timezone-naive datetime is used.
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>n_jobs</code></td>
|
||||
<td>
|
||||
|
||||
`n_jobs`
|
||||
</td>
|
||||
<td>int</td>
|
||||
<td><code>1</code></td>
|
||||
<td>Allows <i>proportionally faster (yet more resource-intensive)</i> evaluations of a config on historical data. Default value is 1, that implies <i>sequential</i> execution. Introduced in <a href="https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130">v1.13.0</a></td>
|
||||
<td>
|
||||
|
||||
`1`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Allows *proportionally faster (yet more resource-intensive)* evaluations of a config on historical data. Default value is 1, that implies *sequential* execution. Introduced in [v1.13.0](../CHANGELOG.md#v1130)
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -306,7 +375,7 @@ If a time zone is omitted, a timezone-naive datetime is used.
|
||||
### Defining overall timeframe
|
||||
|
||||
This timeframe will be used for slicing on intervals `(fit_window, infer_window == fit_every)`, starting from the *latest available* time point, which is `to_*` and going back, until no full `fit_window + infer_window` interval exists within the provided timeframe.
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Format</th>
|
||||
@@ -319,27 +388,48 @@ This timeframe will be used for slicing on intervals `(fit_window, infer_window
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td><code>from_iso</code></td>
|
||||
<td>
|
||||
|
||||
`from_iso`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"2022-04-01T00:00:00Z", "2022-04-01T00:00:00+01:00", "2022-04-01T00:00:00+0100", "2022-04-01T00:00:00+01"</code></td>
|
||||
<td>
|
||||
|
||||
`"2022-04-01T00:00:00Z", "2022-04-01T00:00:00+01:00", "2022-04-01T00:00:00+0100", "2022-04-01T00:00:00+01"`
|
||||
</td>
|
||||
<td rowspan=2>Start datetime to use for backtesting.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>UNIX time</td>
|
||||
<td><code>from_s</code></td>
|
||||
<td>
|
||||
|
||||
`from_s`
|
||||
</td>
|
||||
<td>float</td>
|
||||
<td>1648771200</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td><code>to_iso</code></td>
|
||||
<td>
|
||||
|
||||
`to_iso`
|
||||
</td>
|
||||
<td>str</td>
|
||||
<td><code>"2022-04-10T00:00:00Z", "2022-04-10T00:00:00+01:00", "2022-04-10T00:00:00+0100", "2022-04-10T00:00:00+01"</code></td>
|
||||
<td rowspan=2>End datetime to use for backtesting. Must be greater than <code>from_start_*</code>.</td>
|
||||
<td>
|
||||
|
||||
`"2022-04-10T00:00:00Z", "2022-04-10T00:00:00+01:00", "2022-04-10T00:00:00+0100", "2022-04-10T00:00:00+01"`
|
||||
</td>
|
||||
<td rowspan=2>End datetime to use for backtesting. Must be greater than
|
||||
|
||||
`from_start_*`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>UNIX time</td>
|
||||
<td><code>to_s</code></td>
|
||||
<td>
|
||||
|
||||
`to_s`
|
||||
</td>
|
||||
<td>float</td>
|
||||
<td>1649548800</td>
|
||||
</tr>
|
||||
@@ -348,7 +438,7 @@ This timeframe will be used for slicing on intervals `(fit_window, infer_window
|
||||
|
||||
### Defining training timeframe
|
||||
The same *explicit* logic as in [Periodic scheduler](#periodic-scheduler)
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Format</th>
|
||||
@@ -361,21 +451,30 @@ The same *explicit* logic as in [Periodic scheduler](#periodic-scheduler)
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td rowspan=2><code>fit_window</code></td>
|
||||
<td rowspan=2>
|
||||
|
||||
`fit_window`
|
||||
</td>
|
||||
<td rowspan=2>str</td>
|
||||
<td><code>"PT1M", "P1H"</code></td>
|
||||
<td>
|
||||
|
||||
`"PT1M", "P1H"`
|
||||
</td>
|
||||
<td rowspan=2>What time range to use for training the models. Must be at least 1 second.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Prometheus-compatible</td>
|
||||
<td><code>"1m", "1h"</code></td>
|
||||
<td>
|
||||
|
||||
`"1m", "1h"`
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
### Defining inference timeframe
|
||||
In `BacktestingScheduler`, the inference window is *implicitly* defined as a period between 2 consecutive model `fit_every` runs. The *latest* inference window starts from `to_s` - `fit_every` and ends on the *latest available* time point, which is `to_s`. The previous periods for fit/infer are defined the same way, by shifting `fit_every` seconds backwards until we get the last full fit period of `fit_window` size, which start is >= `from_s`.
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Format</th>
|
||||
@@ -388,14 +487,23 @@ In `BacktestingScheduler`, the inference window is *implicitly* defined as a per
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>ISO 8601</td>
|
||||
<td rowspan=2><code>fit_every</code></td>
|
||||
<td rowspan=2>
|
||||
|
||||
`fit_every`
|
||||
</td>
|
||||
<td rowspan=2>str</td>
|
||||
<td><code>"PT1M", "P1H"</code></td>
|
||||
<td>
|
||||
|
||||
`"PT1M", "P1H"`
|
||||
</td>
|
||||
<td rowspan=2>What time range to use previously trained model to infer on new data until next retrain happens.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Prometheus-compatible</td>
|
||||
<td><code>"1m", "1h"</code></td>
|
||||
<td>
|
||||
|
||||
`"1m", "1h"`
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -424,4 +532,4 @@ schedulers:
|
||||
fit_window: '14d'
|
||||
fit_every: '1h'
|
||||
n_jobs: 1 # default = 1 (sequential), set it up to # of CPUs for parallel execution
|
||||
```
|
||||
```
|
||||
|
||||
@@ -9,9 +9,6 @@ menu:
|
||||
aliases:
|
||||
- /anomaly-detection/components/writer.html
|
||||
---
|
||||
|
||||
# Writer
|
||||
|
||||
For exporting data, VictoriaMetrics Anomaly Detection (`vmanomaly`) primarily employs the [VmWriter](#vm-writer), which writes produced anomaly scores **(preserving initial labelset and optionally applying additional ones)** back to VictoriaMetrics. This writer is tailored for smooth data export within the VictoriaMetrics ecosystem.
|
||||
|
||||
Future updates will introduce additional export methods, offering users more flexibility in data handling and integration.
|
||||
@@ -20,7 +17,7 @@ Future updates will introduce additional export methods, offering users more fle
|
||||
|
||||
### Config parameters
|
||||
|
||||
<table>
|
||||
<table class="params">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
@@ -30,71 +27,190 @@ Future updates will introduce additional export methods, offering users more fle
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>class</code></td>
|
||||
<td><code>"writer.vm.VmWriter" (or "vm" starting from <a href="https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130">v1.13.0</a>)</code></td>
|
||||
<td>Name of the class needed to enable writing to VictoriaMetrics or Prometheus. VmWriter is the default option, if not specified.</td>
|
||||
<td>
|
||||
|
||||
`class`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`writer.vm.VmWriter` or `vm` starting from [`v1.13.0`](../CHANGELOG.md#v1130)
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Name of the class needed to enable writing to VictoriaMetrics or Prometheus. VmWriter is the default option, if not specified.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>datasource_url</code></td>
|
||||
<td><code>"http://localhost:8481/"</code></td>
|
||||
<td>Datasource URL address</td>
|
||||
<td>
|
||||
|
||||
`datasource_url`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`http://localhost:8481/`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Datasource URL address
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>tenant_id</code></td>
|
||||
<td><code>"0:0"</code></td>
|
||||
<td>For VictoriaMetrics Cluster version only, tenants are identified by accountID or accountID:projectID. See VictoriaMetrics Cluster <a href="https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy">multitenancy docs</a></td>
|
||||
<td>
|
||||
|
||||
`tenant_id`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`0:0`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
For VictoriaMetrics Cluster version only, tenants are identified by accountID or accountID:projectID. See VictoriaMetrics Cluster [multitenancy docs](../../Cluster-VictoriaMetrics.md#multitenancy)
|
||||
</td>
|
||||
</tr>
|
||||
<!-- Additional rows for metric_format -->
|
||||
<tr>
|
||||
<td rowspan="4"><code>metric_format</code></td>
|
||||
<td><code>__name__: "vmanomaly_$VAR"</code></td>
|
||||
<td rowspan="4">Metrics to save the output (in metric names or labels). Must have <code>__name__</code> key. Must have a value with <code>$VAR</code> placeholder in it to distinguish between resulting metrics. Supported placeholders:
|
||||
<td rowspan="4">
|
||||
|
||||
`metric_format`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`__name__: "vmanomaly_$VAR"`
|
||||
</td>
|
||||
<td rowspan="4">
|
||||
|
||||
Metrics to save the output (in metric names or labels). Must have `__name__` key. Must have a value with `$VAR` placeholder in it to distinguish between resulting metrics. Supported placeholders:
|
||||
<ul>
|
||||
<li><code>$VAR</code> -- Variables that model provides, all models provide the following set: {"anomaly_score", "y", "yhat", "yhat_lower", "yhat_upper"}. Description of standard output is <a href="/anomaly-detection/components/models.html#vmanomaly-output">here</a>. Depending on <a href="/anomaly-detection/components/models.html">model type</a> it can provide more metrics, like "trend", "seasonality" etc.</li>
|
||||
<li><code>$QUERY_KEY</code> -- E.g. "ingestion_rate".</li>
|
||||
<li>
|
||||
|
||||
`$VAR` -- Variables that model provides, all models provide the following set: {"anomaly_score", "y", "yhat", "yhat_lower", "yhat_upper"}. Description of standard output is [here](./models.md#vmanomaly-output). Depending on [model type](./models.md) it can provide more metrics, like "trend", "seasonality" etc.
|
||||
</li>
|
||||
<li>
|
||||
|
||||
`$QUERY_KEY` -- E.g. "ingestion_rate".
|
||||
</li>
|
||||
</ul>
|
||||
Other keys are supposed to be configured by the user to help identify generated metrics, e.g., specific config file name etc.
|
||||
More details on metric formatting are <a href="#metrics-formatting">here</a>.
|
||||
More details on metric formatting are [here](#metrics-formatting).
|
||||
</td>
|
||||
</tr>
|
||||
<tr><td><code>for: "$QUERY_KEY"</code></td></tr>
|
||||
<tr><td><code>run: "test_metric_format"</code></td></tr>
|
||||
<tr><td><code>config: "io_vm_single.yaml"</code></td></tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
`for: "$QUERY_KEY"`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
`run: "test_metric_format"`
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
|
||||
`config: "io_vm_single.yaml"`
|
||||
</td>
|
||||
</tr>
|
||||
<!-- End of additional rows -->
|
||||
<tr>
|
||||
<td><code>import_json_path</code></td>
|
||||
<td><code>"/api/v1/import"</code></td>
|
||||
<td>Optional, to override the default import path</td>
|
||||
<td>
|
||||
|
||||
`import_json_path`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`/api/v1/import`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Optional, to override the default import path
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>health_path</code></td>
|
||||
<td><code>"health"</code></td>
|
||||
<td>Absolute or relative URL address where to check the availability of the datasource. Optional, to override the default <code>"/health"</code> path.</td>
|
||||
<td>
|
||||
|
||||
`health_path`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`/health`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Absolute or relative URL address where to check the availability of the datasource. Optional, to override the default `/health` path.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>user</code></td>
|
||||
<td><code>"USERNAME"</code></td>
|
||||
<td>BasicAuth username</td>
|
||||
<td>
|
||||
|
||||
`user`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`USERNAME`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
BasicAuth username
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>password</code></td>
|
||||
<td><code>"PASSWORD"</code></td>
|
||||
<td>BasicAuth password</td>
|
||||
<td>
|
||||
|
||||
`password`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`PASSWORD`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
BasicAuth password
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>timeout</code></td>
|
||||
<td><code>"5s"</code></td>
|
||||
<td>Timeout for the requests, passed as a string. Defaults to "5s"</td>
|
||||
<td>
|
||||
|
||||
`timeout`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`5s`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Timeout for the requests, passed as a string
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>verify_tls</code></td>
|
||||
<td><code>"false"</code></td>
|
||||
<td>Allows disabling TLS verification of the remote certificate.</td>
|
||||
<td>
|
||||
|
||||
`verify_tls`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`false`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Allows disabling TLS verification of the remote certificate.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>bearer_token</code></td>
|
||||
<td><code>"token"</code></td>
|
||||
<td>Token is passed in the standard format with the header: "Authorization: bearer {token}"</td>
|
||||
<td>
|
||||
|
||||
`bearer_token`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
`token`
|
||||
</td>
|
||||
<td>
|
||||
|
||||
Token is passed in the standard format with the header: `Authorization: bearer {token}`
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -119,7 +235,7 @@ writer:
|
||||
|
||||
### Healthcheck metrics
|
||||
|
||||
`VmWriter` exposes [several healthchecks metrics](./monitoring.html#writer-behaviour-metrics).
|
||||
`VmWriter` exposes [several healthchecks metrics](./monitoring.md#writer-behaviour-metrics).
|
||||
|
||||
### Metrics formatting
|
||||
|
||||
@@ -130,8 +246,8 @@ __name__: PREFIX1_$VAR
|
||||
for: PREFIX2_$QUERY_KEY
|
||||
```
|
||||
|
||||
* for `__name__` parameter it will name metrics returned by models as `PREFIX1_anomaly_score`, `PREFIX1_yhat_lower`, etc. Vmanomaly output metrics names described [here](/anomaly-detection/components/models.html#vmanomaly-output)
|
||||
* for `for` parameter will add labels `PREFIX2_query_name_1`, `PREFIX2_query_name_2`, etc. Query names are set as aliases in config `reader` section in [`queries`](anomaly-detection/components/reader.html#config-parameters) parameter.
|
||||
* for `__name__` parameter it will name metrics returned by models as `PREFIX1_anomaly_score`, `PREFIX1_yhat_lower`, etc. Vmanomaly output metrics names described [here](./models.md#vmanomaly-output)
|
||||
* for `for` parameter will add labels `PREFIX2_query_name_1`, `PREFIX2_query_name_2`, etc. Query names are set as aliases in config `reader` section in [`queries`](./reader.md#config-parameters) parameter.
|
||||
|
||||
It is possible to specify other custom label names needed.
|
||||
For example:
|
||||
@@ -141,7 +257,7 @@ custom_label_1: label_name_1
|
||||
custom_label_2: label_name_2
|
||||
```
|
||||
|
||||
Apart from specified labels, output metrics will return **labels inherited from input metrics returned by [queries](/anomaly-detection/components/reader.html#config-parameters)**.
|
||||
Apart from specified labels, output metrics will return **labels inherited from input metrics returned by [queries](./reader.md#config-parameters)**.
|
||||
For example if input data contains labels such as `cpu=1, device=eth0, instance=node-exporter:9100` all these labels will be present in vmanomaly output metrics.
|
||||
|
||||
So if metric_format section was set up like this:
|
||||
@@ -156,7 +272,7 @@ metric_format:
|
||||
|
||||
It will return metrics that will look like:
|
||||
|
||||
```yaml
|
||||
```promtextmetric
|
||||
{__name__="PREFIX1_anomaly_score", for="PREFIX2_query_name_1", custom_label_1="label_name_1", custom_label_2="label_name_2", cpu=1, device="eth0", instance="node-exporter:9100"}
|
||||
{__name__="PREFIX1_yhat_lower", for="PREFIX2_query_name_1", custom_label_1="label_name_1", custom_label_2="label_name_2", cpu=1, device="eth0", instance="node-exporter:9100"}
|
||||
{__name__="PREFIX1_anomaly_score", for="PREFIX2_query_name_2", custom_label_1="label_name_1", custom_label_2="label_name_2", cpu=1, device="eth0", instance="node-exporter:9100"}
|
||||
|
||||
@@ -1,19 +1,5 @@
|
||||
---
|
||||
title: Guides
|
||||
weight: 3
|
||||
menu:
|
||||
docs:
|
||||
identifier: "anomaly-detection-guides"
|
||||
parent: "anomaly-detection"
|
||||
weight: 3
|
||||
aliases:
|
||||
- /anomaly-detection/guides.html
|
||||
---
|
||||
|
||||
# Guides
|
||||
|
||||
This section holds guides of how to set up and use VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](/anomaly-detection/Overview.html)) service, integrating it with different observability components.
|
||||
This section holds guides of how to set up and use VictoriaMetrics Anomaly Detection (or simply [`vmanomaly`](../Overview.md)) service, integrating it with different observability components.
|
||||
|
||||
Guides:
|
||||
|
||||
* [Anomaly Detection & Alerting Setup](/anomaly-detection/guides/guide-vmanomaly-vmalert.html)
|
||||
* [Anomaly Detection & Alerting Setup](./guide-vmanomaly-vmalert/README.md)
|
||||
|
||||
12
docs/anomaly-detection/guides/_index.md
Normal file
12
docs/anomaly-detection/guides/_index.md
Normal file
@@ -0,0 +1,12 @@
|
||||
---
|
||||
title: Guides
|
||||
weight: 3
|
||||
menu:
|
||||
docs:
|
||||
identifier: "anomaly-detection-guides"
|
||||
parent: "anomaly-detection"
|
||||
weight: 3
|
||||
aliases:
|
||||
- /anomaly-detection/guides.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -1,46 +1,29 @@
|
||||
---
|
||||
weight: 1
|
||||
sort: 1
|
||||
title: Anomaly Detection and Alerting Setup
|
||||
menu:
|
||||
docs:
|
||||
parent: "anomaly-detection-guides"
|
||||
weight: 1
|
||||
aliases:
|
||||
- /anomaly-detection/guides/guide-vmanomaly-vmalert.html
|
||||
- /guides/guide-vmanomaly-vmalert.html
|
||||
- /guides/vmanomaly-integration.html
|
||||
- /guides/anomaly-detection-and-alerting-setup.html
|
||||
|
||||
---
|
||||
# Anomaly Detection and Alerting Setup
|
||||
|
||||
**Prerequisites**:
|
||||
|
||||
- To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
|
||||
- In the tutorial, we'll be using the following VictoriaMetrics components:
|
||||
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics/) (v1.101.0)
|
||||
- [vmalert](https://docs.victoriametrics.com/vmalert/) (v1.101.0)
|
||||
- [vmagent](https://docs.victoriametrics.com/vmagent/) (v1.101.0)
|
||||
- [VictoriaMetrics Single-Node](../../../Single-Server-VictoriaMetrics.md) (v1.102.0)
|
||||
- [vmalert](../../../vmalert.md) (v1.102.0)
|
||||
- [vmagent](../../../vmagent.md) (v1.102.0)
|
||||
- [Grafana](https://grafana.com/) (v.10.2.1)
|
||||
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
|
||||
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
|
||||
|
||||
<img src="guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_overview.webp" alt="vmanomaly typical setup diagram">
|
||||

|
||||
|
||||
> **Note: Configurations used throughout this guide can be found [here](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/vmanomaly/vmanomaly-integration/)**
|
||||
|
||||
> **Note:** Starting from [v1.13.0](/anomaly-detection/CHANGELOG#v1130) `node-exporter` observability preset is available for `vmanomaly`. Please find the guide [here](/anomaly-detection/presets/#node-exporter).
|
||||
> **Note:** Starting from [v1.13.0](../../CHANGELOG.md#v1130) `node-exporter` observability preset is available for `vmanomaly`. Please find the guide [here](../../Presets.md#node-exporter).
|
||||
|
||||
## 1. What is vmanomaly?
|
||||
|
||||
*VictoriaMetrics Anomaly Detection* ([vmanomaly](/anomaly-detection/Overview.html)) is a service that continuously scans time series stored in VictoriaMetrics and detects unexpected changes within data patterns in real-time. It does so by utilizing user-configurable machine learning models.
|
||||
*VictoriaMetrics Anomaly Detection* ([vmanomaly](../../Overview.md)) is a service that continuously scans time series stored in VictoriaMetrics and detects unexpected changes within data patterns in real-time. It does so by utilizing user-configurable machine learning models.
|
||||
|
||||
All the service parameters are defined in a config file.
|
||||
|
||||
> **Note**: Starting from [1.10.0](/anomaly-detection/changelog/#v1100), each `vmanomaly` configuration file can support more that one model type. To utilize *different models* on your data, it is no longer necessary to run multiple instances of the `vmanomaly` process. Please refer to [model](/anomaly-detection/components/models/) config section for more details.
|
||||
> **Note**: Starting from [1.10.0](../../CHANGELOG.md#v1100), each `vmanomaly` configuration file can support more that one model type. To utilize *different models* on your data, it is no longer necessary to run multiple instances of the `vmanomaly` process. Please refer to [model](../..//components/models.md) config section for more details.
|
||||
|
||||
> **Note**: Starting from [1.11.0](/anomaly-detection/changelog/#v1110), each `vmanomaly` configuration file can support more that one model type, each attached to one (or more) schedulers. To utilize *different models* with *different schedulers* on your data, it is no longer necessary to run multiple instances of the `vmanomaly` process. Please refer to [model](/anomaly-detection/components/models/#schedulers) and [scheduler](/anomaly-detection/components/scheduler/) config sections for more details.
|
||||
> **Note**: Starting from [1.11.0](../../CHANGELOG.md#v1110), each `vmanomaly` configuration file can support more that one model type, each attached to one (or more) schedulers. To utilize *different models* with *different schedulers* on your data, it is no longer necessary to run multiple instances of the `vmanomaly` process. Please refer to [model](../../components/models.md#schedulers) and [scheduler](../../components/scheduler.md) config sections for more details.
|
||||
|
||||
|
||||
**vmanomaly** does the following:
|
||||
@@ -61,9 +44,9 @@ Then, users can enable alerting rules based on the **anomaly score** with [vmale
|
||||
|
||||
## 2. What is vmalert?
|
||||
|
||||
[vmalert](https://docs.victoriametrics.com/vmalert/) is an alerting tool for VictoriaMetrics. It executes a list of the given alerting or recording rules against configured `-datasource.url`.
|
||||
[vmalert](../../../vmalert.md) is an alerting tool for VictoriaMetrics. It executes a list of the given alerting or recording rules against configured `-datasource.url`.
|
||||
|
||||
[Alerting rules](https://docs.victoriametrics.com/vmalert/#alerting-rules) allow you to define conditions that, when met, will notify the user. The alerting condition is defined in a form of a query expression via [MetricsQL query language](https://docs.victoriametrics.com/metricsql/). For example, in our case, the expression `anomaly_score > 1.0` will notify a user when the calculated anomaly score exceeds a threshold of `1.0`.
|
||||
[Alerting rules](../../../vmalert.md#alerting-rules) allow you to define conditions that, when met, will notify the user. The alerting condition is defined in a form of a query expression via [MetricsQL query language](../../../MetricsQL.md). For example, in our case, the expression `anomaly_score > 1.0` will notify a user when the calculated anomaly score exceeds a threshold of `1.0`.
|
||||
|
||||
## 3. How does vmanomaly works with vmalert?
|
||||
|
||||
@@ -72,9 +55,9 @@ Compared to classical alerting rules, anomaly detection is more "hands-off" and
|
||||
Practical use case is to put anomaly score generated by vmanomaly into alerting rules with some threshold.
|
||||
|
||||
**In this tutorial we are going to:**
|
||||
- Configure docker-compose file with all needed services ([VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics/), [vmalert](https://docs.victoriametrics.com/vmalert/), [vmagent](https://docs.victoriametrics.com/vmagent/), [Grafana](https://grafana.com/), [Node Exporter](https://prometheus.io/docs/guides/node-exporter/) and [vmanomaly](https://docs.victoriametrics.com/anomaly-detection/Overview.html) ).
|
||||
- Explore configuration files for [vmanomaly](https://docs.victoriametrics.com/anomaly-detection/Overview.html) and [vmalert](https://docs.victoriametrics.com/vmalert/).
|
||||
- Run our own [VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) database with data scraped from [Node Exporter](https://prometheus.io/docs/guides/node-exporter/).
|
||||
- Configure docker-compose file with all needed services ([VictoriaMetrics Single-Node](../../../Single-Server-VictoriaMetrics.md), [vmalert](../../../vmalert.md), [vmagent](../../../vmagent.md), [Grafana](https://grafana.com/), [Node Exporter](https://prometheus.io/docs/guides/node-exporter/) and [vmanomaly](../../Overview.md) ).
|
||||
- Explore configuration files for [vmanomaly](../../Overview.md) and [vmalert](../../../vmalert.md).
|
||||
- Run our own [VictoriaMetrics](../../../Single-Server-VictoriaMetrics.md) database with data scraped from [Node Exporter](https://prometheus.io/docs/guides/node-exporter/).
|
||||
- Explore data for analysis in [Grafana](https://grafana.com/).
|
||||
- Explore `vmanomaly` results.
|
||||
- Explore `vmalert` alerts
|
||||
@@ -105,10 +88,10 @@ node_cpu_seconds_total{cpu="1",mode="iowait"} 51.22
|
||||
|
||||
In this context, the metric `node_cpu_seconds_total` provides a comprehensive breakdown of the time each CPU core has spent in various operational modes. These modes include: _user_, _system_, _iowait_, _idle_, _irq&softirq_, _guest_, and _steal_. Each of these eight modes is mutually exclusive, offering distinct insights into CPU activity. For instance, a predominant _iowait_ suggests disk or network bottlenecks, while elevated levels in _user_ or _system_ indicate significant CPU utilization.
|
||||
|
||||
The `node_cpu_seconds_total` metric is classified as a [counter](https://docs.victoriametrics.com/keyconcepts/#counter) type. To analyze the duration each CPU core spends in these modes, it is necessary to compute the rate of change per second using the [rate function](https://docs.victoriametrics.com/metricsql/#rate): `rate(node_cpu_seconds_total)`. For a more refined and smoother aggregation of data by mode, we apply the sum function. The resulting query is formulated as follows: `sum(rate(node_cpu_seconds_total[5m])) by (mode, instance, job)`.
|
||||
The `node_cpu_seconds_total` metric is classified as a [counter](../../../keyConcepts.md#counter) type. To analyze the duration each CPU core spends in these modes, it is necessary to compute the rate of change per second using the [rate function](../../../MetricsQL.md#rate): `rate(node_cpu_seconds_total)`. For a more refined and smoother aggregation of data by mode, we apply the sum function. The resulting query is formulated as follows: `sum(rate(node_cpu_seconds_total[5m])) by (mode, instance, job)`.
|
||||
|
||||
Below is an illustrative example of how this query might be visualized in Grafana:
|
||||
<img alt="node_cpu_rate_graph" src="guide-vmanomaly-vmalert/guide-vmanomaly-vmalert-query.webp">
|
||||

|
||||
|
||||
This query will yield a total of eight time series, each corresponding to a CPU mode. The number of series is unaffected by the number of CPU cores, due to the `by` aggregation applied. These series serve as the input for `vmanomaly`, where the service independently fits a separate instance of the configured model type to each of time series.
|
||||
|
||||
@@ -119,25 +102,25 @@ This query will yield a total of eight time series, each corresponding to a CPU
|
||||
**Parameter description**:
|
||||
The configuration file for `vmanomaly` comprises 4 essential sections:
|
||||
|
||||
1. [`scheduler`](/anomaly-detection/components/scheduler.html) - This section determines the frequency of model inferences and training, including the time range for model training. Starting from [v1.11.0](/anomaly-detection/changelog/#v1110), multiple individual schedulers are supported for each model type in a single config.
|
||||
1. [`scheduler`](../../components/scheduler.md) - This section determines the frequency of model inferences and training, including the time range for model training. Starting from [v1.11.0](../../CHANGELOG.md#v1110), multiple individual schedulers are supported for each model type in a single config.
|
||||
|
||||
2. [`models`](/anomaly-detection/components/models.html) - Here, you define specific parameters and configurations for the models being used for anomaly detection. Starting from [v1.10.0](/anomaly-detection/changelog/#v1100), multiple model configurations are supported in a single config.
|
||||
2. [`models`](../../components/models.md) - Here, you define specific parameters and configurations for the models being used for anomaly detection. Starting from [v1.10.0](../../CHANGELOG.md#v1100), multiple model configurations are supported in a single config.
|
||||
|
||||
3. [`reader`](/anomaly-detection/components/reader.html) - This section outlines the methodology for data reading, including the data source location.
|
||||
3. [`reader`](../../components/reader.md) - This section outlines the methodology for data reading, including the data source location.
|
||||
|
||||
4. [`writer`](/anomaly-detection/components/writer.html) - Specifies the destination and method for writing the generated output.
|
||||
4. [`writer`](../../components/writer.md) - Specifies the destination and method for writing the generated output.
|
||||
|
||||
5. [`monitoring`](/anomaly-detection/components/monitoring.html) (optional) - Describes how to monitor and expose health check metrics of `vmanomaly`.
|
||||
5. [`monitoring`](../../components/monitoring.md) (optional) - Describes how to monitor and expose health check metrics of `vmanomaly`.
|
||||
|
||||
Detailed parameters in each section:
|
||||
|
||||
* `schedulers` ([PeriodicScheduler](/anomaly-detection/components/scheduler/#periodic-scheduler) is used here)
|
||||
* `schedulers` ([PeriodicScheduler](../../components/scheduler.md#periodic-scheduler) is used here)
|
||||
* `infer_every` - Specifies the frequency at which the trained models perform inferences on new data, essentially determining how often new anomaly score data points are generated. Format examples: 30s, 4m, 2h, 1d (time units: 's' for seconds, 'm' for minutes, 'h' for hours, 'd' for days). This parameter essentially asks, at regular intervals (e.g., every 1 minute), whether the latest data points appear abnormal based on historical data.
|
||||
* `fit_every` - Sets the frequency for retraining the models. A higher frequency ensures more updated models but requires more CPU resources. If omitted, models are retrained in each `infer_every` cycle. Format is similar to `infer_every`.
|
||||
* `fit_window` - Defines the data interval for training the models. Longer intervals allow for capturing extensive historical behavior and better seasonal pattern detection but may slow down the model's response to permanent metric changes and increase resource consumption. A minimum of two full seasonal cycles is recommended. Example format: 3h for three hours of data.
|
||||
|
||||
* `models`
|
||||
* `class` - Specifies the model to be used. Options include custom models ([guide here](/anomaly-detection/components/models.html#custom-model-guide)) or a selection from [built-in models](/anomaly-detection/components/models.html#built-in-models), such as the [Facebook Prophet](/anomaly-detection/components/models.html#prophet) (`model.prophet.ProphetModel`).
|
||||
* `class` - Specifies the model to be used. Options include custom models ([guide here](../../components/models.md#custom-model-guide)) or a selection from [built-in models](../../components/models.md#built-in-models), such as the [Facebook Prophet](../../components/models.md#prophet) (`model.prophet.ProphetModel`).
|
||||
* `args` - Model-specific parameters, formatted as a YAML dictionary in the `key: value` structure. Parameters available in [FB Prophet](https://facebook.github.io/prophet/docs/quick_start.html) can be used as an example.
|
||||
|
||||
* `reader`
|
||||
@@ -210,7 +193,7 @@ groups:
|
||||
summary: Anomaly Score exceeded 1.0. `sum(rate(node_cpu_seconds_total))` is showing abnormal behavior.
|
||||
```
|
||||
|
||||
In the query expression `expr`, it's crucial to establish a criterion based on the generated anomaly scores. Typically, an [anomaly score](/anomaly-detection/faq/#what-is-anomaly-score) ranging from 0.0 to 1.0 indicates that the analyzed value falls within normal behavior. Scores exceeding 1.0 signal increasing confidence from our model that the observed value is anomalous.
|
||||
In the query expression `expr`, it's crucial to establish a criterion based on the generated anomaly scores. Typically, an [anomaly score](../../FAQ.md#what-is-anomaly-score) ranging from 0.0 to 1.0 indicates that the analyzed value falls within normal behavior. Scores exceeding 1.0 signal increasing confidence from our model that the observed value is anomalous.
|
||||
|
||||
Selecting an appropriate threshold for the anomaly score depends on your specific requirements and the context of the data. One effective method for determining this threshold is through visual analysis. By plotting the `anomaly_score` metric in conjunction with the predicted 'expected' range, delineated by `yhat_lower` and `yhat_upper`, you can make a more informed decision. Later in this tutorial, we will demonstrate this process with a practical example.
|
||||
|
||||
@@ -306,7 +289,7 @@ scrape_configs:
|
||||
|
||||
We will utilize the license key stored locally in the file `vmanomaly_license`.
|
||||
|
||||
For additional licensing options, please refer to the [VictoriaMetrics Anomaly Detection documentation on licensing](https://docs.victoriametrics.com/anomaly-detection/Overview#licensing).
|
||||
For additional licensing options, please refer to the [VictoriaMetrics Anomaly Detection documentation on licensing](../../Overview.md#licensing).
|
||||
|
||||
|
||||
### Alertmanager setup
|
||||
@@ -330,7 +313,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
|
||||
services:
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:v1.101.0
|
||||
image: victoriametrics/vmagent:v1.102.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -347,7 +330,7 @@ services:
|
||||
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.101.0
|
||||
image: victoriametrics/victoria-metrics:v1.102.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -380,7 +363,7 @@ services:
|
||||
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:v1.101.0
|
||||
image: victoriametrics/vmalert:v1.102.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -452,7 +435,7 @@ networks:
|
||||
|
||||
Before running our docker-compose make sure that your directory contains all required files:
|
||||
|
||||
<img src="guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_files.webp" alt="all files">
|
||||

|
||||
|
||||
This docker-compose file will pull docker images, set up each service and run them all together with the command:
|
||||
|
||||
@@ -489,7 +472,7 @@ Each of these metrics will contain same labels our query `sum(rate(node_cpu_seco
|
||||
### Anomaly scores for each metric with its according labels.
|
||||
|
||||
Query: `anomaly_score`
|
||||
<img alt="Anomaly score graph" src="guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_anomaly-score.webp">
|
||||

|
||||
|
||||
<br>Check out if the anomaly score is high for datapoints you think are anomalies. If not, you can try other parameters in the config file or try other model type.
|
||||
|
||||
@@ -500,7 +483,7 @@ As you may notice a lot of data shows anomaly score greater than 1. It is expect
|
||||
|
||||
Queries: `yhat_lower`, `yhat_upper` and `yhat`
|
||||
|
||||
<img alt="yhat lower and yhat upper" src="guide-vmanomaly-vmalert/guide-vmanomaly-vmalert-boundaries.webp">
|
||||

|
||||
|
||||
Boundaries of 'normal' metric values according to model inference.
|
||||
|
||||
@@ -508,10 +491,10 @@ Boundaries of 'normal' metric values according to model inference.
|
||||
|
||||
On the page `http://localhost:8880/vmalert/groups` you can find our configured Alerting rule:
|
||||
|
||||
<img alt="alert rule" src="guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_alert-rule.webp">
|
||||

|
||||
|
||||
According to the rule configured for vmalert we will see Alert when anomaly score exceed 1. You will see an alert on Alert tab. `http://localhost:8880/vmalert/alerts`:
|
||||
<img alt="alerts firing" src="guide-vmanomaly-vmalert/guide-vmanomaly-vmalert_alerts-firing.webp">
|
||||

|
||||
|
||||
## 10. Conclusion
|
||||
|
||||
@@ -529,4 +512,4 @@ Key takeaways include:
|
||||
|
||||
As you continue to use VictoriaMetrics Anomaly Detection and `vmalert`, remember that the effectiveness of anomaly detection largely depends on the appropriateness of the model chosen, the accuracy of configurations and the data patterns observed. This guide serves as a starting point, and we encourage you to experiment with different configurations and models to best suit your specific data needs and use cases. In case you need a helping hand - [contact us](https://victoriametrics.com/contact-us/).
|
||||
|
||||
> **Note:** Starting from [v1.13.0](/anomaly-detection/CHANGELOG#v1130) `node-exporter` observability preset is available for `vmanomaly`. Please find the guide [here](/anomaly-detection/presets/#node-exporter).
|
||||
> **Note:** Starting from [v1.13.0](../../CHANGELOG.md#v1130) `node-exporter` observability preset is available for `vmanomaly`. Please find the guide [here](../../Presets.md#node-exporter).
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
weight: 1
|
||||
sort: 1
|
||||
title: Anomaly Detection and Alerting Setup
|
||||
menu:
|
||||
docs:
|
||||
parent: "anomaly-detection-guides"
|
||||
weight: 1
|
||||
aliases:
|
||||
- /anomaly-detection/guides/guide-vmanomaly-vmalert.html
|
||||
- /guides/guide-vmanomaly-vmalert.html
|
||||
- /guides/vmanomaly-integration.html
|
||||
- /guides/anomaly-detection-and-alerting-setup.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user