Compare commits

..

3 Commits

Author SHA1 Message Date
Hui Wang
d4a40004ef app/vmalert: tolerate 400 evaluation error with -replay.continueWithExecutionErr in replay mode
Follow up https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11348

VM would then return `400` for missing or incorrect parameters, and
`422` for evaluation errors such as `duplicate time series on the left
side`, or when the request hits a [resource
limit](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#resource-usage-limits).
`-replay.continueWithExecutionErr` should work with both status codes.
2026-08-11 22:59:36 +02:00
Zhu Jiekun
8f4fdf0ae3 chore: unify the management of secret flags by app
fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11295
2026-08-11 22:58:16 +02:00
f41gh7
339a1b355c app/vminsert|vmselect|vmagent: enable --enableMultitenancyViaHeaders by default
This change aligns VictoriaMetrics multitenancy behavior with
VictoriaLogs and VictoriaTraces.

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11365
2026-08-11 22:54:14 +02:00
24 changed files with 179 additions and 122 deletions

View File

@@ -63,6 +63,7 @@ func main() {
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
envflag.Parse()
initSecretFlags()
buildinfo.Init()
logger.Init()
@@ -170,3 +171,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
`
flagutil.Usage(s)
}
// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
func initSecretFlags() {
pushmetrics.InitSecretFlags()
vmselect.InitSecretFlags()
}

View File

@@ -84,7 +84,7 @@ var (
maxLabelNameLen = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
maxLabelValueLen = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", false, "Enables multitenancy via HTTP headers. "+
enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", true, "Enables multitenancy via HTTP headers. "+
"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy")
)
@@ -115,7 +115,7 @@ func main() {
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
envflag.Parse()
remotewrite.InitSecretFlags()
initSecretFlags()
buildinfo.Init()
logger.Init()
opentelemetry.Init()
@@ -843,3 +843,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
`
flagutil.Usage(s)
}
// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
func initSecretFlags() {
remotewrite.InitSecretFlags()
pushmetrics.InitSecretFlags()
}

View File

@@ -156,7 +156,8 @@ var maxQueues = cgroup.AvailableCPUs() * 16
const persistentQueueDirname = "persistent-queue"
// InitSecretFlags must be called after flag.Parse and before any logging.
// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
// It should run before logger initialization and package Init() (if exists).
func InitSecretFlags() {
if !*showRemoteWriteURL {
// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.

View File

@@ -60,7 +60,8 @@ var (
`Only valid for VictoriaMetrics as the datasource.`)
)
// InitSecretFlags must be called after flag.Parse and before any logging
// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
// It should run before logger initialization and package Init() (if exists).
func InitSecretFlags() {
if !*showDatasourceURL {
flagutil.RegisterSecretFlag("datasource.url")

View File

@@ -88,10 +88,7 @@ func main() {
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
envflag.Parse()
remoteread.InitSecretFlags()
remotewrite.InitSecretFlags()
datasource.InitSecretFlags()
notifier.InitSecretFlags()
initSecretFlags()
buildinfo.Init()
logger.Init()
@@ -438,3 +435,12 @@ func getLastConfigError() error {
defer lastConfigErrMu.RUnlock()
return lastConfigErr
}
// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
func initSecretFlags() {
remoteread.InitSecretFlags()
remotewrite.InitSecretFlags()
datasource.InitSecretFlags()
notifier.InitSecretFlags()
pushmetrics.InitSecretFlags()
}

View File

@@ -189,7 +189,8 @@ func Init(extLabels map[string]string, extURL string) error {
return nil
}
// InitSecretFlags must be called after flag.Parse and before any logging
// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
// It should run before logger initialization and package Init() (if exists).
func InitSecretFlags() {
if !*showNotifierURL {
flagutil.RegisterSecretFlag("notifier.url")

View File

@@ -56,7 +56,8 @@ var (
oauth2Scopes = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
)
// InitSecretFlags must be called after flag.Parse and before any logging
// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
// It should run before logger initialization and package Init() (if exists).
func InitSecretFlags() {
if !*showRemoteReadURL {
flagutil.RegisterSecretFlag("remoteRead.url")

View File

@@ -57,7 +57,8 @@ var (
oauth2Scopes = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
)
// InitSecretFlags must be called after flag.Parse and before any logging
// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
// It should run before logger initialization and package Init() (if exists).
func InitSecretFlags() {
if !*showRemoteWriteURL {
flagutil.RegisterSecretFlag("remoteWrite.url")

View File

@@ -30,7 +30,8 @@ var (
"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, "+
"which can happen due to an expression syntax error or a resource limit being hit.")
)
func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {

View File

@@ -132,9 +132,10 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
var esc *httpserver.ErrorWithStatusCode
if errors.As(err, &esc) {
statusCode := esc.StatusCode
// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
if statusCode == http.StatusUnprocessableEntity {
// if the status code is 400 or 422, the query failed due to reasons such as an expression syntax error or a resource limit being hit,
// rather than datasource unavailability.
// Continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
if statusCode == http.StatusUnprocessableEntity || statusCode == http.StatusBadRequest {
if continueWithExecutionErr {
logger.Errorf("rule %q: %s", r, err)
return 0, nil

View File

@@ -96,6 +96,7 @@ func main() {
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
envflag.Parse()
initSecretFlags()
buildinfo.Init()
logger.Init()
@@ -911,3 +912,8 @@ func slowdownUnauthorizedResponse(r *http.Request) {
}
timerpool.Put(t)
}
// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
func initSecretFlags() {
pushmetrics.InitSecretFlags()
}

View File

@@ -47,9 +47,8 @@ func main() {
// Write flags and help message to stdout, since it is easier to grep or pipe.
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
flagutil.RegisterSecretFlag("snapshot.createURL")
flagutil.RegisterSecretFlag("snapshot.deleteURL")
envflag.Parse()
initSecretFlags()
buildinfo.Init()
logger.Init()
@@ -273,3 +272,10 @@ func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
}
return fs, nil
}
// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
func initSecretFlags() {
flagutil.RegisterSecretFlag("snapshot.createURL")
flagutil.RegisterSecretFlag("snapshot.deleteURL")
pushmetrics.InitSecretFlags()
}

View File

@@ -47,6 +47,7 @@ func main() {
start := time.Now()
beforeFn := func(c *cli.Context) error {
flag.Parse()
initSecretFlags()
logger.Init()
isSilent = c.Bool(globalSilent)
if c.Bool(globalDisableProgressBar) {
@@ -619,3 +620,8 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
Backoff: bf,
}, nil
}
// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
func initSecretFlags() {
pushmetrics.InitSecretFlags()
}

View File

@@ -38,6 +38,7 @@ func main() {
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
envflag.Parse()
initSecretFlags()
buildinfo.Init()
logger.Init()
@@ -112,3 +113,8 @@ func newSrcFS(ctx context.Context) (common.RemoteFS, error) {
}
return fs, nil
}
// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
func initSecretFlags() {
pushmetrics.InitSecretFlags()
}

View File

@@ -59,6 +59,12 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
initVMUIConfig()
vmalertproxy.Init(*vmalertProxyURL)
}
// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
// It should run before logger initialization and package Init() (if exists).
func InitSecretFlags() {
flagutil.RegisterSecretFlag("vmalert.proxyURL")
}

View File

@@ -135,7 +135,7 @@ func tenantViaURL(addr, prefix, tenant, suffix string) string {
}
// tenantViaHeaders returns path in cluster's URL format where tenant is omitted in URL
// Only supported if -enableMultitenancyViaHeaders is specified
// Only supported if -enableMultitenancyViaHeaders is enabled
func tenantViaHeaders(addr, prefix, suffix string) string {
return fmt.Sprintf("http://%s/%s/%s", addr, prefix, suffix)
}

View File

@@ -25,12 +25,10 @@ func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
})
vminsert := tc.MustStartVminsert("vminsert", []string{
"-storageNode=" + vmstorage.VminsertAddr(),
"-enableMultitenancyViaHeaders",
})
vmselect := tc.MustStartVmselect("vmselect", []string{
"-storageNode=" + vmstorage.VmselectAddr(),
"-search.tenantCacheExpireDuration=0",
"-enableMultitenancyViaHeaders",
})
multitenant := make(http.Header)

View File

@@ -594,7 +594,6 @@ func TestSingleVMAgentMultitenancy(t *testing.T) {
fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-multitenancy",
"-enableMultitenantHandlers",
"-enableMultitenancyViaHeaders",
})
vmagent.APIV1ImportPrometheus(t, []string{

View File

@@ -95,16 +95,15 @@ See also multitenancy [via headers](#multitenancy-via-headers) and [via labels](
### Multitenancy via headers
By default, VictoriaMetrics allows specifying `accountID` and `projectID` only in the request URL.
With `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag enabled (enabled by default {{% available_from "#" %}})
tenant ID can be specified via HTTP headers `AccountID` and `ProjectID`. This flag needs to be enabled on vminserts and vmselects.
Set `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag to support
specifying `accountID` and `projectID` via HTTP headers `AccountID` and `ProjectID` respectively.
This flag needs to be specified separately for vminserts and vmselects.
When `--enableMultitenancyViaHeaders` is enabled, [URL format](#url-format) can be simplified to the following:
With `--enableMultitenancyViaHeaders` enabled [URL format](#url-format) can be simplified to the following:
- `http://<vminsert>:8480/insert/<suffix>` for writes
- `http://<vmselect>:8481/select/prometheus/<suffix>` for reads
> Set --enableMultitenancyViaHeaders=false to disable simplified URL format.
For example, the following query will only select metric `up` from `accountID=2` and `projectID=3`:
```
curl 'https://<vmselect>:8481/select/prometheus/api/v1/query' \

View File

@@ -26,6 +26,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
**Update Note 1:** `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: default value of `-enableMultitenancyViaHeaders` command-line flag has changed from `false` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default. With this change, mentioned components will start supporting URLs with omitted tenant ID in the path: `https://<vmselect>:8481/select/prometheus/api/v1/query` will become a valid URL. To disable multitenancy via headers and simplified URLs set `--enableMultitenancyViaHeaders=false` on vmagent, vminsert and vmselect.
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): extend `-replay.continueWithExecutionErr` to also handle the `400 Bad Request` response code, since it is used for Prometheus querying API requests when request parameters are missing or incorrect. See [#11352](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11352).
* FEATURE: `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: set default value of `-enableMultitenancyViaHeaders` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default, aligning VictoriaMetrics multitenancy behavior with [multitenancy in VictoriaLogs](https://docs.victoriametrics.com/victorialogs/#multitenancy). See related ticket [#11365](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11365).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add an option to customize the favicon color. This makes it easier to distinguish between different installations opened in multiple browser tabs. See [#11329](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11329).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly parse small fractional Unix timestamps in timestamp args such as `start` and `end` in `/api/v1/query_range` and `--vm-native-filter-time-start` and `--vm-native-filter-time-end` in `vmctl`. Previously, fractional Unix timestamps with the integer part below `9223372` were interpreted with the wrong unit, for example `12.0` was parsed as `12000` seconds instead of `12` seconds. See [#11324](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11324).

View File

@@ -6,16 +6,15 @@ build:
sitemap:
disable: true
---
## Data model
### What is a metric
Simply put, `metric` is a numeric measure or observation of something.
The most common use cases for metrics are:
The most common use-cases for metrics are:
- check how the system behaves at a particular time period;
- check how the system behaves at the particular time period;
- correlate behavior changes to other measurements;
- observe or forecast trends;
- trigger events (alerts) if the metric exceeds a threshold.
@@ -26,7 +25,7 @@ Let's start with an example. To track how many requests our application serves,
name `requests_total`.
You can be more specific here by saying `requests_success_total` (for only successful requests)
or `request_errors_total` (for requests which failed). Choosing a metric name is very important and is supposed to clarify
or `request_errors_total` (for requests which failed). Choosing a metric name is very important and supposed to clarify
what is actually measured to every person who reads it, just like **variable names** in programming.
#### Labels
@@ -55,14 +54,14 @@ requests_total{path="/", code="200"}
Labels can be automatically attached to the [time series](#time-series)
written via [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics)
or [Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/).
VictoriaMetrics supports enforcing label filters for the [query API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-enhancements)
VictoriaMetrics supports enforcing of label filters for [query API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-enhancements)
to emulate data isolation. However, the real data isolation can be achieved via [multi-tenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).
#### Time series
A combination of a metric name and its labels defines a `time series`. For example,
`requests_total{path="/", code="200"}` and `requests_total{path="/", code="403"}`
are two different time series because they have different values for the `code` label.
are two different time series because they have different values for `code` label.
The number of unique time series has an impact on database resource usage.
See [what is an active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) and
@@ -70,8 +69,8 @@ See [what is an active time series](https://docs.victoriametrics.com/victoriamet
#### Cardinality
The number of unique [time series](#time-series) is named `cardinality`. Having too many unique time series is named `high cardinality`.
High cardinality may result in increased resource usage in VictoriaMetrics.
The number of unique [time series](#time-series) is named `cardinality`. Too big number of unique time series is named `high cardinality`.
High cardinality may result in increased resource usage at VictoriaMetrics.
See [these docs](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-cardinality) for more details.
#### Raw samples
@@ -109,13 +108,13 @@ of the [time series](https://docs.victoriametrics.com/victoriametrics/keyconcept
| requests_total{path="/health", code="200"} | 4 | 1676297730 |
....
```
Here we have a time series `requests_total{path="/health", code="200"}` which has a value updated every `30s`.
This means its resolution is also `30s`.
Here we have a time series `requests_total{path="/health", code="200"}` which has a value update each `30s`.
This means, its resolution is also a `30s`.
> In terms of [pull model](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#pull-model), resolution is equal
> to `scrape_interval` and is controlled by the monitoring system (server).
> For [push model](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#push-model), resolution is an interval between
> sample timestamps and is controlled by a client (metrics collector).
> samples timestamps and is controlled by a client (metrics collector).
Try to keep time series resolution consistent, since some [MetricsQL](#metricsql) functions may expect it to be so.
@@ -127,10 +126,10 @@ type exists specifically to help users to understand how the metric was measured
#### Counter
A counter is a metric that counts an event. Its value increases or stays the same over time.
It cannot decrease in the general case. The only exception is, e.g., `counter reset`,
Counter is a metric, which counts some events. Its value increases or stays the same over time.
It cannot decrease in general case. The only exception is e.g. `counter reset`,
when the metric resets to zero. The `counter reset` can occur when the service, which exposes the counter, restarts.
So, the `counter` metric shows the number of observed events since the service started.
So, the `counter` metric shows the number of observed events since the service start.
In programming, `counter` is a variable that you **increment** each time something happens.
@@ -140,7 +139,7 @@ In programming, `counter` is a variable that you **increment** each time somethi
above is that time series `vm_http_requests_total{instance="localhost:8428", job="victoriametrics", path="api/v1/query_range"}`
was rapidly changing from 1:38 pm to 1:39 pm, then there were no changes until 1:41 pm.
A counter is used for measuring the number of events, like the number of requests, errors, logs, messages, etc.
Counter is used for measuring the number of events, like the number of requests, errors, logs, messages, etc.
The most common [MetricsQL](#metricsql) functions used with counters are:
* [rate](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate) - calculates the average per-second speed of metric change.
@@ -149,7 +148,7 @@ The most common [MetricsQL](#metricsql) functions used with counters are:
time period specified in square brackets.
For example, `increase(requests_total[1h])` shows the number of requests served over the last hour.
It is OK to have fractional counters. For example, the `request_duration_seconds_sum` counter may sum the durations of all the requests.
It is OK to have fractional counters. For example, `request_duration_seconds_sum` counter may sum the durations of all the requests.
Every duration may have a fractional value in seconds, e.g. `0.5` of a second. So the cumulative sum of all the request durations
may be fractional too.
@@ -163,12 +162,12 @@ Gauge is used for measuring a value that can go up and down:
![gauge](gauge.webp)
The metric `process_resident_memory_anon_bytes` on the graph shows the memory usage of the application at every given time.
It is changing frequently, going up and down, showing how the process allocates and frees the memory.
It is changing frequently, going up and down showing how the process allocates and frees the memory.
In programming, `gauge` is a variable to which you **set** a specific value as it changes.
Gauge is used in the following scenarios:
* measuring temperature, memory usage, disk usage, etc;
* measuring temperature, memory usage, disk usage etc;
* storing the state of some process. For example, gauge `config_reloaded_successful` can be set to `1` if everything is
good, and to `0` if configuration failed to reload;
* storing the timestamp when the event happened. For example, `config_last_reload_success_timestamp_seconds`
@@ -179,11 +178,11 @@ and [rollup functions](https://docs.victoriametrics.com/victoriametrics/metricsq
#### Histogram
A histogram is a set of [counter](#counter) metrics with different `vmrange` or `le` labels.
Histogram is a set of [counter](#counter) metrics with different `vmrange` or `le` labels.
The `vmrange` or `le` labels define measurement boundaries of a particular bucket.
When the observed measurement hits a particular bucket, then the corresponding counter is incremented.
Histogram buckets usually have a `_bucket` suffix in their names.
Histogram buckets usually have `_bucket` suffix in their names.
For example, VictoriaMetrics tracks the distribution of rows processed per query with the `vm_rows_read_per_query` histogram.
The exposition format for this histogram has the following form:
@@ -201,10 +200,10 @@ The `vm_rows_read_per_query_bucket{vmrange="4.084e+02...4.642e+02"} 2` line mean
that there were 2 queries with the number of rows in the range `(408.4 - 464.2]`
since the last VictoriaMetrics start.
The counters ending with the `_bucket` suffix allow estimating arbitrary percentiles
The counters ending with `_bucket` suffix allow estimating arbitrary percentile
for the observed measurement with the help of [histogram_quantile](https://docs.victoriametrics.com/victoriametrics/metricsql/#histogram_quantile)
function. For example, the following query returns the estimated 99th percentile
on the number of rows read per query during the last hour (see `1h` in square brackets):
on the number of rows read per each query during the last hour (see `1h` in square brackets):
```metricsql
histogram_quantile(0.99, sum(increase(vm_rows_read_per_query_bucket[1h])) by (vmrange))
@@ -216,15 +215,15 @@ This query works in the following way:
number of events over the last hour.
1. The `sum(...) by (vmrange)` calculates per-bucket events by summing per-instance buckets
with the same `vmrange` values.
1. The `histogram_quantile(0.99, ...)` calculates the 99th percentile over `vmrange` buckets returned at step 2.
1. The `histogram_quantile(0.99, ...)` calculates 99th percentile over `vmrange` buckets returned at step 2.
Histogram metric type exposes two additional counters ending with `_sum` and `_count` suffixes:
- the `vm_rows_read_per_query_sum` is a sum of all the observed measurements,
e.g., the sum of rows served by all the queries since the last VictoriaMetrics start.
e.g. the sum of rows served by all the queries since the last VictoriaMetrics start.
- the `vm_rows_read_per_query_count` is the total number of observed events,
e.g., the total number of observed queries since the last VictoriaMetrics start.
e.g. the total number of observed queries since the last VictoriaMetrics start.
These counters allow calculating the average measurement value on a particular lookbehind window.
For example, the following query calculates the average number of rows read per query
@@ -234,7 +233,7 @@ during the last 5 minutes (see `5m` in square brackets):
increase(vm_rows_read_per_query_sum[5m]) / increase(vm_rows_read_per_query_count[5m])
```
The `vm_rows_read_per_query` histogram may be used in a Go application in the following way
The `vm_rows_read_per_query` histogram may be used in Go application in the following way
by using the [github.com/VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) package:
```go
@@ -247,7 +246,7 @@ for _, query := range queries {
}
```
Now let's see what happens each time `rowsReadPerQuery.Update` is called:
Now let's see what happens each time when `rowsReadPerQuery.Update` is called:
* counter `vm_rows_read_per_query_sum` is incremented by value of `len(query.Rows)` expression;
* counter `vm_rows_read_per_query_count` increments by 1;
@@ -263,7 +262,7 @@ and calculating [quantiles](https://prometheus.io/docs/practices/histograms/#qua
Grafana doesn't understand buckets with `vmrange` labels, so the [prometheus_buckets](https://docs.victoriametrics.com/victoriametrics/metricsql/#prometheus_buckets)
function must be used for converting buckets with `vmrange` labels to buckets with `le` labels before building heatmaps in Grafana.
Histograms are usually used for measuring the distribution of latency, sizes of elements (batch size, for example), etc. There are two
Histograms are usually used for measuring the distribution of latency, sizes of elements (batch size, for example) etc. There are two
implementations of a histogram supported by VictoriaMetrics:
1. [Prometheus histogram](https://prometheus.io/docs/practices/histograms/). The canonical histogram implementation is
@@ -272,7 +271,7 @@ implementations of a histogram supported by VictoriaMetrics:
histogram requires a user to define ranges (`buckets`) statically.
1. [VictoriaMetrics histogram](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
supported by [VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) instrumentation library.
VictoriaMetrics histogram automatically handles bucket boundaries, so users don't need to think about them.
Victoriametrics histogram automatically handles bucket boundaries, so users don't need to think about them.
We recommend reading the following articles before you start using histograms:
@@ -304,7 +303,7 @@ The visualization of summaries is pretty straightforward:
Such an approach makes summaries easier to use but also puts significant limitations compared to [histograms](#histogram):
- It is impossible to calculate a quantile over multiple summary metrics, e.g. `sum(go_gc_duration_seconds{quantile="0.75"})`,
- It is impossible to calculate quantile over multiple summary metrics, e.g. `sum(go_gc_duration_seconds{quantile="0.75"})`,
`avg(go_gc_duration_seconds{quantile="0.75"})` or `max(go_gc_duration_seconds{quantile="0.75"})`
won't return the expected 75th percentile over `go_gc_duration_seconds` metrics collected from multiple instances
of the application. See [this article](https://latencytipoftheday.blogspot.de/2014/06/latencytipoftheday-you-cant-average.html) for details.
@@ -314,16 +313,16 @@ Such an approach makes summaries easier to use but also puts significant limitat
- It is impossible to calculate quantiles for measurements collected over an arbitrary time range. Usually, `summary`
quantiles are calculated over a fixed time range such as the last 5 minutes.
Summaries are usually used for tracking the pre-defined percentiles for latency, sizes of elements (batch size, for example), etc.
Summaries are usually used for tracking the pre-defined percentiles for latency, sizes of elements (batch size, for example) etc.
### Instrumenting application with metrics
As was said at the beginning of the [types of metrics](#types-of-metrics) section, metric type defines how it was
measured. VictoriaMetrics TSDB doesn't know about metric types. All it sees are metric names, labels, values, and timestamps.
What these metrics are, what they measure, and how - all these depend on the application which emits them.
What are these metrics, what do they measure, and how - all this depends on the application which emits them.
To instrument your application with metrics compatible with VictoriaMetrics, we recommend
using the [github.com/VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) package.
To instrument your application with metrics compatible with VictoriaMetrics we recommend
using [github.com/VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) package.
See more details on how to use it in [this article](https://victoriametrics.medium.com/how-to-monitor-go-applications-with-victoriametrics-c04703110870).
VictoriaMetrics is also compatible with [Prometheus client libraries for metrics instrumentation](https://prometheus.io/docs/instrumenting/clientlibs/).
@@ -332,20 +331,20 @@ VictoriaMetrics is also compatible with [Prometheus client libraries for metrics
We recommend following [Prometheus naming convention for metrics](https://prometheus.io/docs/practices/naming/). There
are no strict restrictions, so any metric name and labels are accepted by VictoriaMetrics.
But this convention helps to keep names meaningful, descriptive, and clear to other people.
Following the convention is a good practice.
But the convention helps to keep names meaningful, descriptive, and clear to other people.
Following convention is a good practice.
#### Labels
Every measurement can contain an arbitrary number of `key="value"` labels. The good practice is to keep this number limited.
Otherwise, it would be difficult to deal with measurements containing a large number of labels.
Otherwise, it would be difficult to deal with measurements containing a big number of labels.
By default, VictoriaMetrics limits the number of labels per measurement to `40` and drops other labels.
This limit can be changed via the `-maxLabelsPerTimeseries` command-line flag if necessary (but this isn't recommended).
This limit can be changed via `-maxLabelsPerTimeseries` command-line flag if necessary (but this isn't recommended).
Every label value can contain an arbitrary string value. The good practice is to use short and meaningful label values to
describe the attribute of the metric, not to tell the story about it. For example, label-value pair
`environment="prod"` is OK, but `log_message="long log message with a lot of details..."` is not OK. By default,
VictoriaMetrics limits label values to 4KiB. This limit can be changed via the `-maxLabelValueLen` command-line flag.
`environment="prod"` is ok, but `log_message="long log message with a lot of details..."` is not ok. By default,
VictoriaMetrics limits label's value size with 4KiB. This limit can be changed via `-maxLabelValueLen` command-line flag.
It is very important to keep under control the number of unique label values, since every unique label value
leads to a new [time series](#time-series). Try to avoid using volatile label values such as session ID or query ID in order to
@@ -357,7 +356,7 @@ avoid excessive resource usage and database slowdown.
supports [multi-tenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
for data isolation.
Multi-tenancy can be emulated for the [single-server](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)
Multi-tenancy can be emulated for [single-server](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)
version of VictoriaMetrics by adding [labels](#labels) on [write path](#write-data)
and enforcing [labels filtering](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-enhancements)
on [read path](#query-data).
@@ -392,10 +391,10 @@ It is allowed to push/write metrics to [single-node VictoriaMetrics](https://doc
to [cluster component vminsert](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#architecture-overview)
and to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
The pros of the push model:
The pros of push model:
* Simpler configuration at VictoriaMetrics side - there is no need to configure VictoriaMetrics with locations of the monitored applications.
There is no need for complex [service discovery schemes](https://docs.victoriametrics.com/victoriametrics/sd_configs/).
There is no need in complex [service discovery schemes](https://docs.victoriametrics.com/victoriametrics/sd_configs/).
* Simpler security setup - there is no need to set up access from VictoriaMetrics to each monitored application.
See [Foiled by the Firewall: A Tale of Transition From Prometheus to VictoriaMetrics](https://www.percona.com/blog/2020/12/01/foiled-by-the-firewall-a-tale-of-transition-from-prometheus-to-victoriametrics/)
@@ -407,18 +406,18 @@ The cons of push protocol:
Every application needs to be individually configured with the address of the monitoring system
for metrics delivery. It also needs to be configured with the interval between metric pushes
and the strategy in case of metric delivery failure.
* Non-trivial setup for metrics delivery into multiple monitoring systems.
* Non-trivial setup for metrics' delivery into multiple monitoring systems.
* It may be hard to tell whether the application went down or just stopped sending metrics for a different reason.
* Applications can overload the monitoring system by pushing metrics at too short intervals.
### Pull model
The pull model is an approach popularized by [Prometheus](https://prometheus.io/), where the monitoring system decides when
Pull model is an approach popularized by [Prometheus](https://prometheus.io/), where the monitoring system decides when
and where to pull metrics from:
![pull model](pull_model.webp)
In the pull model, the monitoring system needs to be aware of all the applications it needs to monitor. The metrics are
In pull model, the monitoring system needs to be aware of all the applications it needs to monitor. The metrics are
scraped (pulled) from the known applications (aka `scrape targets`) via HTTP protocol on a regular basis (aka `scrape_interval`).
VictoriaMetrics supports discovering Prometheus-compatible targets and scraping metrics from them in the same way as Prometheus does -
@@ -432,7 +431,7 @@ The pros of the pull model:
* Easier to debug - VictoriaMetrics knows about all the monitored applications (aka `scrape targets`).
The `up == 0` query instantly shows unavailable scrape targets.
The actual information about scrape targets is available at `http://victoriametrics:8428/targets` and `http://vmagent:8429/targets`.
* The monitoring system controls the frequency of metrics scraping, so it is easier to control its load.
* Monitoring system controls the frequency of metrics' scrape, so it is easier to control its load.
* Applications aren't aware of the monitoring system and don't need to implement the logic for metrics delivery.
The cons of the pull model:
@@ -449,13 +448,13 @@ The most common approach for data collection is using both models:
![data collection](data_collection.webp)
In this approach, the additional component is used - [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/). Vmagent is
a lightweight agent whose main purpose is to collect, filter, relabel, and deliver metrics to VictoriaMetrics.
In this approach the additional component is used - [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/). Vmagent is
a lightweight agent whose main purpose is to collect, filter, relabel and deliver metrics to VictoriaMetrics.
It supports all [push](#push-model) and [pull](#pull-model) protocols mentioned above.
The basic monitoring setup of VictoriaMetrics and vmagent is described
in the [example docker-compose manifest](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#readme).
In this example, vmagent [scrapes a list of targets](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-single.yml)
In this example vmagent [scrapes a list of targets](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-single.yml)
and [forwards collected data to VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/9751ea10983d42068487624849cac7ad6fd7e1d8/deployment/docker/compose-vm-single.yml#L16).
VictoriaMetrics is then used as a [datasource for Grafana](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/provisioning/datasources/prometheus/single.yml)
installation for querying collected data.
@@ -481,7 +480,7 @@ The API consists of two main handlers for serving [instant queries](#instant-que
### Instant query
An instant query executes the `query` expression at the given `time`:
Instant query executes the `query` expression at the given `time`:
```
GET | POST /api/v1/query?query=...&time=...&step=...&timeout=...
@@ -498,13 +497,13 @@ Params:
For example, the request `/api/v1/query?query=up&step=1m` looks for the last written raw sample for the metric `up`
in the `(now()-1m, now()]` interval (the first millisecond is not included). If omitted, `step` is set to `5m` (5 minutes)
by default.
* `timeout` - optional query timeout. For example, `timeout=5s`. The query is canceled when the timeout is reached.
By default, the timeout is set to the value of the `-search.maxQueryDuration` command-line flag passed to the single-node VictoriaMetrics
or to the `vmselect` component of the VictoriaMetrics cluster.
* `timeout` - optional query timeout. For example, `timeout=5s`. Query is canceled when the timeout is reached.
By default the timeout is set to the value of `-search.maxQueryDuration` command-line flag passed to single-node VictoriaMetrics
or to `vmselect` component of VictoriaMetrics cluster.
The result of an Instant query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
matching the filter in the `query` expression. Each returned series contains exactly one `(timestamp, value)` entry,
where `timestamp` equals the `time` query arg, while the `value` contains the `query` result at the requested `time`.
The result of Instant query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
matching the filter in `query` expression. Each returned series contains exactly one `(timestamp, value)` entry,
where `timestamp` equals to the `time` query arg, while the `value` contains `query` result at the requested `time`.
To understand how instant queries work, let's begin with a data sample:
@@ -531,7 +530,7 @@ ranging from 1m to 3m. If we plot this data sample on the graph, it will have th
{width="500"}
To get the value of the `foo_bar` series at some specific moment of time, for example `2022-05-10T08:03:00Z`, in
VictoriaMetrics, we need to issue an **instant query**:
VictoriaMetrics we need to issue an **instant query**:
```sh
curl "http://<victoria-metrics-addr>/api/v1/query?query=foo_bar&time=2022-05-10T08:03:00.000Z"
@@ -596,13 +595,13 @@ Params:
The `query` is executed at `start`, `start+step`, `start+2*step`, ..., `start+N*step` timestamps,
where `N` is the whole number of steps that fit between `start` and `end`.
`end` is included only when it equals to `start+N*step`.
If the `step` isn't set, then it defaults to `5m` (5 minutes).
* `timeout` - optional query timeout. For example, `timeout=5s`. The query is canceled when the timeout is reached.
By default, the timeout is set to the value of the `-search.maxQueryDuration` command-line flag passed to the single-node VictoriaMetrics
or to the `vmselect` component in a VictoriaMetrics cluster.
If the `step` isn't set, then it default to `5m` (5 minutes).
* `timeout` - optional query timeout. For example, `timeout=5s`. Query is canceled when the timeout is reached.
By default the timeout is set to the value of `-search.maxQueryDuration` command-line flag passed to single-node VictoriaMetrics
or to `vmselect` component in VictoriaMetrics cluster.
The result of a Range query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
matching the filter in the `query` expression. Each returned series contains `(timestamp, value)` results for the `query` executed
The result of Range query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
matching the filter in `query` expression. Each returned series contains `(timestamp, value)` results for the `query` executed
at `start`, `start+step`, `start+2*step`, ..., `start+N*step` timestamps. In other words, Range query is an [Instant query](#instant-query)
executed independently at `start`, `start+step`, ..., `start+N*step` timestamps with the only difference that an instant query
does not return `ephemeral` samples (see below). Instead, if the database does not contain any samples for the requested time and step,
@@ -706,7 +705,7 @@ In response, VictoriaMetrics returns `17` sample-timestamp pairs for the series
from `2022-05-10T07:59:00Z` to `2022-05-10T08:17:00Z`. But, if we take a look at the original data sample again, we'll
see that it contains only 13 raw samples. What happens here is that the range query is actually
an [instant query](#instant-query) executed `1 + (start-end)/step` times on the time range from `start` to `end`. If we plot
this request in VictoriaMetrics, the graph will be shown as follows:
this request in VictoriaMetrics the graph will be shown as the following:
![range query](range_query.webp)
{width="500"}
@@ -721,13 +720,13 @@ This behavior of adding ephemeral data points comes from the specifics of the [p
* Scrape may be skipped if the monitoring system is overloaded.
* Scrape may fail due to network issues.
According to these specifics, the range query assumes that if there is a missing raw sample, then it is likely a missed
According to these specifics, the range query assumes that if there is a missing raw sample then it is likely a missed
scrape, so it fills it with the previous raw sample. The same will work for cases when `step` is lower than the actual
interval between samples. In fact, if we set `step=1s` for the same request, we'll get about 1 thousand data points in
response, where most of them are `ephemeral`.
Sometimes, the lookbehind window for locating the datapoint isn't big enough and the graph will contain a gap. For range
queries, the lookbehind window isn't equal to the `step` parameter. It is calculated as the median of the intervals between
queries, lookbehind window isn't equal to the `step` parameter. It is calculated as the median of the intervals between
the last 20 raw samples in the requested time range. In this way, VictoriaMetrics automatically adjusts the lookbehind
window to fill gaps and detect stale series at the same time.
@@ -735,7 +734,7 @@ Range queries are mostly used for plotting time series data over specified time
useful in the following scenarios:
* Track the state of a metric on the given time interval;
* Correlate changes between multiple metrics over the time interval;
* Correlate changes between multiple metrics on the time interval;
* Observe trends and dynamics of the metric change.
If you need to export raw samples from VictoriaMetrics, then take a look at [export APIs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-time-series).
@@ -746,7 +745,7 @@ By default, Victoria Metrics does not immediately return the recently written sa
written prior to the time specified by the `-search.latencyOffset` command-line flag, which has a default offset of 30 seconds.
This is true for both `query` and `query_range` and may give the impression that data is written to the VM with a 30-second delay.
This flag prevents inconsistent results due to the fact that only part of the values are scraped in the last scrape interval.
This flag prevents from non-consistent results due to the fact that only part of the values are scraped in the last scrape interval.
Here is an illustration of a potential problem when `-search.latencyOffset` is set to zero:
@@ -759,12 +758,12 @@ duration throughout the `-search.latencyOffset` duration:
![with latency offset](with_latencyOffset.webp)
{width="1000"}
It can be overridden on a per-query basis via the `latency_offset` query arg.
It can be overridden on per-query basis via `latency_offset` query arg.
VictoriaMetrics buffers recently ingested samples in memory for up to a few seconds and then periodically flushes these samples to disk.
This buffering improves data ingestion performance. The buffered samples are invisible in query results, even if the `-search.latencyOffset` command-line flag is set to 0,
This buffering improves data ingestion performance. The buffered samples are invisible in query results, even if `-search.latencyOffset` command-line flag is set to 0,
or if `latency_offset` query arg is set to 0.
You can send a GET request to the `/internal/force_flush` HTTP handler at a single-node VictoriaMetrics
You can send GET request to `/internal/force_flush` http handler at single-node VictoriaMetrics
or to `vmstorage` at [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/)
in order to forcibly flush the buffered samples to disk, so they become visible for querying. The `/internal/force_flush` handler
is provided for debugging and testing purposes only. Do not call it in production, since this may significantly slow down data ingestion
@@ -772,7 +771,7 @@ performance and increase resource usage.
### MetricsQL
VictoriaMetrics provides a special query language for executing read queries - [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/).
VictoriaMetrics provide a special query language for executing read queries - [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/).
It is a [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics)-like query language with a powerful set of
functions and features for working specifically with time series data. MetricsQL is backward-compatible with PromQL,
so it shares most of the query concepts. The basic concepts for PromQL and MetricsQL are
@@ -780,7 +779,7 @@ described [here](https://valyala.medium.com/promql-tutorial-for-beginners-9ab455
#### Filtering
In sections [instant query](#instant-query) and [range query](#range-query), we've already used MetricsQL to get data for
In sections [instant query](#instant-query) and [range query](#range-query) we've already used MetricsQL to get data for
metric `foo_bar`. It is as simple as just writing a metric name in the query:
```metricsql
@@ -794,14 +793,14 @@ requests_total{path="/", code="200"}
requests_total{path="/", code="403"}
```
To select only time series with a specific label value, specify the matching filter in curly braces:
To select only time series with specific label value specify the matching filter in curly braces:
```metricsql
requests_total{code="200"}
```
The query above returns all time series with the name `requests_total` and label `code="200"`. We use the operator `=` to
match the label value. For negative matches, use the `!=` operator. Filters also support positive regex matching via `=~`
match label value. For negative match use `!=` operator. Filters also support positive regex matching via `=~`
and negative regex matching via `!~`:
```metricsql
@@ -814,7 +813,7 @@ Filters can also be combined:
requests_total{code=~"200", path="/home"}
```
The query above returns all time series with the `requests_total` name, which simultaneously have labels `code="200"` and `path="/home"`.
The query above returns all time series with `requests_total` name, which simultaneously have labels `code="200"` and `path="/home"`.
#### Filtering by name
@@ -830,7 +829,7 @@ The query above returns series for two metrics: `requests_error_total` and `requ
#### Filtering by multiple "or" filters
[MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) supports selecting time series that match at least one of multiple "or" filters.
[MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) supports selecting time series, which match at least one of multiple "or" filters.
Such filters must be delimited by `or` inside curly braces. For example, the following query selects time series with
`{job="app1",env="prod"}` or `{job="app2",env="dev"}` labels:
@@ -839,7 +838,7 @@ Such filters must be delimited by `or` inside curly braces. For example, the fol
```
The number of `or` groups can be arbitrary. The number of `,`-delimited label filters per each `or` group can be arbitrary.
Per-group filters are applied with the `and` operation, e.g., they select series simultaneously matching all the filters in the group.
Per-group filters are applied with `and` operation, e.g. they select series simultaneously matching all the filters in the group.
This functionality allows passing the selected series to [rollup functions](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions)
such as [rate()](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate)
@@ -850,7 +849,7 @@ rate({job="app1",env="prod" or job="app2",env="dev"}[5m])
```
If you need to select series matching multiple filters for the same label, then it is better from a performance PoV
If you need to select series matching multiple filters for the same label, then it is better from performance PoV
to use regexp filter `{label=~"value1|...|valueN"}` instead of `{label="value1" or ... or label="valueN"}`.
@@ -879,8 +878,8 @@ query may break or may lead to incorrect results. The basics of the matching rul
* MetricsQL engine strips metric names from all the time series on the left and right side of the arithmetic operation
without touching labels.
* For each time series on the left side, the MetricsQL engine searches for the corresponding time series on the right side
with the same set of labels, applies the operation for each data point, and returns the resulting time series with the
* For each time series on the left side MetricsQL engine searches for the corresponding time series on the right side
with the same set of labels, applies the operation for each data point and returns the resulting time series with the
same set of labels. If there are no matches, then the time series is dropped from the result.
* The matching rules may be augmented with `ignoring`, `on`, `group_left` and `group_right` modifiers.
See [these docs](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching) for details.
@@ -897,7 +896,7 @@ MetricsQL supports the following comparison operators:
* less-or-equal - `<=`
These operators may be applied to arbitrary MetricsQL expressions as with arithmetic operators. The result of the
comparison operation is a time series with only matching data points. For instance, the following query would return
comparison operation is time series with only matching data points. For instance, the following query would return
series only for processes where memory usage exceeds `100MB`:
```metricsql
@@ -907,7 +906,7 @@ process_resident_memory_bytes > 100*1024*1024
#### Aggregation and grouping functions
MetricsQL allows aggregating and grouping of time series. Time series are grouped by the given set of labels and then the
given aggregation function is applied individually to each group. For instance, the following query returns
given aggregation function is applied individually per each group. For instance, the following query returns
summary memory usage for each `job`:
```metricsql
@@ -920,14 +919,14 @@ See [docs for aggregate functions in MetricsQL](https://docs.victoriametrics.com
One of the most widely used functions for [counters](#counter)
is [rate](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate). It calculates the average per-second increase rate individually
for each matching time series. For example, the following query shows the average per-second data receive speed
for each monitored `node_exporter` instance, which exposes the `node_network_receive_bytes_total` metric:
per each matching time series. For example, the following query shows the average per-second data receive speed
per each monitored `node_exporter` instance, which exposes the `node_network_receive_bytes_total` metric:
```metricsql
rate(node_network_receive_bytes_total)
```
By default, VictoriaMetrics calculates the `rate` over [raw samples](#raw-samples) on the lookbehind window specified in the `step` parameter
By default, VictoriaMetrics calculates the `rate` over [raw samples](#raw-samples) on the lookbehind window specified in the `step` param
passed either to [instant query](#instant-query) or to [range query](#range-query).
The interval on which `rate` needs to be calculated can be specified explicitly
as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations) in square brackets:
@@ -936,10 +935,10 @@ as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#floa
rate(node_network_receive_bytes_total[5m])
```
In this case, VictoriaMetrics uses the specified lookbehind window - `5m` (5 minutes) - for calculating the average per-second increase rate.
In this case VictoriaMetrics uses the specified lookbehind window - `5m` (5 minutes) - for calculating the average per-second increase rate.
Bigger lookbehind windows usually lead to smoother graphs.
`rate` strips the metric name while leaving all the labels for the inner time series. If you need to keep the metric name,
`rate` strips metric name while leaving all the labels for the inner time series. If you need to keep the metric name,
then add [keep_metric_names](https://docs.victoriametrics.com/victoriametrics/metricsql/#keep_metric_names) modifier
after the `rate(..)`. For example, the following query leaves metric names after calculating the `rate()`:
@@ -953,7 +952,7 @@ rate(node_network_receive_bytes_total) keep_metric_names
VictoriaMetrics has a built-in graphical User Interface for querying and visualizing metrics -
[VMUI](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui).
Open the `http://victoriametrics:8428/vmui` page, type the query, and see the results:
Open `http://victoriametrics:8428/vmui` page, type the query and see the results:
![vmui](vmui.webp)
@@ -964,8 +963,8 @@ in the same way as Grafana queries Prometheus.
## Modify data
VictoriaMetrics stores time series data in [MergeTree](https://en.wikipedia.org/wiki/Log-structured_merge-tree)-like
data structures. While this approach is very efficient for write-heavy databases, it imposes some limitations on data
updates. In short, modifying already written [time series](#time-series) requires rewriting the whole data block where
data structures. While this approach is very efficient for write-heavy databases, it applies some limitations on data
updates. In short, modifying already written [time series](#time-series) requires re-writing the whole data block where
it is stored. Due to this limitation, VictoriaMetrics does not support direct data modification.
### Deletion

View File

@@ -642,7 +642,7 @@ specified via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` c
vmagent can write data to multiple distinct tenants if:
* its `-remoteWrite.url` points to the [VictoriaMetrics cluster multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are both set
* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` (enabled by default {{% available_from "#" %}}) command-line flags are both set
* clients ingest data into vmagent with the tenants specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}
```mermaid

View File

@@ -405,7 +405,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
-remoteWrite.url string
Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
-replay.continueWithExecutionErr
Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
-replay.disableProgressBar
Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
-replay.maxDatapointsPerQuery int

View File

@@ -24,7 +24,9 @@ var (
disableCompression = flag.Bool("pushmetrics.disableCompression", false, "Whether to disable request body compression when pushing metrics to every -pushmetrics.url")
)
func init() {
// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
// It should run before logger initialization and package Init() (if exists).
func InitSecretFlags() {
// The -pushmetrics.url flag can contain basic auth creds, so it mustn't be visible when exposing the flags.
flagutil.RegisterSecretFlag("pushmetrics.url")
}