mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-07-23 17:21:27 +03:00
2edf1729d3
The alert requires path, but it may be absent from the metrics. Ignore metrics without path set
193 lines
10 KiB
YAML
193 lines
10 KiB
YAML
# File contains default list of alerts for VictoriaMetrics single server.
|
|
# The alerts below are just recommendations and may require some updates
|
|
# and threshold calibration according to every specific setup.
|
|
groups:
|
|
# Alerts group for VM single assumes that Grafana dashboard
|
|
# https://grafana.com/grafana/dashboards/10229 is installed.
|
|
# Pls update the `dashboard` annotation according to your setup.
|
|
- name: vmsingle
|
|
interval: 30s
|
|
concurrency: 2
|
|
rules:
|
|
- alert: DiskRunsOutOfSpaceIn3Days
|
|
expr: |
|
|
sum(vm_free_disk_space_bytes) without(path) /
|
|
(
|
|
(rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (
|
|
sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) /
|
|
sum(vm_rows{type!~"indexdb.*"}) without(type)
|
|
)
|
|
+
|
|
rate(vm_new_timeseries_created_total[1d]) * (
|
|
sum(vm_data_size_bytes{type="indexdb/file"}) without(type)/
|
|
sum(vm_rows{type="indexdb/file"}) without(type)
|
|
)
|
|
) < 3 * 24 * 3600 > 0
|
|
for: 30m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
dashboard: "{{ $externalURL }}/d/wNf0q_kZk?viewPanel=53&var-instance={{ $labels.instance }}"
|
|
summary: "Instance {{ $labels.instance }} will run out of disk space soon"
|
|
description: "Taking into account current ingestion rate, free disk space will be enough only
|
|
for {{ $value | humanizeDuration }} on instance {{ $labels.instance }}.\n
|
|
Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
|
|
|
|
- alert: NodeBecomesReadonlyIn3Days
|
|
expr: |
|
|
sum(vm_free_disk_space_bytes - vm_free_disk_space_limit_bytes) without(path) /
|
|
(
|
|
(rate(vm_rows_added_to_storage_total[1d]) - sum(rate(vm_deduplicated_samples_total[1d])) without(type)) * (
|
|
sum(vm_data_size_bytes{type!~"indexdb.*"}) without(type) /
|
|
sum(vm_rows{type!~"indexdb.*"}) without(type)
|
|
)
|
|
+
|
|
rate(vm_new_timeseries_created_total[1d]) * (
|
|
sum(vm_data_size_bytes{type="indexdb/file"}) without(type) /
|
|
sum(vm_rows{type="indexdb/file"}) without(type)
|
|
)
|
|
) < 3 * 24 * 3600 > 0
|
|
for: 30m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=53&var-instance={{ $labels.instance }}"
|
|
summary: "Instance {{ $labels.instance }} will become read-only in 3 days"
|
|
description: "Taking into account current ingestion rate and free disk space
|
|
instance {{ $labels.instance }} is writable for {{ $value | humanizeDuration }}.\n
|
|
Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible."
|
|
|
|
- alert: DiskRunsOutOfSpace
|
|
expr: |
|
|
sum(vm_data_size_bytes) by(job, instance) /
|
|
(
|
|
sum(vm_free_disk_space_bytes) by(job, instance) +
|
|
sum(vm_data_size_bytes) by(job, instance)
|
|
) > 0.8
|
|
for: 30m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
dashboard: "{{ $externalURL }}/d/wNf0q_kZk?viewPanel=53&var-instance={{ $labels.instance }}"
|
|
summary: "Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon"
|
|
description: "Disk utilisation on instance {{ $labels.instance }} is more than 80%.\n
|
|
Having less than 20% of free disk space could cripple merge processes and overall performance.
|
|
Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
|
|
|
|
- alert: RequestErrorsToAPI
|
|
expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "{{ $externalURL }}/d/wNf0q_kZk?viewPanel=35&var-instance={{ $labels.instance }}"
|
|
summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
|
|
description: "Requests to path {{ $labels.path }} are receiving errors.
|
|
Please verify if clients are sending correct requests."
|
|
|
|
- alert: TooHighChurnRate
|
|
expr: |
|
|
(
|
|
sum(rate(vm_new_timeseries_created_total[5m])) by(instance)
|
|
/
|
|
sum(rate(vm_rows_inserted_total[5m])) by(instance)
|
|
) > 0.1
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "{{ $externalURL }}/d/wNf0q_kZk?viewPanel=66&var-instance={{ $labels.instance }}"
|
|
summary: "Churn rate is more than 10% on \"{{ $labels.instance }}\" for the last 15m"
|
|
description: "VM constantly creates new time series on \"{{ $labels.instance }}\".\n
|
|
This effect is known as Churn Rate.\n
|
|
High Churn Rate is tightly connected with database performance and may
|
|
result in unexpected OOM's or slow queries."
|
|
|
|
- alert: TooHighChurnRate24h
|
|
expr: |
|
|
sum(increase(vm_new_timeseries_created_total[24h])) by(instance)
|
|
>
|
|
(sum(vm_cache_entries{type="storage/hour_metric_ids"}) by(instance) * 3)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "{{ $externalURL }}/d/wNf0q_kZk?viewPanel=66&var-instance={{ $labels.instance }}"
|
|
summary: "Too high number of new series on \"{{ $labels.instance }}\" created over last 24h"
|
|
description: "The number of created new time series over last 24h is 3x times higher than
|
|
current number of active series on \"{{ $labels.instance }}\".\n
|
|
This effect is known as Churn Rate.\n
|
|
High Churn Rate is tightly connected with database performance and may
|
|
result in unexpected OOM's or slow queries."
|
|
|
|
- alert: TooHighSlowInsertsRate
|
|
expr: |
|
|
(
|
|
sum(rate(vm_slow_row_inserts_total[5m])) by(instance)
|
|
/
|
|
sum(rate(vm_rows_inserted_total[5m])) by(instance)
|
|
) > 0.05
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "{{ $externalURL }}/d/wNf0q_kZk?viewPanel=68&var-instance={{ $labels.instance }}"
|
|
summary: "Percentage of slow inserts is more than 5% on \"{{ $labels.instance }}\" for the last 15m"
|
|
description: "High rate of slow inserts on \"{{ $labels.instance }}\" may be a sign of resource exhaustion
|
|
for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series.
|
|
See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183"
|
|
|
|
- alert: MetadataCacheUtilizationIsTooHigh
|
|
expr: |
|
|
vm_metrics_metadata_storage_size_bytes / vm_metrics_metadata_storage_max_size_bytes > 0.95
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Metadata cache capacity on {{ $labels.instance }} (job={{ $labels.job }}) is utilized for more than 95% for the last 15min"
|
|
description: "Metadata cache stores meta information about ingested time series - see https://docs.victoriametrics.com/victoriametrics/#metrics-metadata.
|
|
When cache is overutilized, the oldest entries will be dropped out automatically. It may result into incomplete
|
|
response for /api/v1/metadata API calls. It doesn't impact regular queries or alerts. Cache size is controlled
|
|
via -storage.maxMetadataStorageSize cmd-line flag."
|
|
|
|
- alert: MetricNameStatsCacheUtilizationIsTooHigh
|
|
expr: |
|
|
vm_cache_size_bytes{type="storage/metricNamesStatsTracker"} / vm_cache_size_max_bytes{type="storage/metricNamesStatsTracker"} > 0.95
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Cache capacity for tracking metric names usage on {{ $labels.instance }} (job={{ $labels.job }}) is utilized for more than 95% during the last 15min"
|
|
description: "Metric names usage cache stores information about unique metric names and how frequently they are queried - see https://docs.victoriametrics.com/victoriametrics/#track-ingested-metrics-usage.
|
|
When cache is overutilized, it will stop tracking the new metric names. It has no other negative impact.
|
|
Usually, the number of unique metric names is very limited (thousands). The cache can be overutilized only if metric names
|
|
are changing too frequently or if the cache size is too low. There are following ways to mitigate cache overutilization:
|
|
- disable cache via `--storage.trackMetricNamesStats=false` flag, so metric names usage will stop tracking
|
|
- increase the cache size via `--storage.cacheSizeMetricNamesStats` flag
|
|
- reset the cache (see docs for details)"
|
|
|
|
- alert: IndexDBRecordsDrop
|
|
expr: increase(vm_indexdb_items_dropped_total[5m]) > 0
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IndexDB skipped registering items during data ingestion with reason={{ $labels.reason }}."
|
|
description: |
|
|
VictoriaMetrics could skip registering new timeseries during ingestion if they fail the validation process.
|
|
For example, `reason=too_long_item` means that time series cannot exceed 64KB. Please, reduce the number
|
|
of labels or label values for such series. Or enforce these limits via `-maxLabelsPerTimeseries` and
|
|
`-maxLabelValueLen` command-line flags.
|
|
|
|
- alert: TooManyTSIDMisses
|
|
expr: increase(vm_missing_tsids_for_metric_id_total[5m]) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Unexpected TSID misses for job \"{{ $labels.job }}\" ({{ $labels.instance }}) for the last 15 minutes"
|
|
description: |
|
|
Unexpected TSID misses for \"{{ $labels.job }}\" ({{ $labels.instance }}) for the last 15 minutes.
|
|
If this happens after unclean shutdown of VictoriaMetrics process (via \"kill -9\", OOM or power off),
|
|
then this is OK - the alert must go away in a few minutes after the restart.
|
|
Otherwise this may point to the corruption of index data.
|