deployment/docker/rules: add RequestErrorsToUnexpectedPaths alert which focuses on errors without

path set (i.e. invalid authentication)
deployment/docker/rules: update RequestErrorsToAPI alert definition
2026-07-16 22:00:54 +03:00 · 2026-07-13 14:55:00 +02:00 · 2026-07-01 12:32:22 +02:00
3 changed files with 24 additions and 12 deletions
--- a/app/vmalert/config/testdata/rules/rules-replay-good.rules
+++ b/app/vmalert/config/testdata/rules/rules-replay-good.rules
@@ -18,7 +18,7 @@ groups:
    concurrency: 2
    rules:
      - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
--- a/deployment/docker/rules/alerts-cluster.yml
+++ b/deployment/docker/rules/alerts-cluster.yml
@@ -75,7 +75,7 @@ groups:
            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
      - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
@@ -86,6 +86,18 @@ groups:
          description: "Requests to path {{ $labels.path }} are receiving errors.
            Please verify if clients are sending correct requests."
      - alert: RequestErrorsToUnexpectedPaths
        expr: sum(increase(vm_http_request_errors_total{path="*"}[5m])) by(job, instance, reason) > 0
        for: 15m
        labels:
          severity: warning
          show_at: dashboard
        annotations:
          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
          summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
          description: "Requests are failing with reason {{ $labels.reason }}.
            Please verify if clients are sending correct requests."
      - alert: RPCErrors
        expr: |
          (
--- a/deployment/docker/rules/alerts-single-node.yml
+++ b/deployment/docker/rules/alerts-single-node.yml
@@ -75,7 +75,7 @@ groups:
            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
      - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
Author	SHA1	Message	Date
Vadim Rutkovsky	4f8fa1d44f	deployment/docker/rules: add RequestErrorsToUnexpectedPaths alert which focuses on errors without path set (i.e. invalid authentication)	2026-07-13 14:55:00 +02:00
Vadim Rutkovsky	2edf1729d3	deployment/docker/rules: update RequestErrorsToAPI alert definition The alert requires path, but it may be absent from the metrics. Ignore metrics without path set	2026-07-01 12:32:22 +02:00