fix review comment

add changelog
dashboards: add MTTR to alerts statistics dashboard
2026-08-12 10:48:27 +03:00 · 2026-08-11 17:28:29 +02:00 · 2026-08-11 17:00:22 +02:00 · 2026-08-11 16:49:55 +02:00
5 changed files with 299 additions and 60 deletions
--- a/app/vmalert/config/testdata/rules/rules-replay-good.rules
+++ b/app/vmalert/config/testdata/rules/rules-replay-good.rules
@@ -18,7 +18,7 @@ groups:
    concurrency: 2
    rules:
      - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
+        expr: increase(vm_http_request_errors_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
--- a/dashboards/alert-statistics.json
+++ b/dashboards/alert-statistics.json
@@ -31,6 +31,12 @@
      "id": "table",
      "name": "Table",
      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
    }
  ],
  "annotations": {
@@ -466,6 +472,106 @@
        "type": "prometheus",
        "uid": "${ds}"
      },
+      "description": "Mean Time To Resolution (MTTR) aggregated across the selected jobs, instances and groups.\n\nAt every point the value is computed over the trailing $mttr_window window as the accumulated firing time (integral of the vmalert_alerts_firing gauge, in alert-seconds) divided by the number of resolved alerts (sum of the gauge decreases) within that window.\n\nCaveats:\n* alerts which are still firing contribute firing time but no resolve event, inflating the value;\n* resolve time granularity is limited by the scrape interval of vmalert metrics.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 30,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": 7200000,
+            "lineInterpolation": "stepAfter",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 18
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(integrate(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$mttr_window])) / (sum(descent_over_time(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$mttr_window])) > 0)",
+          "format": "time_series",
+          "instant": false,
+          "legendFormat": "MTTR",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "MTTR ($mttr_window window)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${ds}"
+      },
+      "description": "Number of times alerts switched to the FIRING state (Count) and the mean time to resolve (MTTR) per group over the selected time range.\n\nMTTR = total firing time / number of resolved alerts. Empty MTTR means no alerts were resolved within the selected range.\n\nMTTR is an approximation: alerts which are still firing add firing time but no resolve event (inflating MTTR), resolve time granularity is limited by the scrape interval of vmalert metrics, and vmalert restarts may register spurious resolve events.",
      "fieldConfig": {
        "defaults": {
          "custom": {
@@ -485,10 +591,6 @@
              {
                "color": "green",
                "value": 0
-              },
-              {
-                "color": "red",
-                "value": 80
              }
            ]
          }
@@ -502,7 +604,50 @@
            "properties": [
              {
                "id": "custom.width",
-                "value": 100
+                "value": 150
+              },
+              {
+                "id": "fieldMinMax",
+                "value": true
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "gauge"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "MTTR"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "s"
+              },
+              {
+                "id": "custom.width",
+                "value": 150
+              },
+              {
+                "id": "noValue",
+                "value": "-"
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "gauge",
+                  "valueDisplayMode": "text"
+                }
+              },
+              {
+                "id": "fieldMinMax",
+                "value": true
              }
            ]
          }
@@ -512,12 +657,12 @@
        "h": 13,
        "w": 9,
        "x": 0,
-        "y": 18
+        "y": 26
      },
      "id": 2,
      "options": {
        "cellHeight": "sm",
-        "frameIndex": 1,
+        "frameIndex": 0,
        "showHeader": true,
        "sortBy": [
          {
@@ -542,10 +687,29 @@
          "legendFormat": "__auto",
          "range": false,
          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "(sum(integrate(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by (group) / (sum(descent_over_time(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by (group) > 0)) and on (group) (topk_max(100, sum(increases_over_time(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by(group) > 0))",
+          "format": "table",
+          "hide": false,
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "B"
        }
      ],
      "title": "FIRING over time by group",
      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
        {
          "id": "organize",
          "options": {
@@ -556,13 +720,13 @@
            "includeByName": {},
            "indexByName": {
              "Time": 0,
-              "Value": 3,
-              "alertname": 2,
+              "Value #A": 2,
+              "Value #B": 3,
              "group": 1
            },
            "renameByName": {
-              "Value": "Count",
-              "alertname": "Alert",
+              "Value #A": "Count",
+              "Value #B": "MTTR",
              "group": "Group"
            }
          }
@@ -575,6 +739,7 @@
        "type": "prometheus",
        "uid": "${ds}"
      },
+      "description": "Number of times alerts switched to the FIRING state (Count) and the mean time to resolve (MTTR) per alerting rule over the selected time range.\n\nMTTR = total firing time / number of resolved alerts. Empty MTTR means no alerts were resolved within the selected range.\n\nMTTR is an approximation: alerts which are still firing add firing time but no resolve event (inflating MTTR), resolve time granularity is limited by the scrape interval of vmalert metrics, and vmalert restarts may register spurious resolve events.",
      "fieldConfig": {
        "defaults": {
          "custom": {
@@ -591,10 +756,6 @@
              {
                "color": "green",
                "value": 0
-              },
-              {
-                "color": "red",
-                "value": 80
              }
            ]
          }
@@ -608,7 +769,18 @@
            "properties": [
              {
                "id": "custom.width",
-                "value": 100
+                "value": 150
+              },
+              {
+                "id": "fieldMinMax",
+                "value": true
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "gradient",
+                  "type": "gauge"
+                }
              }
            ]
          },
@@ -641,6 +813,36 @@
                ]
              }
            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "MTTR"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "s"
+              },
+              {
+                "id": "custom.width",
+                "value": 150
+              },
+              {
+                "id": "noValue",
+                "value": "-"
+              },
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "type": "gauge"
+                }
+              },
+              {
+                "id": "fieldMinMax",
+                "value": true
+              }
+            ]
          }
        ]
      },
@@ -648,12 +850,12 @@
        "h": 13,
        "w": 15,
        "x": 9,
-        "y": 18
+        "y": 26
      },
      "id": 1,
      "options": {
        "cellHeight": "sm",
-        "frameIndex": 1,
+        "frameIndex": 0,
        "showHeader": true,
        "sortBy": [
          {
@@ -678,10 +880,29 @@
          "legendFormat": "__auto",
          "range": false,
          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${ds}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "(sum(integrate(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by (group, alertname) / (sum(descent_over_time(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by (group, alertname) > 0)) and on (group, alertname) (topk_max($topk, sum(increases_over_time(vmalert_alerts_firing{job=~\"$job\",instance=~\"$instance\",group=~\"$group\"}[$__range])) by (group, alertname) > 0))",
+          "format": "table",
+          "hide": false,
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "B"
        }
      ],
      "title": "FIRING over time by rule",
      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
        {
          "id": "organize",
          "options": {
@@ -690,16 +911,16 @@
            },
            "indexByName": {
              "Time": 0,
-              "Value": 3,
+              "Value #A": 3,
+              "Value #B": 4,
              "alertname": 2,
              "group": 1
            },
            "renameByName": {
-              "Time": "",
-              "Value": "Count",
+              "Value #A": "Count",
+              "Value #B": "MTTR",
              "alertname": "Alert",
-              "group": "Group",
-              "range": ""
+              "group": "Group"
            }
          }
        }
@@ -819,6 +1040,55 @@
        ],
        "query": "5, 10, 20, 30, 50",
        "type": "custom"
+      },
+      {
+        "current": {
+          "text": "1d",
+          "value": "1d"
+        },
+        "description": "Sliding window interval for calculation of Mean time to resolution",
+        "includeAll": false,
+        "label": "MTTR window",
+        "name": "mttr_window",
+        "options": [
+          {
+            "selected": false,
+            "text": "1h",
+            "value": "1h"
+          },
+          {
+            "selected": false,
+            "text": "6h",
+            "value": "6h"
+          },
+          {
+            "selected": false,
+            "text": "12h",
+            "value": "12h"
+          },
+          {
+            "selected": true,
+            "text": "1d",
+            "value": "1d"
+          },
+          {
+            "selected": false,
+            "text": "7d",
+            "value": "7d"
+          },
+          {
+            "selected": false,
+            "text": "14d",
+            "value": "14d"
+          },
+          {
+            "selected": false,
+            "text": "30d",
+            "value": "30d"
+          }
+        ],
+        "query": "1h, 6h, 12h, 1d, 7d, 14d, 30d",
+        "type": "custom"
      }
    ]
  },
--- a/deployment/docker/rules/alerts-cluster.yml
+++ b/deployment/docker/rules/alerts-cluster.yml
@@ -75,7 +75,7 @@ groups:
            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."

      - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
+        expr: increase(vm_http_request_errors_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
@@ -83,24 +83,8 @@ groups:
        annotations:
          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
          summary: "Too many errors served for {{ $labels.job }} path {{ $labels.path }} (instance {{ $labels.instance }})"
-          description: |
-              Requests to path {{ $labels.path }} are receiving errors.
-              Please verify if clients are sending correct requests.
-
-      # Auth errors and unknown paths should be handled by a different alert
-      # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/fdd9a221df835daa378ae2e6c9f12e4e3be79c76/lib/httpserver/httpserver.go#L589-L591
-      - alert: RequestErrorsToUnknownPaths
-        expr: sum(increase(vm_http_request_errors_total{path=~"^(\*|)$"}[5m])) by(job, instance, reason) > 0
-        for: 15m
-        labels:
-          severity: warning
-          show_at: dashboard
-        annotations:
-          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
-          summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
-          description: |
-              Requests are failing with reason {{ $labels.reason }}.
-              Please verify if clients are sending correct requests.
+          description: "Requests to path {{ $labels.path }} are receiving errors.
+            Please verify if clients are sending correct requests."

      - alert: RPCErrors
        expr: |
--- a/deployment/docker/rules/alerts-single-node.yml
+++ b/deployment/docker/rules/alerts-single-node.yml
@@ -75,7 +75,7 @@ groups:
            Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."

      - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
+        expr: increase(vm_http_request_errors_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
@@ -85,21 +85,6 @@ groups:
          description: "Requests to path {{ $labels.path }} are receiving errors.
            Please verify if clients are sending correct requests."

-      # Auth errors and unknown paths should be handled by a different alert
-      # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/fdd9a221df835daa378ae2e6c9f12e4e3be79c76/lib/httpserver/httpserver.go#L589-L591
-      - alert: RequestErrorsToUnknownPaths
-        expr: sum(increase(vm_http_request_errors_total{path=~"^(\*|)$"}[5m])) by(job, instance, reason) > 0
-        for: 15m
-        labels:
-          severity: warning
-          show_at: dashboard
-        annotations:
-          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
-          summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
-          description: |
-            Requests are failing with reason {{ $labels.reason }}.
-            Please verify if clients are sending correct requests.
-
      - alert: TooHighChurnRate
        expr: |
          (
--- a/docs/victoriametrics/changelog/CHANGELOG.md
+++ b/docs/victoriametrics/changelog/CHANGELOG.md
@@ -27,7 +27,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 ## tip

 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add an option to customize the favicon color. This makes it easier to distinguish between different installations opened in multiple browser tabs. See [#11329](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11329).
-* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): limit the `RequestErrorsToAPI` alert to known API paths. Introduce a new `RequestErrorsToUnknownPaths` alert for authentication failures and requests to unknown paths. See [#11200](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11200).
+* FEATURE: [dashboards/alert-statistics](https://grafana.com/grafana/dashboards/24553): add `MTTR` panel showing the Mean Time To Resolution trend over a configurable sliding window, add `MTTR` column to groups and alerts tables.

 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly parse small fractional Unix timestamps in timestamp args such as `start` and `end` in `/api/v1/query_range` and `--vm-native-filter-time-start` and `--vm-native-filter-time-end` in `vmctl`. Previously, fractional Unix timestamps with the integer part below `9223372` were interpreted with the wrong unit, for example `12.0` was parsed as `12000` seconds instead of `12` seconds. See [#11324](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11324).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): respect the custom query step specified via `g0.step_input` when opening a URL. Previously, it could be reset to the automatically calculated step and potentially cause dashboards to freeze. See [#11137](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11137).
Author	SHA1	Message	Date
Alexander Marshalov	752028ab11	fix review comment	2026-08-11 17:28:29 +02:00
Alexander Marshalov	88d0d489d8	add changelog	2026-08-11 17:00:22 +02:00
Alexander Marshalov	61a4c19da3	dashboards: add MTTR to alerts statistics dashboard	2026-08-11 16:49:55 +02:00