deployment/docker/rules: add RequestErrorsToUnknownPaths alert which focuses on errors without path

set (i.e. invalid authentication)

A new alert would help us separating two cases:
* API errors on known paths, indicating an application issue
* errors where path is not recorded - this indicates authentication problems

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -18,7 +18,7 @@ groups:
     concurrency: 2
     rules:
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -22,10 +22,12 @@ const StepConfigurator: FC = () => {
   const { isMobile } = useDeviceDetect();
 
   const { customStep: value, isHistogram } = useGraphState();
-  const { period: { end, start } } = useTimeState();
+  const { period: { step, end, start } } = useTimeState();
   const graphDispatch = useGraphDispatch();
   const { displayType } = useCustomPanelState();
 
+  const prevDuration = usePrevious(end - start);
+
   const defaultStep = useMemo(() => {
     return getStepFromDuration(end - start, isHistogram, displayType);
   }, [end, start, isHistogram, displayType]);
@@ -104,14 +106,16 @@ const StepConfigurator: FC = () => {
   }, [defaultStep]);
 
   useEffect(() => {
-    if (!prevDefaultStep) return;
-    if (value !== prevDefaultStep) return;
-    if (value === defaultStep) return;
+    const dur = end - start;
+    if (dur === prevDuration || !prevDuration || value !== prevDefaultStep) return;
+    if (defaultStep) {
+      handleApply(defaultStep);
+    }
+  }, [prevDuration, defaultStep]);
 
-    graphDispatch({ type: "SET_CUSTOM_STEP", payload: defaultStep });
-    setCustomStep(defaultStep);
-    setError("");
-  }, [defaultStep, prevDefaultStep, value, graphDispatch]);
+  useEffect(() => {
+    if (step === value || step === defaultStep) handleApply(defaultStep);
+  }, [isHistogram, displayType]);
 
   return (
     <div

deployment/docker/rules/alerts-cluster.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
         for: 15m
         labels:
           severity: warning
@@ -83,8 +83,24 @@ groups:
         annotations:
           dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
           summary: "Too many errors served for {{ $labels.job }} path {{ $labels.path }} (instance {{ $labels.instance }})"
-          description: "Requests to path {{ $labels.path }} are receiving errors.
-            Please verify if clients are sending correct requests."
+          description: |
+              Requests to path {{ $labels.path }} are receiving errors.
+              Please verify if clients are sending correct requests.
+
+      # Auth errors and unknown paths should be handled by a different alert
+      # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/fdd9a221df835daa378ae2e6c9f12e4e3be79c76/lib/httpserver/httpserver.go#L589-L591
+      - alert: RequestErrorsToUnknownPaths
+        expr: sum(increase(vm_http_request_errors_total{path=~"^(\*|)$"}[5m])) by(job, instance, reason) > 0
+        for: 15m
+        labels:
+          severity: warning
+          show_at: dashboard
+        annotations:
+          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
+          description: |
+              Requests are failing with reason {{ $labels.reason }}.
+              Please verify if clients are sending correct requests.
 
       - alert: RPCErrors
         expr: |

deployment/docker/rules/alerts-single-node.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,7 +26,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): respect the custom query step specified via `g0.step_input` when opening a URL. Previously, it could be reset to the automatically calculated step and potentially cause dashboards to freeze. See [#11137](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11137).
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): update `RequestErrorsToAPI` alerting rule and add `RequestErrorsToUnknownPaths` to [cluster alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml). The new rule notifies when authentication fails or unknown paths are requested. Previously both cases were treated as errors to the API.
 
 ## [v1.149.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.149.0)