deployment/docker/rules: add RequestErrorsToUnknownPaths alert which focuses on errors without path

set (i.e. invalid authentication)

A new alert would help us separating two cases:
* API errors on known paths, indicating an application issue
* errors where path is not recorded - this indicates authentication problems

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -18,7 +18,7 @@ groups:
     concurrency: 2
     rules:
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

app/vmalert/replay.go

@@ -30,7 +30,7 @@ var (
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
 		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
-	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
+	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {

app/vmalert/rule/rule.go

@@ -132,9 +132,9 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
 		var esc *httpserver.ErrorWithStatusCode
 		if errors.As(err, &esc) {
 			statusCode := esc.StatusCode
-			// if the status code is 400 or 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
+			// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
 			// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
-			if statusCode == http.StatusUnprocessableEntity || statusCode == http.StatusBadRequest {
+			if statusCode == http.StatusUnprocessableEntity {
 				if continueWithExecutionErr {
 					logger.Errorf("rule %q: %s", r, err)
 					return 0, nil

deployment/docker/rules/alerts-cluster.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
         for: 15m
         labels:
           severity: warning
@@ -83,8 +83,24 @@ groups:
         annotations:
           dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
           summary: "Too many errors served for {{ $labels.job }} path {{ $labels.path }} (instance {{ $labels.instance }})"
-          description: "Requests to path {{ $labels.path }} are receiving errors.
-            Please verify if clients are sending correct requests."
+          description: |
+              Requests to path {{ $labels.path }} are receiving errors.
+              Please verify if clients are sending correct requests.
+
+      # Auth errors and unknown paths should be handled by a different alert
+      # See https://github.com/VictoriaMetrics/VictoriaMetrics/blob/fdd9a221df835daa378ae2e6c9f12e4e3be79c76/lib/httpserver/httpserver.go#L589-L591
+      - alert: RequestErrorsToUnknownPaths
+        expr: sum(increase(vm_http_request_errors_total{path=~"^(\*|)$"}[5m])) by(job, instance, reason) > 0
+        for: 15m
+        labels:
+          severity: warning
+          show_at: dashboard
+        annotations:
+          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
+          description: |
+              Requests are failing with reason {{ $labels.reason }}.
+              Please verify if clients are sending correct requests.
 
       - alert: RPCErrors
         expr: |

deployment/docker/rules/alerts-single-node.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): update `RequestErrorsToAPI` alerting rule and add `RequestErrorsToUnknownPaths` to [cluster alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml). The new rule notifies when authentication fails or unknown paths are requested. Previously both cases were treated as errors to the API.
+
 ## [v1.149.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.149.0)
 
 Release candidate

docs/victoriametrics/vmalert_common_flags.md

@@ -405,7 +405,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -remoteWrite.url string
      Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.continueWithExecutionErr bool
-     Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
+     Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int