deployment/docker/rules: add RequestErrorsToUnknownPaths alert which focuses on errors without

path set (i.e. invalid authentication)

A new alert would help us separating two cases:
* API errors on known paths, indicating an application issue
* errors where path is not recorded - this indicates authentication problems

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -18,7 +18,7 @@ groups:
     concurrency: 2
     rules:
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

app/vmselect/promql/binary_op.go

@@ -164,11 +164,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		left := bfa.left
 		right := bfa.right
 		op := bfa.be.Op
-		isCmpOp := metricsql.IsBinaryOpCmp(op)
-		if !isCmpOp {
-			// Do not remove empty series for comparison operations, since NaN can be an
-			// explicitly present sample value. In particular, `value != NaN` is true.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
+		switch true {
+		case metricsql.IsBinaryOpCmp(op):
+			// Do not remove empty series for comparison operations,
+			// since this may lead to missing result.
+		default:
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
@@ -191,14 +191,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		isBool := bfa.be.Bool
 		fillLeft := bfa.be.FillLeft
 		fillRight := bfa.be.FillRight
-		// A NaN produced by a vector comparison denotes a filtered-out sample rather
-		// than an explicitly present NaN value. Drop it when this result is used as
-		// the right-hand side of another comparison.
-		// Note that filtered-out samples surviving as NaN inside other wrappers such as
-		// transform functions or arithmetic operations aren't detected, since such NaN
-		// is indistinguishable from an explicitly present NaN value there.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
-		dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -211,16 +203,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 				b := rightValues[j]
 				leftIsNaN := math.IsNaN(a)
 				rightIsNaN := math.IsNaN(b)
-				// Both sides are NaN.
+				// apply the fill value when either the left or right side is NaN, but not both.
 				if leftIsNaN && rightIsNaN {
 					dstValues[j] = bf(a, b, isBool)
 					continue
 				}
-				if dropNaNRight && rightIsNaN && fillRight == nil {
-					dstValues[j] = nan
-					continue
-				}
-				// Apply the fill value when either the left or right side is NaN, but not both.
 				if leftIsNaN && fillLeft != nil {
 					a = fillLeft.N
 				}
@@ -236,38 +223,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
-// isVectorComparisonExpr returns whether e is a comparison operation
-// with at least one non-scalar operand.
-func isVectorComparisonExpr(e metricsql.Expr) bool {
-	if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
-		// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
-		// the shape of the result. Subqueries are left as is, since rollup
-		// functions skip NaN values on their own.
-		e = re.Expr
-	}
-	be, ok := e.(*metricsql.BinaryOpExpr)
-	if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
-		return false
-	}
-	return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
-}
-
-// isScalarLikeExpr returns whether e always evaluates to a scalar value.
-func isScalarLikeExpr(e metricsql.Expr) bool {
-	switch e := e.(type) {
-	case *metricsql.NumberExpr, *metricsql.DurationExpr:
-		return true
-	case *metricsql.BinaryOpExpr:
-		return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
-	case *metricsql.FuncExpr:
-		switch strings.ToLower(e.Name) {
-		case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
-			return true
-		}
-	}
-	return false
-}
-
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {

app/vmselect/promql/exec_test.go

@@ -2994,136 +2994,6 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != NaN`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != 1200`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != (1 > 2)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
-		// A missing sample on the left side results in NaN for every comparison
-		// operation, so no special handling is needed there.
-		t.Parallel()
-		q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
-		// Prometheus drops the timestamps filtered out by the comparison on the right.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
 	t.Run(`-1 < 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `-1 < 2`

app/vmui/packages/vmui/src/pages/Relabel/index.tsx

@@ -72,7 +72,7 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header-configs">
           <TextField
             type="textarea"
-            label="Config"
+            label="Relabel configs"
             value={config}
             autofocus
             onChange={handleChangeConfig}
@@ -82,9 +82,8 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header__labels">
           <TextField
             type="textarea"
-            label="A time series"
+            label="Labels"
             value={labels}
-            placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}"
             onChange={handleChangeLabels}
             onEnter={handleRunQuery}
           />
@@ -98,22 +97,22 @@ const Relabel: FC = () => {
             <Button
               variant="text"
               color="gray"
-              startIcon={<WikiIcon/>}
+              startIcon={<InfoIcon/>}
             >
               Relabeling cookbook
             </Button>
           </a>
           <a
             target="_blank"
-            href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages"
+            href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
             rel="help noreferrer"
           >
             <Button
               variant="text"
               color="gray"
-              startIcon={<InfoIcon/>}
+              startIcon={<WikiIcon/>}
             >
-              Relabeling Stages
+              Documentation
             </Button>
           </a>
           <Button

deployment/docker/rules/alerts-cluster.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
         for: 15m
         labels:
           severity: warning
@@ -86,6 +86,18 @@ groups:
           description: "Requests to path {{ $labels.path }} are receiving errors.
             Please verify if clients are sending correct requests."
 
+      - alert: RequestErrorsToUnknownPaths
+        expr: sum(increase(vm_http_request_errors_total{path="*"}[5m])) by(job, instance, reason) > 0
+        for: 15m
+        labels:
+          severity: warning
+          show_at: dashboard
+        annotations:
+          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
+          description: "Requests are failing with reason {{ $labels.reason }}.
+            Please verify if clients are sending correct requests."
+
       - alert: RPCErrors
         expr: |
           (

deployment/docker/rules/alerts-single-node.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

docs/victoriametrics/changelog/CHANGELOG.md

@@ -34,7 +34,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
 * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
 
-* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
 
@@ -123,7 +122,6 @@ Released at 2026-06-22
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
 
-
 ## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
 
 Released at 2026-06-08