deployment/docker/rules: add RequestErrorsToUnknownPaths alert which focuses on errors without

path set (i.e. invalid authentication)

A new alert would help us separating two cases:
* API errors on known paths, indicating an application issue
* errors where path is not recorded - this indicates authentication problems

app/vmalert/config/testdata/rules/rules-replay-good.rules

@@ -18,7 +18,7 @@ groups:
     concurrency: 2
     rules:
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

app/vmselect/promql/binary_op.go

@@ -164,11 +164,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		left := bfa.left
 		right := bfa.right
 		op := bfa.be.Op
-		isCmpOp := metricsql.IsBinaryOpCmp(op)
-		if !isCmpOp {
-			// Do not remove empty series for comparison operations, since NaN can be an
-			// explicitly present sample value. In particular, `value != NaN` is true.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
+		switch true {
+		case metricsql.IsBinaryOpCmp(op):
+			// Do not remove empty series for comparison operations,
+			// since this may lead to missing result.
+		default:
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
@@ -191,14 +191,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		isBool := bfa.be.Bool
 		fillLeft := bfa.be.FillLeft
 		fillRight := bfa.be.FillRight
-		// A NaN produced by a vector comparison denotes a filtered-out sample rather
-		// than an explicitly present NaN value. Drop it when this result is used as
-		// the right-hand side of another comparison.
-		// Note that filtered-out samples surviving as NaN inside other wrappers such as
-		// transform functions or arithmetic operations aren't detected, since such NaN
-		// is indistinguishable from an explicitly present NaN value there.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
-		dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
 		for i, tsLeft := range left {
 			leftValues := tsLeft.Values
 			rightValues := right[i].Values
@@ -211,16 +203,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 				b := rightValues[j]
 				leftIsNaN := math.IsNaN(a)
 				rightIsNaN := math.IsNaN(b)
-				// Both sides are NaN.
+				// apply the fill value when either the left or right side is NaN, but not both.
 				if leftIsNaN && rightIsNaN {
 					dstValues[j] = bf(a, b, isBool)
 					continue
 				}
-				if dropNaNRight && rightIsNaN && fillRight == nil {
-					dstValues[j] = nan
-					continue
-				}
-				// Apply the fill value when either the left or right side is NaN, but not both.
 				if leftIsNaN && fillLeft != nil {
 					a = fillLeft.N
 				}
@@ -236,38 +223,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 	}
 }
 
-// isVectorComparisonExpr returns whether e is a comparison operation
-// with at least one non-scalar operand.
-func isVectorComparisonExpr(e metricsql.Expr) bool {
-	if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
-		// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
-		// the shape of the result. Subqueries are left as is, since rollup
-		// functions skip NaN values on their own.
-		e = re.Expr
-	}
-	be, ok := e.(*metricsql.BinaryOpExpr)
-	if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
-		return false
-	}
-	return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
-}
-
-// isScalarLikeExpr returns whether e always evaluates to a scalar value.
-func isScalarLikeExpr(e metricsql.Expr) bool {
-	switch e := e.(type) {
-	case *metricsql.NumberExpr, *metricsql.DurationExpr:
-		return true
-	case *metricsql.BinaryOpExpr:
-		return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
-	case *metricsql.FuncExpr:
-		switch strings.ToLower(e.Name) {
-		case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
-			return true
-		}
-	}
-	return false
-}
-
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
 		if isScalar(left) {

app/vmselect/promql/exec_test.go

@@ -2994,136 +2994,6 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != NaN`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != 1200`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != (1 > 2)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
-		// A missing sample on the left side results in NaN for every comparison
-		// operation, so no special handling is needed there.
-		t.Parallel()
-		q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
-		// Prometheus drops the timestamps filtered out by the comparison on the right.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
-		resultExpected := []netstorage.Result{}
-		f(q, resultExpected)
-	})
-	t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
-		t.Parallel()
-		q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
-		r := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
-			Timestamps: timestampsExpected,
-		}
-		r.MetricName.Tags = []storage.Tag{{
-			Key:   []byte("foo"),
-			Value: []byte("bar"),
-		}}
-		resultExpected := []netstorage.Result{r}
-		f(q, resultExpected)
-	})
 	t.Run(`-1 < 2`, func(t *testing.T) {
 		t.Parallel()
 		q := `-1 < 2`

app/vmui/packages/vmui/src/pages/Relabel/index.tsx

@@ -72,7 +72,7 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header-configs">
           <TextField
             type="textarea"
-            label="Config"
+            label="Relabel configs"
             value={config}
             autofocus
             onChange={handleChangeConfig}
@@ -82,9 +82,8 @@ const Relabel: FC = () => {
         <div className="vm-relabeling-header__labels">
           <TextField
             type="textarea"
-            label="A time series"
+            label="Labels"
             value={labels}
-            placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}"
             onChange={handleChangeLabels}
             onEnter={handleRunQuery}
           />
@@ -98,22 +97,22 @@ const Relabel: FC = () => {
             <Button
               variant="text"
               color="gray"
-              startIcon={<WikiIcon/>}
+              startIcon={<InfoIcon/>}
             >
               Relabeling cookbook
             </Button>
           </a>
           <a
             target="_blank"
-            href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages"
+            href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
             rel="help noreferrer"
           >
             <Button
               variant="text"
               color="gray"
-              startIcon={<InfoIcon/>}
+              startIcon={<WikiIcon/>}
             >
-              Relabeling Stages
+              Documentation
             </Button>
           </a>
           <Button

deployment/docker/rules/alerts-cluster.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
         for: 15m
         labels:
           severity: warning
@@ -86,6 +86,18 @@ groups:
           description: "Requests to path {{ $labels.path }} are receiving errors.
             Please verify if clients are sending correct requests."
 
+      - alert: RequestErrorsToUnknownPaths
+        expr: sum(increase(vm_http_request_errors_total{path="*"}[5m])) by(job, instance, reason) > 0
+        for: 15m
+        labels:
+          severity: warning
+          show_at: dashboard
+        annotations:
+          dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
+          summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
+          description: "Requests are failing with reason {{ $labels.reason }}.
+            Please verify if clients are sending correct requests."
+
       - alert: RPCErrors
         expr: |
           (

deployment/docker/rules/alerts-single-node.yml

@@ -75,7 +75,7 @@ groups:
             Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
 
       - alert: RequestErrorsToAPI
-        expr: increase(vm_http_request_errors_total[5m]) > 0
+        expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
         for: 15m
         labels:
           severity: warning

docs/guides/multi-regional-setup-dedicated-regions/README.md

@@ -6,282 +6,80 @@ build:
 sitemap:
   disable: true
 ---
+### Scenario
 
-## Overview {#scenario}
+Let's cover the case. You have multiple regions with workloads and want to collect metrics.
 
-This guide shows how to run VictoriaMetrics across many regions in high-availability mode. Each workload runs a local vmagent and sends metrics to dedicated monitoring deployments, so metric data is duplicated and available even if one monitoring region is down.
+The monitoring setup is in the dedicated regions as shown below:
 
-Use this architecture when you need region-level resilience and want monitoring to keep working even if one region becomes unavailable.
+![Multi-regional setup with VictoriaMetrics: Dedicated regions for monitoring](setup.webp)
 
-This setup gives you:
+Every workload region (Earth, Mars, Venus) has a vmagent that sends data to multiple regions with a monitoring setup. 
+The monitoring setup (Ground Control 1,2) contains VictoriaMetrics Time Series Database(TSDB) cluster or single.
 
-* High availability of metric data across regions.
-* A single global query endpoint.
-* Simpler disaster recovery.
+Using this schema, you can achieve:
 
-The trade-off is that you store and send the same data twice, so storage and compute requirements are increased.
+* Global Querying View
+* Querying all metrics from one monitoring installation
+* High Availability
+* You can lose one region, but your experience will be the same.
+* Of course, that means you duplicate your traffic twice.
 
-## Architecture
+### How to write the data to Ground Control regions
 
-The example architecture separates workloads into three regions, called Earth, Mars, and Venus. These represent the systems you want to monitor (e.g., your applications or your infrastructure). For monitoring there are two separate regions, Ground Control 1 and 2, each running its own VictoriaMetrics deployment. The workload regions (the planets) run a local vmagent that forwards the same metrics to the two dedicated Ground Control regions.
+* You need to pass two `-remoteWrite.url` command-line options to `vmagent`:
 
-![Multi-regional setup with VictoriaMetrics: Dedicated regions for monitoring](setup-1.webp)
-{width="700"}
+```sh
+/path/to/vmagent-prod \
+  -remoteWrite.url=<ground-control-1-remote-write> \
+  -remoteWrite.url=<ground-control-2-remote-write>
+```
+
+* If you scrape data from Prometheus-compatible targets, then please specify `-promscrape.config` parameter as well.
+
+Here is a Quickstart guide for [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#quick-start)
+
+### How to read the data from Ground Control regions
+
+You can use one of the following options:
+
+1. Multi-level [vmselect setup](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup) in cluster setup, top-level vmselect(s) reads data from cluster-level vmselects
+   * Returns data in one of the clusters is unavailable  
+   * Merges data from both sources. You need to turn on [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) to remove duplicates 
+1. Regional endpoints - use one regional endpoint as default and switch to another if there is an issue.
+1. Load balancer - that sends queries to a particular region. The benefit and disadvantage of this setup is that it's simple.
+1. Promxy - proxy that reads data from multiple Prometheus-like sources. It allows reading data more intelligently to cover the region's unavailability out of the box. It doesn't support MetricsQL yet (please check this issue).
+1. Global vmselect in cluster setup - you can set up an additional subset of vmselects that knows about all storages in all regions.
+   * The [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) in 1ms on the vmselect side must be turned on. This setup allows you to query data using MetricsQL.
+   * The downside is that vmselect waits for a response from all storages in all regions.
 
-The role of the Ground Controls can be filled by VictoriaMetrics in [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) or [cluster mode](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/). Since vmagent keeps a separate queue for each remote-write destination, an outage in one Ground Control region does not block delivery to the other.
 
 ### High Availability
 
-The architecture provides high availability by storing two full copies of the data: one in Ground Control 1 and the other in Ground Control 2. Since both store the same data, losing one region doesn't result in a monitoring outage. You can still run queries, view dashboards, and receive alerts.
+The data is duplicated twice, and every region contains a full copy of the data. That means one region can be offline.
 
-Note that you do not need to use the VictoriaMetrics cluster `-replicationFactor` flag for this cross-region setup, as high availability comes from vmagent replicating writes to independent monitoring regions.
+You don't need to set up a replication factor using the VictoriaMetrics cluster.
 
-## How to write the data to Ground Control regions
+### Alerting
 
-Run a local vmagent in each workload region (Earth, Mars, Venus) and point it to the Ground Control URLs using the `-remoteWrite.url` flag. For example:
+You can set up vmalert in each Ground control region that evaluates recording and alerting rules. As every region contains a full copy of the data, you don't need to synchronize recording rules from one region to another.
 
-```sh
-/path/to/vmagent-prod \
-  -remoteWrite.url=https://<ground-control-1-remote-write> \
-  -remoteWrite.url=https://<ground-control-2-remote-write>
-```
+For alert deduplication, please use [cluster mode in Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability).
 
-For single-node VictoriaMetrics the URLs follow this pattern:
+We also recommend adopting the list of [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
+for VictoriaMetrics components.
 
-```sh
-/path/to/vmagent-prod \
-  -remoteWrite.url=https://ground-control-1:8428/api/v1/write \
-  -remoteWrite.url=https://ground-control-2:8428/api/v1/write
-```
+### Monitoring
 
-For a VictoriaMetrics cluster, we use the following URL pattern (assuming [accountID](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) is 0):
+An additional VictoriaMetrics single can be set up in every region, scraping metrics from the main TSDB.
 
-```sh
-/path/to/vmagent-prod \
-  -remoteWrite.url=https://ground-control-1-vminsert:8480/insert/0/prometheus/api/v1/write \
-  -remoteWrite.url=https://ground-control-2-vminsert:8480/insert/0/prometheus/api/v1/write
-```
+You also may evaluate the option to send these metrics to the neighbour region to achieve HA.
 
-If you scrape Prometheus-compatible targets in your workloads, also pass a `-promscrape.config` file so vmagent knows what to scrape before it forwards the data.
+Additional context
+* VictoriaMetrics Single - [https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
+* VictoriaMetrics Cluster - [https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
 
-```sh
-/path/to/vmagent-prod \
-  -promscrape.config=/path/to/scrape.yaml \
-  -remoteWrite.url=https://<ground-control-1-remote-write> \
-  -remoteWrite.url=https://<ground-control-2-remote-write>
-```
 
-For more details, see the [vmagent quickstart guide](https://docs.victoriametrics.com/victoriametrics/vmagent/#quick-start)
-
-## How to read the data from Ground Control regions
-
-You can read data from Ground Control regions in a few different ways. The best option depends on your needs and operational complexity:
-
-* Regional endpoints: use one region endpoint as default and manually switch to the other during an outage. This is the simplest option but needs manual failover.
-* Load balancer: put a load balancer in front of both Ground Control regions. Route traffic to a preferred region, with automatic failover to the other region in case of failure.
-* Global vmselect: wire vmselect directly to the vmstorage nodes for both Ground Control regions (which must run in cluster mode).
-* Multi-level vmselect: run a dedicated vmselect on top of the Ground Control local vmselect nodes. This setup also requires both Ground Control instances to run in cluster mode.
-
-You can read more about choosing the right architecture in the [VictoriaMetrics topologies guide](https://docs.victoriametrics.com/guides/vm-architectures/).
-
-### Regional endpoints
-
-In this setup, Grafana, vmalert, or any other query client sends requests to one region. This is the default datasource. In case of an outage, you manually switch to the other region (standby datasource). For instance, use Ground Control 1 as the primary datasource and keep Ground Control 2 as a standby endpoint.
-
-![Diagram shows two Ground Control regions. Grafana connects to one by default leaving the other as failover](regional-endpoints.webp)
-{width="700"}
-
-Choose this option if you prioritize operational simplicity over automatic failover or a unified global query endpoint.
-
-If you use VictoriaMetrics single-node, the endpoints should point directly to the single-node HTTP API. For example:
-
-- Primary endpoint: `https://ground-control-1:8428/api/v1/query`
-- Standby endpoint: `https://ground-control-2:8428/api/v1/query`
-
-On the VictoriaMetrics cluster, the endpoints point to the cluster's vmselect HTTP API. For example:
-
-- Primary endpoint: `https://ground-control-1-vmselect:8481/select/0/prometheus/api/v1/query`
-- Standby endpoint: `https://ground-control-2-vmselect:8481/select/0/prometheus/api/v1/query`
-
-### Load balancer
-
-Use a load balancer when you want one stable query endpoint in front of your Ground Control regions. In this setup, dashboards and tools send queries to a single URL, and the load balancer routes them to the first available region. If that region becomes unavailable, the load balancer fails over to the next available region.
-
-![Diagram shows vmauth between Grafana and Ground Control regions](load-balancer-vmauth.webp)
-{width="700"}
-
-This option provides a single endpoint for queries. You can use [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) as a load balancer. For VictoriaMetrics single node, you can run it with the following configuration:
-
-```yaml
-unauthorized_user:
-  url_prefix:
-    - "http://ground-control-1:8428"
-    - "http://ground-control-2:8428"
-  load_balancing_policy: first_available
-```
-
-On the VictoriaMetrics cluster, the URLs must point to the Ground Control vmselect nodes. For example:
-
-```yaml
-unauthorized_user:
-  url_prefix:
-    - "http://ground-control-1-vmselect:8481"
-    - "http://ground-control-2-vmselect:8481"
-  load_balancing_policy: first_available
-```
-
-The examples above show how to load balance requests without authentication. You can optionally configure authentication in several ways; for more details, read the [vmauth authorization section](https://docs.victoriametrics.com/victoriametrics/vmauth/#authorization).
-
-To start vmauth with your configuration, use the `-auth.config` flag. For example:
-
-```sh
-/path/to/vmauth-prod -auth.config=/path/to/auth.yaml
-```
-
-You can test that queries work with curl:
-
-```sh
-# single node
-curl http://vmauth-node:8427/api/v1/query?query=up
-
-# cluster
-curl http://vmauth-node:8427/select/0/prometheus/api/v1/query?query=up
-```
-
-### Global vmselect
-
-> This option requires a [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) for the role of Ground Control.
-
-You can use a global [vmselect](https://docs.victoriametrics.com/victoriametrics/quick-start/#installing-vmselect) when each Ground Control region runs a VictoriaMetrics cluster. In this setup, you run a global vmselect node that connects directly to the storage nodes in both Ground Control regions.
-
-![Diagram showing a vmselect node connecting to the vmstorage nodes in each ground control region. The global vmselect serves Grafana](global-vmselect.webp)
-{width="700"}
-
-Since the samples are duplicated across Ground Control regions, you must enable [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) on the global vmselect. You must set `-dedup.minScrapeInterval=1ms` at a minimum or to the same value as the scrape interval if there is one.
-
-For example:
-
-```sh
-/path/to/vmselect-prod \
-  -storageNode=ground-control-1-vmstorage-1:8401,ground-control-1-vmstorage-2:8401,ground-control-2-vmstorage-1:8401,ground-control-2-vmstorage-2:8401 \
-  -dedup.minScrapeInterval=1ms
-```
-
-This option supports [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) and gives you a single query endpoint for all Ground Control regions. It can also continue to serve data during an outage of one Ground Control region.
-
-The downside is query behavior during region failures or high latency. The global vmselect waits for responses from all storage nodes across regions, so a slow or unavailable backend can slow queries or lead to [partial responses](https://docs.victoriametrics.com/guides/vm-architectures/#query-consistency-partial-vs-complete-responses).
-
-### Multi-level vmselect
-
-> This option requires a [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) for the role of Ground Control.
-
-In this setup, each Ground Control region has its own local vmselect. A top-level vmselect queries these instead of connecting directly to vmstorage nodes.
-
-![Diagram shows top-level vmselect connecting to the regional vmselect nodes in each Ground Control cluster](top-level-vmselect.webp)
-{width="700"}
-
-This option is useful when direct access to vmstorage nodes is not practical or desirable. For example, when running on Kubernetes, the vmstorage services don't provide an HTTP query endpoint by default.
-
-To enable this setup, each Ground Control regional vmselect must listen for requests from the top layer by setting the `-clusternativeListenAddr` flag. The top-level vmselect must then use `-storageNode` to point to the regional vmselect nodes and must set a [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) interval to handle duplicated data.
-
-For example, here's how we can run the local and top-level vmselect nodes:
-
-```sh
-# Ground Control 1 cluster vmselect
-/path/to/vmselect-prod \
-  -storageNode=ground-control-1-vmstorage-1:8401,ground-control-1-vmstorage-2:8401 \
-  -clusternativeListenAddr=:8401
-
-# Ground Control 2 cluster vmselect
-/path/to/vmselect-prod \
-  -storageNode=ground-control-2-vmstorage-1:8401,ground-control-2-vmstorage-2:8401 \
-  -clusternativeListenAddr=:8401
-
-# Top-level vmselect
-/path/to/vmselect-prod \
-  -storageNode=ground-control-1-vmselect:8401,ground-control-2-vmselect:8401 \
-  -dedup.minScrapeInterval=1ms
-```
-
-This option supports MetricsQL and can keep serving queries during a regional outage because the top-level vmselect can still query the remaining Ground Control vmselect node. It also gives you a cleaner separation between regional and top-level query layers than a single global vmselect that talks to all vmstorage nodes directly.
-
-The main trade-off is complexity. You add another query layer and more moving parts, so this setup is harder to deploy and operate than regional endpoints, a load balancer, or global vmselect.
-
-## Alerting
-
-Run a vmalert node in each Ground Control region and point it to the local VictoriaMetrics endpoint. Since each region stores the same data, you can deploy the same alerting and recording rules in every region without needing cross-region rule synchronization. Send alerts to an [Alertmanager cluster](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability) to deduplicate firing alerts.
-
-![Diagram showing vmalert nodes running in each Ground Control region. An Alertmanager cluster connects to each vmalert and deduplicates notifications](vmalert-alertmanager.webp)
-{width="700"}
-
-A simple vmalert example for a single-node VictoriaMetrics looks like this:
-
-```sh
-/path/to/vmalert \
-  -rule=/path/to/rules.yaml \
-  -datasource.url=http://ground-control-1:8428 \
-  -notifier.url=http://alertmanager-1:9093 \
-  -notifier.url=http://alertmanager-2:9093
-```
-
-In VictoriaMetrics cluster mode, point `-datasource.url` to the regional vmselect endpoint. For example:
-
-```sh
-/path/to/vmalert \
-  -rule=/path/to/rules.yaml \
-  -datasource.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
-  -notifier.url=http://alertmanager-1:9093,http://alertmanager-2:9093
-```
-
-If you want vmalert to preserve alert state and recording rule results across restarts, configure `-remoteWrite.url` and `-remoteRead.url` to point to VictoriaMetrics as well. For example, for a VictoriaMetrics cluster:
-
-```sh
-/path/to/vmalert \
-  -rule=/path/to/rules.yaml \
-  -datasource.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
-  -remoteRead.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
-  -remoteWrite.url=http://ground-control-1-vminsert:8480/insert/0/prometheus \
-  -notifier.url=http://alertmanager-1:9093,http://alertmanager-2:9093
-```
-
-We recommend using the list of [VictoriaMetrics alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts).
-
-## Monitoring
-
-You can monitor Ground Control instances themselves using a separate monitoring path. In this setup, each region runs its own monitoring instance that scrapes metrics from the Ground Control components.
-
-![Diagram of the original setup with monitoring of monitoring added. Each region has a dedicated VictoriaMetrics instance dedicated to monitoring the main TSDB](setup-mom-1.webp)
-{width="700"}
-
-You can optionally duplicate the monitored metrics to the neighboring region for extra resilience. That way, if a whole Ground Control region goes down, you still have access to the telemetry of the downed VictoriaMetrics instance, which can help you troubleshoot and restore service more easily.
-
-Refer to the following pages on how to monitor your VictoriaMetrics deployments:
-
-* [How to monitor VictoriaMetrics single node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
-* [How to monitor a VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
-
-## What more can we do?
-
-You can deploy extra vmagent instances in Ground Control regions and use them as regional ingestion proxies. This places the write endpoint closer to storage and adds another disk-backed buffer, which improves resilience when storage is temporarily unavailable.
-
-![Diagram of the original setup where a vmagent node runs in front of each Ground Control region](setup-vmagent-1.webp)
-{width="700"}
-
-This pattern is useful when you want more reliable delivery, local relabeling, or a cleaner separation between cross-region traffic and local storage ingestion.
-
-For a Ground Control running VictoriaMetrics single node, you can run vmagent as follows:
-
-```sh
-# vmagent next to Ground Control 1
-/path/to/vmagent-prod \
-  -remoteWrite.url=http://ground-control-1:8428/api/v1/write
-```
-
-If running in cluster mode, use this instead:
-
-```sh
-# vmagent next to Ground Control 1 for cluster mode
-/path/to/vmagent-prod \
-  -remoteWrite.url=http://ground-control-1-vminsert:8480/insert/0/prometheus/api/v1/write
-```
+### What more can we do?
 
+Setup vmagents in Ground Control regions. That allows it to accept data close to storage and add more reliability if storage is temporarily offline.

docs/victoriametrics/changelog/CHANGELOG.md

@@ -34,7 +34,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
 * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
 
-* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
 
@@ -123,7 +122,6 @@ Released at 2026-06-22
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
 
-
 ## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
 
 Released at 2026-06-08