init

app/vmalert/config/testdata/rules/rules-query-good.rules

@@ -1,5 +1,7 @@
 groups:
   - name: alertmanager.rules
+    labels:
+      team: wow
     rules:
       - alert: AlertmanagerConfigInconsistent
         annotations:
@@ -12,4 +14,12 @@ groups:
           count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="openshift-monitoring"})) != 1
         for: 5m
         labels:
-          severity: critical
+          severity: critical
+  - name: g1
+    interval: 30s
+    labels:
+      team: wow
+    rules:
+      - alert: a1
+        expr: up>0
+        for: 5m

app/vmalert/remotewrite/client_timing_test.go

@@ -0,0 +1,46 @@
+package remotewrite
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+)
+
+func BenchmarkClientFlushEndToEnd(b *testing.B) {
+	srv := newRWServer()
+	defer srv.Close()
+
+	client, err := NewClient(context.Background(), Config{
+		Addr:          srv.URL,
+		MaxBatchSize:  10000,
+		Concurrency:   1,
+		MaxQueueSize:  100000,
+		FlushInterval: time.Hour,
+	})
+	if err != nil {
+		b.Fatalf("failed to create client: %s", err)
+	}
+	defer client.Close()
+
+	const tsCount = 100000
+	tss := make([]prompb.TimeSeries, tsCount)
+	for i := range tss {
+		tss[i] = prompb.TimeSeries{
+			Labels:  []prompb.Label{{Name: "__name__", Value: fmt.Sprintf("metric_%d", i)}},
+			Samples: []prompb.Sample{{Value: float64(i), Timestamp: 1000}},
+		}
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// WriteRequest is stack-allocated each iter; re-assigning tss is just a slice header copy
+		wr := prompb.WriteRequest{Timeseries: tss}
+		client.flush(context.Background(), &wr)
+		// flush calls wr.Reset() via defer; restore the slice for next iteration
+		wr.Timeseries = tss
+	}
+}

app/vmalert/web.go

@@ -22,6 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmalertapi"
 )
 
 var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
@@ -281,16 +282,8 @@ func (rh *requestHandler) getAlert(r *http.Request) (*rule.ApiAlert, *httpserver
 	return a, nil
 }
 
-type listGroupsResponse struct {
-	Status      string `json:"status"`
-	Page        int    `json:"page,omitempty"`
-	TotalPages  int    `json:"total_pages,omitempty"`
-	TotalGroups int    `json:"total_groups,omitempty"`
-	TotalRules  int    `json:"total_rules,omitempty"`
-	Data        struct {
-		Groups []*rule.ApiGroup `json:"groups"`
-	} `json:"data"`
-}
+// listGroupsResponse is shared with lib/vmalertproxy, which merges responses from multiple vmalerts.
+type listGroupsResponse = vmalertapi.ListGroupsResponse[*rule.ApiGroup]
 
 type groupsFilter struct {
 	groupNames []string
@@ -596,12 +589,8 @@ func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, *httpserver.Error
 	return b, nil
 }
 
-type listAlertsResponse struct {
-	Status string `json:"status"`
-	Data   struct {
-		Alerts []*rule.ApiAlert `json:"alerts"`
-	} `json:"data"`
-}
+// listAlertsResponse is shared with lib/vmalertproxy, which merges responses from multiple vmalerts.
+type listAlertsResponse = vmalertapi.ListAlertsResponse[*rule.ApiAlert]
 
 func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	rh.m.groupsMu.RLock()
@@ -665,12 +654,8 @@ func (rh *requestHandler) listAlerts(af *alertsFilter) ([]byte, *httpserver.Erro
 	return b, nil
 }
 
-type listNotifiersResponse struct {
-	Status string `json:"status"`
-	Data   struct {
-		Notifiers []*notifier.ApiNotifier `json:"notifiers"`
-	} `json:"data"`
-}
+// listNotifiersResponse is shared with lib/vmalertproxy, which merges responses from multiple vmalerts.
+type listNotifiersResponse = vmalertapi.ListNotifiersResponse[*notifier.ApiNotifier]
 
 func (rh *requestHandler) listNotifiers() ([]byte, *httpserver.ErrorWithStatusCode) {
 	targets := notifier.GetTargets()

app/vmselect/main.go

@@ -38,9 +38,15 @@ var (
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
 
-	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , "+
+	vmalertProxyURL = flagutil.NewArrayString("vmalert.proxyURL", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , "+
 		"then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules . "+
+		"If multiple URLs are set, then alerting API requests are sent to all of them and the responses are merged. Every returned group, alert and notifier target "+
+		"is marked with the `__vmalert_source` label containing the name of the vmalert it came from - see -vmalert.proxyName. "+
+		"Unavailable vmalerts do not fail the request - they are reported via the `warnings` field in the response. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert")
+	vmalertProxyName = flagutil.NewArrayString("vmalert.proxyName", "Optional name for the vmalert at the corresponding -vmalert.proxyURL. "+
+		"It is used as a value for the `__vmalert_source` label and for routing requests to a particular vmalert via `vmalert_source` query arg. "+
+		"By default the name is set to `vmalert_proxy_N`, where N is the one-based position of the corresponding -vmalert.proxyURL")
 )
 
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
@@ -56,10 +62,8 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
 	maxQueueDuration = vmselectMaxQueueDuration
 	concurrencyLimitCh = make(chan struct{}, maxConcurrentRequests)
 
+	vmalertproxy.Init(*vmalertProxyURL, *vmalertProxyName)
 	initVMUIConfig()
-
-	vmalertproxy.Init(*vmalertProxyURL)
-
 }
 
 // InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
@@ -527,7 +531,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	}
 	if strings.HasPrefix(path, "/vmalert/") {
 		vmalertRequests.Inc()
-		if len(*vmalertProxyURL) == 0 {
+		if !vmalertproxy.Enabled() {
 			w.WriteHeader(http.StatusBadRequest)
 			w.Header().Set("Content-Type", "application/json")
 			fmt.Fprintf(w, "%s", `{"status":"error","msg":"the '-vmalert.proxyURL' command-line must be configured; `+
@@ -571,7 +575,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/api/v1/rules", "/rules":
 		rulesRequests.Inc()
-		if len(*vmalertProxyURL) > 0 {
+		if vmalertproxy.Enabled() {
 			vmalertproxy.HandleRequest(w, r, path)
 			return true
 		}
@@ -581,7 +585,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/api/v1/alerts", "/alerts":
 		alertsRequests.Inc()
-		if len(*vmalertProxyURL) > 0 {
+		if vmalertproxy.Enabled() {
 			vmalertproxy.HandleRequest(w, r, path)
 			return true
 		}
@@ -591,7 +595,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/api/v1/notifiers", "/notifiers":
 		notifiersRequests.Inc()
-		if len(*vmalertProxyURL) > 0 {
+		if vmalertproxy.Enabled() {
 			vmalertproxy.HandleRequest(w, r, path)
 			return true
 		}
@@ -749,6 +753,9 @@ func initVMUIConfig() {
 		} `json:"license"`
 		VMAlert struct {
 			Enabled bool `json:"enabled"`
+			// Sources contains names of the vmalerts at -vmalert.proxyURL.
+			// vmui uses them for filtering rules by the vmalert they come from.
+			Sources []string `json:"sources,omitempty"`
 		} `json:"vmalert"`
 	}
 	data, err := vmuiFiles.ReadFile("vmui/config.json")
@@ -764,7 +771,11 @@ func initVMUIConfig() {
 		// buildinfo.ShortVersion() may return empty result for builds without tags
 		cfg.Version = buildinfo.Version
 	}
-	cfg.VMAlert.Enabled = len(*vmalertProxyURL) != 0
+	cfg.VMAlert.Enabled = vmalertproxy.Enabled()
+	if names := vmalertproxy.SourceNames(); len(names) > 1 {
+		// A single vmalert needs no filtering by source.
+		cfg.VMAlert.Sources = names
+	}
 	data, err = json.Marshal(&cfg)
 	if err != nil {
 		logger.Fatalf("cannot create vmui config: %s", err)

app/vmselect/vmui/assets/favicon.svg

@@ -0,0 +1,5 @@
+<svg width="48" height="48" fill="#020202" xmlns="http://www.w3.org/2000/svg">
+    <path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0"/>
+    <path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z"/>
+    <path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z"/>
+</svg>

app/vmselect/vmui/favicon.svg

@@ -1 +0,0 @@
-<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>

app/vmselect/vmui/index.html

@@ -2,9 +2,9 @@
 <html lang="en">
 <head>
   <meta charset="utf-8"/>
-  <link rel="icon" href="./favicon.svg"/>
-  <link rel="apple-touch-icon" href="./favicon.svg"/>
-  <link rel="mask-icon" href="./favicon.svg" color="#000000">
+  <link id="favicon" rel="icon" href="./assets/favicon.svg" />
+  <link rel="apple-touch-icon" href="./assets/favicon.svg" />
+  <link id="mask-icon" rel="mask-icon" href="./assets/favicon.svg" color="#000000">
 
   <meta name="robots" content="noindex">
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-B1dXK3k7.js"></script>
+  <script type="module" crossorigin src="./assets/index-CLkMlXOw.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-B7GLMMVh.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmselect/vmui/manifest.json

@@ -3,7 +3,7 @@
   "name": "vmui",
   "icons": [
     {
-      "src": "favicon.svg",
+      "src": "./assets/favicon.svg",
       "sizes": "any",
       "type": "image/svg+xml"
     }

app/vmui/packages/vmui/src/api/explore-alerts.ts

@@ -1,5 +1,10 @@
-export const getGroupsUrl = (server: string, search: string, type: string, states: string[], maxGroups: number): string => {
-  return `${server}/vmalert/api/v1/rules?datasource_type=prometheus&search=${encodeURIComponent(search)}&type=${encodeURIComponent(type)}&state=${states.map(encodeURIComponent).join(",")}&group_limit=${maxGroups}&extended_states=true`;
+// vmalertSourceParam routes the request to a single vmalert from -vmalert.proxyURL.
+// An empty source means that the request is sent to all the configured vmalerts.
+const vmalertSourceParam = (source: string): string =>
+  source ? `&vmalert_source=${encodeURIComponent(source)}` : "";
+
+export const getGroupsUrl = (server: string, search: string, type: string, states: string[], maxGroups: number, source: string): string => {
+  return `${server}/vmalert/api/v1/rules?datasource_type=prometheus&search=${encodeURIComponent(search)}&type=${encodeURIComponent(type)}&state=${states.map(encodeURIComponent).join(",")}&group_limit=${maxGroups}&extended_states=true${vmalertSourceParam(source)}`;
 };
 
 export const getItemUrl = (

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/index.tsx

@@ -11,9 +11,12 @@ interface RulesHeaderProps {
   allRuleTypes: string[];
   allStates: string[];
   states: string[];
+  sources: string[];
+  allSources: string[];
   search: string;
   onChangeRuleType: (input: string) => void;
   onChangeStates: (input: string) => void;
+  onChangeSource: (input: string) => void;
   onChangeSearch: (input: string) => void;
 }
 
@@ -22,9 +25,12 @@ const RulesHeader = ({
   allRuleTypes,
   allStates,
   states,
+  sources,
+  allSources,
   search,
   onChangeRuleType,
   onChangeStates,
+  onChangeSource,
   onChangeSearch,
 }: RulesHeaderProps) => {
   const noStateText = useMemo(
@@ -67,6 +73,19 @@ const RulesHeader = ({
             searchable
           />
         </div>
+        {allSources.length > 1 && (
+          <div className="vm-explore-alerts-header__vmalert_source">
+            <Select
+              value={sources}
+              list={allSources}
+              label="vmalert source"
+              placeholder="Please select vmalert source"
+              onChange={onChangeSource}
+              includeAll
+              searchable
+            />
+          </div>
+        )}
         <div className="vm-explore-alerts-header-search">
           <TextField
             label="Search"

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/style.scss

@@ -21,6 +21,10 @@
     min-width: 150px;
   }
 
+  &__vmalert_source {
+    min-width: 180px;
+  }
+
   &-search {
     flex-grow: 1;
     .vm-text-field__input {

app/vmui/packages/vmui/src/pages/ExploreAlerts/ExploreRules.tsx

@@ -17,11 +17,13 @@ import { getQueryStringValue } from "../../utils/query-string";
 import { getChanges } from "./helpers";
 import debounce from "lodash.debounce";
 import { getStates } from "../../components/ExploreAlerts/helpers";
+import { useAppState } from "../../state/common/StateContext";
 
 const defaultRuleType = getQueryStringValue("type", "") as string;
 const defaultStatesStr = getQueryStringValue("states", "") as string;
 const defaultStates = defaultStatesStr.split("&").filter((s) => s) as string[];
 const defaultSearchInput = getQueryStringValue("search", "") as string;
+const defaultSource = getQueryStringValue("vmalert_source", "") as string;
 const TYPE_STATES: Record<string, string[]> = {
   alert:  ["inactive", "firing", "nomatch", "pending", "unhealthy"],
   record: ["unhealthy", "nomatch", "ok"],
@@ -36,8 +38,10 @@ const ExploreRules: FC = () => {
   const [searchInput, setSearchInput] = useState(defaultSearchInput);
   const [ruleType, setRuleType] = useState(defaultRuleType);
   const [states, setStates] = useState(defaultStates);
+  const [source, setSource] = useState(defaultSource);
   const [modalOpen, setModalOpen] = useState(false);
   const [searchParams, setSearchParams] = useSearchParams();
+  const { appConfig } = useAppState();
 
   useEffect(() => {
     setModalOpen(!!groupId);
@@ -47,6 +51,7 @@ const ExploreRules: FC = () => {
     type: ruleType,
     states: states.join("&"),
     search: searchInput,
+    vmalert_source: source,
     group_id: groupId,
     alert_id: alertId,
     rule_id: ruleId,
@@ -113,19 +118,28 @@ const ExploreRules: FC = () => {
     [ruleType]
   );
   const selectedRuleTypes = [ruleType].filter(Boolean);
+  // allSources is set by vmselect only when more than a single -vmalert.proxyURL is configured.
+  const allSources = useMemo(() => appConfig?.vmalert?.sources || [], [appConfig]);
+  const selectedSources = [source].filter(Boolean);
   useEffect(() => {
     if (!states.every(v => allStates.includes(v))) {
       setStates([]);
     }
   }, [states, allStates]);
+  useEffect(() => {
+    if (source && allSources.length && !allSources.includes(source)) {
+      setSource("");
+    }
+  }, [source, allSources]);
 
   const pageNumInt: number = Math.max(1, parseInt(pageNum, 10) || 1);
   const {
     groups,
     isLoading,
     error,
+    warnings,
     pageInfo,
-  } = useFetchGroups({ blockFetch: modalOpen, search: searchInput, ruleType, states, pageNum: pageNumInt, onPageChange });
+  } = useFetchGroups({ blockFetch: modalOpen, search: searchInput, ruleType, states, source, pageNum: pageNumInt, onPageChange });
 
   const handleChangeStates = useCallback((title: string) => {
     const newParams = new URLSearchParams(searchParams);
@@ -143,6 +157,14 @@ const ExploreRules: FC = () => {
     setRuleType(changes.length && changes.length !== allRuleTypes.length ? changes[0] : "");
   }, [ruleType, searchParams]);
 
+  const handleChangeSource = useCallback((title: string) => {
+    const newParams = new URLSearchParams(searchParams);
+    newParams.set("page_num", "1");
+    setSearchParams(newParams);
+    const changes = getChanges(title, selectedSources);
+    setSource(changes.length && changes.length !== allSources.length ? changes[0] : "");
+  }, [source, searchParams, allSources]);
+
   return (
     <>
       {modalOpen && getModal()}
@@ -153,11 +175,20 @@ const ExploreRules: FC = () => {
             allRuleTypes={allRuleTypes}
             states={states}
             allStates={allStates}
+            sources={selectedSources}
+            allSources={allSources}
             search={searchInput}
             onChangeRuleType={handleChangeRuleType}
             onChangeStates={handleChangeStates}
+            onChangeSource={handleChangeSource}
             onChangeSearch={debounce(handleChangeSearch, 500)}
           />
+          {warnings.map((warning) => (
+            <Alert
+              key={warning}
+              variant="warning"
+            >{warning}</Alert>
+          ))}
           <Pagination
             page={pageInfo.page}
             totalPages={pageInfo.total_pages}

app/vmui/packages/vmui/src/pages/ExploreAlerts/hooks/useFetchGroups.ts

@@ -8,6 +8,9 @@ interface FetchGroupsReturn {
   groups: Group[];
   isLoading: boolean;
   error?: ErrorTypes | string;
+  // warnings is non-empty when some of the vmalerts at -vmalert.proxyURL are unavailable.
+  // The rest of vmalerts still return their groups in this case.
+  warnings: string[];
   pageInfo: PageInfo;
 }
 
@@ -16,6 +19,7 @@ interface FetchGroupsProps {
   search: string;
   ruleType: string;
   states: string[];
+  source: string;
   pageNum: number;
   onPageChange: (num: number) => () => void;
 }
@@ -29,7 +33,7 @@ interface PageInfo {
 
 const MAX_GROUPS = 100;
 
-export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states, onPageChange }: FetchGroupsProps): FetchGroupsReturn => {
+export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states, source, onPageChange }: FetchGroupsProps): FetchGroupsReturn => {
   const { serverUrl } = useAppState();
   const { period } = useTimeState();
 
@@ -42,10 +46,11 @@ export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states,
     total_rules: 0,
   });
   const [error, setError] = useState<ErrorTypes | string>();
+  const [warnings, setWarnings] = useState<string[]>([]);
 
   const fetchUrl = useMemo(
-    () => getGroupsUrl(serverUrl, search, ruleType, states, MAX_GROUPS),
-    [serverUrl, search, ruleType, states],
+    () => getGroupsUrl(serverUrl, search, ruleType, states, MAX_GROUPS, source),
+    [serverUrl, search, ruleType, states, source],
   );
 
   const loaded = !!groups.length || !blockFetch;
@@ -66,6 +71,7 @@ export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states,
             total_groups: resp.total_groups || 0,
             total_rules: resp.total_rules || 0,
           });
+          setWarnings((resp.warnings || []) as string[]);
           setError(undefined);
         } else if (response.status === 400 && resp?.error?.includes("exceeds total amount of pages")) {
           onPageChange(1)();
@@ -83,5 +89,5 @@ export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states,
     fetchData().catch(console.error);
   }, [fetchUrl, period, loaded, pageNum]);
 
-  return { groups, isLoading, error, pageInfo };
+  return { groups, isLoading, error, warnings, pageInfo };
 };

app/vmui/packages/vmui/src/pages/ExploreAlerts/hooks/useSetQueryParams.ts

@@ -6,6 +6,7 @@ interface rulesQueryProps {
   type?: string;
   states?: string;
   search?: string;
+  vmalert_source?: string;
   rule_id: string;
   group_id: string;
   alert_id: string;
@@ -15,6 +16,7 @@ export const useRulesSetQueryParams = ({
   type,
   states,
   search,
+  vmalert_source,
   rule_id,
   alert_id,
   group_id,
@@ -26,6 +28,7 @@ export const useRulesSetQueryParams = ({
       type,
       states,
       search,
+      vmalert_source,
       alert_id,
       rule_id,
       group_id,
@@ -38,6 +41,7 @@ export const useRulesSetQueryParams = ({
     type,
     states,
     search,
+    vmalert_source,
     rule_id,
     group_id,
     alert_id,

app/vmui/packages/vmui/src/types/index.ts

@@ -184,6 +184,9 @@ export interface AppConfig {
   };
   vmalert?: {
     enabled: boolean;
+    // sources contains names of the vmalerts configured via -vmalert.proxyURL.
+    // It is set only if more than a single vmalert is configured.
+    sources?: string[];
   };
   version?: string;
 }

docs/guides/multi-regional-setup-dedicated-regions/README.md

@@ -6,237 +6,80 @@ build:
 sitemap:
   disable: true
 ---
+### Scenario
 
-## Overview {#scenario}
+Let's cover the case. You have multiple regions with workloads and want to collect metrics.
 
-This guide shows how to run VictoriaMetrics across many regions in high-availability mode. Each workload runs a local vmagent and sends metrics to dedicated monitoring deployments, so metric data is duplicated and available even if one monitoring region is down.
+The monitoring setup is in the dedicated regions as shown below:
 
-Use this architecture when you need region-level resilience and want monitoring to keep working even if one region becomes unavailable.
+![Multi-regional setup with VictoriaMetrics: Dedicated regions for monitoring](setup.webp)
 
-This setup gives you:
+Every workload region (Earth, Mars, Venus) has a vmagent that sends data to multiple regions with a monitoring setup. 
+The monitoring setup (Ground Control 1,2) contains VictoriaMetrics Time Series Database(TSDB) cluster or single.
 
-* High availability of metric data across regions.
-* A single global query endpoint.
-* Simpler disaster recovery.
+Using this schema, you can achieve:
 
-The trade-off is that you store and send the same data twice, so storage and compute requirements are increased.
-
-## Architecture
-
-The example architecture separates workloads into three regions, called Earth, Mars, and Venus. These represent the systems you want to monitor (e.g., your applications or your infrastructure). For monitoring, there are two separate regions, Ground Control 1 and 2, each running its own VictoriaMetrics deployment. The workload regions (the planets) run a local vmagent that forwards the same metrics to the two dedicated Ground Control regions.
-
-![Multi-regional setup with VictoriaMetrics: Dedicated regions for monitoring](setup-1.webp)
-{width="700"}
-
-The role of the Ground Controls can be filled by VictoriaMetrics in [single-node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) or [cluster mode](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
-
-## High Availability
-
-The architecture provides high availability by storing two full copies of the data: one in Ground Control 1 and the other in Ground Control 2. Since both store the same data, losing one region doesn't result in a monitoring outage. You can still run queries, view dashboards, and receive alerts.
-
-vmagent keeps a separate persistent queue for each `-remoteWrite.url` destination. If one Ground Control region is unavailable, vmagent continues sending data to the other region. The samples for the unavailable region stay in the file-based queue, and vmagent delivers them after the region recovers. The queue size is limited by disk space available to the vmagent or group of vmagents. This helps restore consistency across both regions.
-
-This setup provides two logical copies of the data in separate monitoring regions. That lets you fail over to the healthy region if one region becomes unavailable, or spread read load across both regions if needed.
+* Global Querying View
+* Querying all metrics from one monitoring installation
+* High Availability
+* You can lose one region, but your experience will be the same.
+* Of course, that means you duplicate your traffic twice.
 
 ### How to write the data to Ground Control regions
 
-Run one or more vmagent nodes in each workload region and configure them to send metrics to both Ground Control regions. This gives each workload region a local write path and keeps delivery going if one monitoring region is unavailable. 
-
-For example, a vmagent that sends data to two single-node VictoriaMetrics instances looks like this:
+* You need to pass two `-remoteWrite.url` command-line options to `vmagent`:
 
 ```sh
 /path/to/vmagent-prod \
-  -remoteWrite.url=https://ground-control-1:8428/api/v1/write \
-  -remoteWrite.url=https://ground-control-2:8428/api/v1/write
+  -remoteWrite.url=<ground-control-1-remote-write> \
+  -remoteWrite.url=<ground-control-2-remote-write>
 ```
 
-For a VictoriaMetrics cluster, use the following URLs for [`accountID=0`](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
+* If you scrape data from Prometheus-compatible targets, then please specify `-promscrape.config` parameter as well.
 
-```sh
-/path/to/vmagent-prod \
-  -remoteWrite.url=https://ground-control-1-vminsert:8480/insert/0/prometheus/api/v1/write \
-  -remoteWrite.url=https://ground-control-2-vminsert:8480/insert/0/prometheus/api/v1/write
-```
-For more details, see [data ingestion with vmagent](https://docs.victoriametrics.com/victoriametrics/data-ingestion/vmagent/).
-vmagent [alerting rules and dashboards](https://docs.victoriametrics.com/vmagent/index.html#monitoring) help to monitor 
-the health state of each configured destination and its queue size.
+Here is a Quickstart guide for [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#quick-start)
 
 ### How to read the data from Ground Control regions
 
-You can read data from Ground Control regions in a few different ways. The best option depends on your needs and operational complexity:
+You can use one of the following options:
 
-* Choose region via load balancer: put a load balancer in front of both Ground Control regions. Route traffic to a preferred region, with automatic failover to the other region in case of failure.
-* Merge results from multiple regions via vmselect: run a dedicated vmselect that would be configured to read from both regions and merge the results.
+1. Multi-level [vmselect setup](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup) in cluster setup, top-level vmselect(s) reads data from cluster-level vmselects
+   * Returns data in one of the clusters is unavailable  
+   * Merges data from both sources. You need to turn on [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) to remove duplicates 
+1. Regional endpoints - use one regional endpoint as default and switch to another if there is an issue.
+1. Load balancer - that sends queries to a particular region. The benefit and disadvantage of this setup is that it's simple.
+1. Promxy - proxy that reads data from multiple Prometheus-like sources. It allows reading data more intelligently to cover the region's unavailability out of the box. It doesn't support MetricsQL yet (please check this issue).
+1. Global vmselect in cluster setup - you can set up an additional subset of vmselects that knows about all storages in all regions.
+   * The [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) in 1ms on the vmselect side must be turned on. This setup allows you to query data using MetricsQL.
+   * The downside is that vmselect waits for a response from all storages in all regions.
 
-You can read more about choosing the right architecture in the [VictoriaMetrics topologies guide](https://docs.victoriametrics.com/guides/vm-architectures/).
 
-#### Load balancer
+### High Availability
 
-Use a load balancer when you want one stable query endpoint in front of your Ground Control regions. In this setup, dashboards and tools send queries to a single URL, and vmauth routes each request to one available region.
+The data is duplicated twice, and every region contains a full copy of the data. That means one region can be offline.
 
-The following diagram shows [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) performing the role of [load balancer for HA setups](https://docs.victoriametrics.com/vmauth/index.html#high-availability).
+You don't need to set up a replication factor using the VictoriaMetrics cluster.
 
-![Diagram shows vmauth between Grafana and Ground Control regions](load-balancer-vmauth.webp)
-{width="700"}
+### Alerting
 
-This approach is faster than [merging results with vmselect](#vmselect), because each query goes to only one region. It can also reduce query latency by roughly half compared with a topology that reads and merges data from both regions.
+You can set up vmalert in each Ground control region that evaluates recording and alerting rules. As every region contains a full copy of the data, you don't need to synchronize recording rules from one region to another.
 
-The main downside is that vmauth does not know whether a recovered region has already finished replaying delayed data from the vmagent queue. If you send queries to that region too early, recent data may still be incomplete. In that case, it is better to wait until the region catches up before routing traffic there.
+For alert deduplication, please use [cluster mode in Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability).
 
-For VictoriaMetrics single node, you can vmauth it with the following configuration:
+We also recommend adopting the list of [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
+for VictoriaMetrics components.
 
-```yaml
-unauthorized_user:
-  url_prefix:
-    - "http://ground-control-1:8428"
-    - "http://ground-control-2:8428"
-  load_balancing_policy: first_available
-```
+### Monitoring
 
-On the VictoriaMetrics cluster, the URLs must point to the Ground Control vmselect nodes. For example:
+An additional VictoriaMetrics single can be set up in every region, scraping metrics from the main TSDB.
 
-```yaml
-unauthorized_user:
-  url_prefix:
-    - "http://ground-control-1-vmselect:8481"
-    - "http://ground-control-2-vmselect:8481"
-  load_balancing_policy: first_available
-```
+You also may evaluate the option to send these metrics to the neighbour region to achieve HA.
 
-The examples above show how to load balance requests without authentication. You can optionally configure authentication in several ways; for more details, read the [vmauth authorization section](https://docs.victoriametrics.com/victoriametrics/vmauth/#authorization).
+Additional context
+* VictoriaMetrics Single - [https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
+* VictoriaMetrics Cluster - [https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
 
-To start vmauth with your configuration, use the `-auth.config` flag. For example:
 
-```sh
-/path/to/vmauth-prod -auth.config=/path/to/auth.yaml
-```
-
-You can test that queries work with curl:
-
-```sh
-# single node
-curl http://vmauth-node:8427/api/v1/query?query=up
-
-# cluster
-curl http://vmauth-node:8427/select/0/prometheus/api/v1/query?query=up
-```
-
-For an example of this topology in Kubernetes, see the [`VMDistributed` resource](https://docs.victoriametrics.com/helm/victoriametrics-k8s-stack/#vmdistributed-enabled).
-
-#### vmselect
-
-> This option requires that Ground Control regions are deployed in one of these modes:
-> - As a [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
-> - Or as VictoriaMetrics [single-node with multitenant support enabled](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy). In other words, VictoriaMetrics should be started with the optional `-vmselectAddr=:8401` command line flag to enable the vmselect RPC server.
-
-In this setup, each Ground Control region has its own local vmselect. A top-level vmselect queries these instead of connecting directly to vmstorage nodes.
-
-![Diagram shows top-level vmselect connecting to the regional vmselect nodes in each Ground Control cluster](top-level-vmselect.webp)
-{width="700"}
-
-This option is useful when direct access to vmstorage nodes is not practical or desirable. For example, when running on Kubernetes, the vmstorage services don't provide an HTTP query endpoint by default.
-
-To enable this setup, each Ground Control regional vmselect must listen for requests from the top layer by setting the `-clusternativeListenAddr` flag. The top-level vmselect must then use `-storageNode` to point to the regional vmselect nodes and must set a [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) interval to handle duplicated data.
-
-For example, here's how we can run the local cluster vmselect nodes and a top-level vmselect node:
-
-```sh
-# Ground Control 1 cluster vmselect
-/path/to/vmselect-prod \
-  -storageNode=ground-control-1-vmstorage-1:8401,ground-control-1-vmstorage-2:8401 \
-  -clusternativeListenAddr=:8401
-
-# Ground Control 2 cluster vmselect
-/path/to/vmselect-prod \
-  -storageNode=ground-control-2-vmstorage-1:8401,ground-control-2-vmstorage-2:8401 \
-  -clusternativeListenAddr=:8401
-
-# Top-level vmselect
-/path/to/vmselect-prod \
-  -storageNode=ground-control-1-vmselect:8401,ground-control-2-vmselect:8401 \
-  -dedup.minScrapeInterval=1ms \
-  -replicationFactor=2
-```
-
-This option provides a single query endpoint for both Ground Control regions. If one region becomes unavailable, the global vmselect can still query the healthy region, so dashboards and queries can continue to work.
-
-The main trade-off is performance. In a two-level vmselect topology, queries pass through two query layers, so they usually take longer than using regional endpoints directly, or through a load balancer. The benefit is that the topology is easy to understand; it keeps working if one region is lost, and it can merge data from both regions while one region is still catching up after recovery.
-
-## Alerting
-
-Run a vmalert node in each Ground Control region and point it to the local VictoriaMetrics endpoint. Since each region stores the same data, you can deploy the same alerting and recording rules in every region without needing cross-region rule synchronization. Send alerts to an [Alertmanager cluster](https://prometheus.io/docs/alerting/latest/alertmanager/#high-availability) to deduplicate firing alerts.
-
-![Diagram showing vmalert nodes running in each Ground Control region. An Alertmanager cluster connects to each vmalert and deduplicates notifications](vmalert-alertmanager.webp)
-{width="700"}
-
-A simple vmalert example for a single-node VictoriaMetrics looks like this:
-
-```sh
-/path/to/vmalert \
-  -rule=/path/to/rules.yaml \
-  -datasource.url=http://ground-control-1:8428 \
-  -notifier.url=http://alertmanager-1:9093 \
-  -notifier.url=http://alertmanager-2:9093
-```
-
-In VictoriaMetrics cluster mode, point `-datasource.url` to the regional vmselect endpoint. For example:
-
-```sh
-/path/to/vmalert \
-  -rule=/path/to/rules.yaml \
-  -datasource.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
-  -notifier.url=http://alertmanager-1:9093,http://alertmanager-2:9093
-```
-
-If you want vmalert to preserve alert state and recording rule results across restarts, configure `-remoteWrite.url` and `-remoteRead.url` to point to VictoriaMetrics as well. For example, for a VictoriaMetrics cluster:
-
-```sh
-/path/to/vmalert \
-  -rule=/path/to/rules.yaml \
-  -datasource.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
-  -remoteRead.url=http://ground-control-1-vmselect:8481/select/0/prometheus \
-  -remoteWrite.url=http://ground-control-1-vminsert:8480/insert/0/prometheus \
-  -notifier.url=http://alertmanager-1:9093,http://alertmanager-2:9093
-```
-
-We recommend using the list of [VictoriaMetrics alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts).
-
-## Monitoring
-
-You can monitor Ground Control instances themselves using a separate monitoring path. In this setup, each region runs its own monitoring instance that scrapes metrics from the Ground Control components.
-
-![Diagram of the original setup with monitoring of monitoring added. Each region has a dedicated VictoriaMetrics instance dedicated to monitoring the main TSDB](setup-mom-1.webp)
-{width="700"}
-
-You can optionally duplicate the monitored metrics to the neighboring region for extra resilience. That way, if a whole Ground Control region goes down, you still have access to the telemetry of the downed VictoriaMetrics instance, which can help you troubleshoot and restore service more easily.
-
-Refer to the following pages on how to monitor your VictoriaMetrics deployments:
-
-* [How to monitor VictoriaMetrics single node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
-* [How to monitor a VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#monitoring)
-
-## What more can we do?
-
-You can deploy extra vmagent instances in Ground Control regions and use them as regional ingestion proxies. This places the write endpoint closer to storage and adds another disk-backed buffer, which improves resilience when storage is temporarily unavailable.
-
-![Diagram of the original setup where a vmagent node runs in front of each Ground Control region](setup-vmagent-1.webp)
-{width="700"}
-
-This pattern is useful when you want more reliable delivery, local relabeling, or a cleaner separation between cross-region traffic and local storage ingestion.
-
-For a Ground Control running VictoriaMetrics single node, you can run vmagent as follows:
-
-```sh
-# vmagent next to Ground Control 1
-/path/to/vmagent-prod \
-  -remoteWrite.url=http://ground-control-1:8428/api/v1/write
-```
-
-If running in cluster mode, use this instead:
-
-```sh
-# vmagent next to Ground Control 1 for cluster mode
-/path/to/vmagent-prod \
-  -remoteWrite.url=http://ground-control-1-vminsert:8480/insert/0/prometheus/api/v1/write
-```
+### What more can we do?
 
+Setup vmagents in Ground Control regions. That allows it to accept data close to storage and add more reliability if storage is temporarily offline.

lib/vmalertproxy/vmalertproxy.go

@@ -1,30 +1,107 @@
 package vmalertproxy
 
 import (
+	"fmt"
 	"net/http"
-	"net/http/httputil"
+	nethttputil "net/http/httputil"
 	"net/url"
 	"strings"
 
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-// Init initializes proxying requests to the given proxyURL when calling HandleRequest.
+// SourceLabel is the meta-label added to every group, alert and notifier target
+// returned by HandleRequest when more than a single -vmalert.proxyURL is configured.
 //
-// Init must be called after flag.Parse(), since it uses command-line flags.
-func Init(proxyURL string) {
-	if len(proxyURL) == 0 {
-		return
-	}
-	pu, err := url.Parse(proxyURL)
-	if err != nil {
-		logger.Fatalf("cannot parse -vmalert.proxyURL=%q: %s", proxyURL, err)
-	}
-	vmalertProxyHost = pu.Host
-	vmalertProxy = httputil.NewSingleHostReverseProxy(pu)
+// It contains the name of the vmalert, which returned the corresponding entity.
+const SourceLabel = "__vmalert_source"
+
+// SourceQueryArg is the name of the query arg, which routes the request
+// to a single vmalert with the given name instead of fanning it out to all
+// the configured -vmalert.proxyURL.
+const SourceQueryArg = "vmalert_source"
+
+type backend struct {
+	// name is the value of SourceLabel added to entities returned by this backend.
+	name string
+
+	// u is the parsed -vmalert.proxyURL.
+	u *url.URL
+
+	rp *nethttputil.ReverseProxy
+
+	requests *metrics.Counter
+	errors   *metrics.Counter
 }
 
-// HandleRequest proxies the given request path to vmalert at proxyURL passed to Init().
+var backends []*backend
+
+// Init initializes proxying requests to the given proxyURLs when calling HandleRequest.
+//
+// proxyNames contains optional names for the vmalert at the corresponding proxyURLs.
+// Missing names are set to `vmalert_proxy_N`, where N is the one-based index of the proxyURL.
+//
+// Init must be called after flag.Parse(), since it uses command-line flags.
+func Init(proxyURLs, proxyNames []string) {
+	backends = nil
+	names := make(map[string]struct{}, len(proxyURLs))
+	for i, proxyURL := range proxyURLs {
+		if len(proxyURL) == 0 {
+			continue
+		}
+		pu, err := url.Parse(proxyURL)
+		if err != nil {
+			logger.Fatalf("cannot parse -vmalert.proxyURL=%q: %s", proxyURL, err)
+		}
+		name := fmt.Sprintf("vmalert_proxy_%d", i+1)
+		if i < len(proxyNames) && len(proxyNames[i]) > 0 {
+			name = proxyNames[i]
+		}
+		if _, ok := names[name]; ok {
+			logger.Fatalf("duplicate name %q for -vmalert.proxyURL #%d; every vmalert must have unique name at -vmalert.proxyName", name, i+1)
+		}
+		names[name] = struct{}{}
+		backends = append(backends, &backend{
+			name:     name,
+			u:        pu,
+			rp:       nethttputil.NewSingleHostReverseProxy(pu),
+			requests: metrics.GetOrCreateCounter(fmt.Sprintf(`vm_vmalert_proxy_requests_total{source=%q}`, name)),
+			errors:   metrics.GetOrCreateCounter(fmt.Sprintf(`vm_vmalert_proxy_request_errors_total{source=%q}`, name)),
+		})
+	}
+	if len(proxyNames) > len(proxyURLs) {
+		logger.Fatalf("-vmalert.proxyName cannot contain more items (%d) than -vmalert.proxyURL (%d)", len(proxyNames), len(proxyURLs))
+	}
+}
+
+// Enabled returns true if at least a single -vmalert.proxyURL is configured.
+func Enabled() bool {
+	return len(backends) > 0
+}
+
+// SourceNames returns names of the configured vmalerts in the order of -vmalert.proxyURL.
+//
+// The returned names are used as values for the SourceLabel meta-label and for the SourceQueryArg query arg.
+func SourceNames() []string {
+	names := make([]string, len(backends))
+	for i, b := range backends {
+		names[i] = b.name
+	}
+	return names
+}
+
+// HandleRequest proxies the given request path to vmalert at proxyURLs passed to Init().
+//
+// If multiple proxyURLs are configured, then requests to the alerting API are sent to all of them
+// and the responses are merged. Every returned entity is marked with the SourceLabel meta-label,
+// so it is possible to determine the vmalert it came from. Unavailable vmalerts don't fail
+// the whole request - they are reported via the `warnings` field of the response instead.
+//
+// Requests to a non-mergeable path (such as vmalert UI) are proxied to the first configured vmalert.
+// Pass SourceQueryArg query arg for proxying the request to the given vmalert only.
 func HandleRequest(w http.ResponseWriter, r *http.Request, path string) {
 	defer func() {
 		err := recover()
@@ -36,11 +113,59 @@ func HandleRequest(w http.ResponseWriter, r *http.Request, path string) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
+
+	bs := backends
+	if len(bs) == 0 {
+		writeAPIError(w, http.StatusBadRequest, "the '-vmalert.proxyURL' command-line flag must be configured; "+
+			"see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert")
+		return
+	}
+
+	if name := r.URL.Query().Get(SourceQueryArg); len(name) > 0 {
+		b := getBackendByName(name)
+		if b == nil {
+			writeAPIError(w, http.StatusBadRequest, fmt.Sprintf("unknown %s=%q; make sure it matches -vmalert.proxyName", SourceQueryArg, name))
+			return
+		}
+		b.proxyRequest(w, r, path)
+		return
+	}
+	if len(bs) == 1 {
+		bs[0].proxyRequest(w, r, path)
+		return
+	}
+	if handleFanOut(w, r, path, bs) {
+		return
+	}
+	// The path cannot be merged across multiple vmalerts (e.g. vmalert UI and its static assets).
+	// Proxy it to the first configured vmalert.
+	bs[0].proxyRequest(w, r, path)
+}
+
+func getBackendByName(name string) *backend {
+	for _, b := range backends {
+		if b.name == name {
+			return b
+		}
+	}
+	return nil
+}
+
+// proxyRequest proxies r to b as is.
+func (b *backend) proxyRequest(w http.ResponseWriter, r *http.Request, path string) {
+	b.requests.Inc()
 	req := r.Clone(r.Context())
 	req.URL.Path = path
-	req.Host = vmalertProxyHost
+	req.Host = b.u.Host
 
-	if strings.HasPrefix(r.Header.Get(`User-Agent`), `Grafana`) {
+	q := req.URL.Query()
+	changed := false
+	if q.Has(SourceQueryArg) {
+		// SourceQueryArg is consumed by HandleRequest - do not pass it to vmalert.
+		q.Del(SourceQueryArg)
+		changed = true
+	}
+	if isGrafanaRequest(r) {
 		// Grafana currently supports only Prometheus-style alerts. If other alert types
 		// (e.g. logs or traces) are returned, it may fail with "Error loading alerts".
 		//
@@ -53,16 +178,21 @@ func HandleRequest(w http.ResponseWriter, r *http.Request, path string) {
 		// See:
 		// - https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/329#issuecomment-3847585443
 		// - https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/59
-		q := req.URL.Query()
 		q.Set("datasource_type", "prometheus")
+		changed = true
+	}
+	if changed {
 		req.URL.RawQuery = q.Encode()
 		req.RequestURI = ""
 	}
 
-	vmalertProxy.ServeHTTP(w, req)
+	b.rp.ServeHTTP(w, req)
 }
 
-var (
-	vmalertProxyHost string
-	vmalertProxy     *httputil.ReverseProxy
-)
+func isGrafanaRequest(r *http.Request) bool {
+	return strings.HasPrefix(r.Header.Get(`User-Agent`), `Grafana`)
+}
+
+var proxyClient = &http.Client{
+	Transport: httputil.NewTransport(false, "vm_vmalert_proxy_client"),
+}