init

app/vmalert/config/testdata/rules/rules-query-good.rules

@@ -1,5 +1,7 @@
 groups:
   - name: alertmanager.rules
+    labels:
+      team: wow
     rules:
       - alert: AlertmanagerConfigInconsistent
         annotations:
@@ -12,4 +14,12 @@ groups:
           count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="openshift-monitoring"})) != 1
         for: 5m
         labels:
-          severity: critical
+          severity: critical
+  - name: g1
+    interval: 30s
+    labels:
+      team: wow
+    rules:
+      - alert: a1
+        expr: up>0
+        for: 5m

app/vmalert/remotewrite/client_timing_test.go

@@ -0,0 +1,46 @@
+package remotewrite
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+)
+
+func BenchmarkClientFlushEndToEnd(b *testing.B) {
+	srv := newRWServer()
+	defer srv.Close()
+
+	client, err := NewClient(context.Background(), Config{
+		Addr:          srv.URL,
+		MaxBatchSize:  10000,
+		Concurrency:   1,
+		MaxQueueSize:  100000,
+		FlushInterval: time.Hour,
+	})
+	if err != nil {
+		b.Fatalf("failed to create client: %s", err)
+	}
+	defer client.Close()
+
+	const tsCount = 100000
+	tss := make([]prompb.TimeSeries, tsCount)
+	for i := range tss {
+		tss[i] = prompb.TimeSeries{
+			Labels:  []prompb.Label{{Name: "__name__", Value: fmt.Sprintf("metric_%d", i)}},
+			Samples: []prompb.Sample{{Value: float64(i), Timestamp: 1000}},
+		}
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// WriteRequest is stack-allocated each iter; re-assigning tss is just a slice header copy
+		wr := prompb.WriteRequest{Timeseries: tss}
+		client.flush(context.Background(), &wr)
+		// flush calls wr.Reset() via defer; restore the slice for next iteration
+		wr.Timeseries = tss
+	}
+}

app/vmalert/web.go

@@ -22,6 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmalertapi"
 )
 
 var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
@@ -281,16 +282,8 @@ func (rh *requestHandler) getAlert(r *http.Request) (*rule.ApiAlert, *httpserver
 	return a, nil
 }
 
-type listGroupsResponse struct {
-	Status      string `json:"status"`
-	Page        int    `json:"page,omitempty"`
-	TotalPages  int    `json:"total_pages,omitempty"`
-	TotalGroups int    `json:"total_groups,omitempty"`
-	TotalRules  int    `json:"total_rules,omitempty"`
-	Data        struct {
-		Groups []*rule.ApiGroup `json:"groups"`
-	} `json:"data"`
-}
+// listGroupsResponse is shared with lib/vmalertproxy, which merges responses from multiple vmalerts.
+type listGroupsResponse = vmalertapi.ListGroupsResponse[*rule.ApiGroup]
 
 type groupsFilter struct {
 	groupNames []string
@@ -596,12 +589,8 @@ func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, *httpserver.Error
 	return b, nil
 }
 
-type listAlertsResponse struct {
-	Status string `json:"status"`
-	Data   struct {
-		Alerts []*rule.ApiAlert `json:"alerts"`
-	} `json:"data"`
-}
+// listAlertsResponse is shared with lib/vmalertproxy, which merges responses from multiple vmalerts.
+type listAlertsResponse = vmalertapi.ListAlertsResponse[*rule.ApiAlert]
 
 func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	rh.m.groupsMu.RLock()
@@ -665,12 +654,8 @@ func (rh *requestHandler) listAlerts(af *alertsFilter) ([]byte, *httpserver.Erro
 	return b, nil
 }
 
-type listNotifiersResponse struct {
-	Status string `json:"status"`
-	Data   struct {
-		Notifiers []*notifier.ApiNotifier `json:"notifiers"`
-	} `json:"data"`
-}
+// listNotifiersResponse is shared with lib/vmalertproxy, which merges responses from multiple vmalerts.
+type listNotifiersResponse = vmalertapi.ListNotifiersResponse[*notifier.ApiNotifier]
 
 func (rh *requestHandler) listNotifiers() ([]byte, *httpserver.ErrorWithStatusCode) {
 	targets := notifier.GetTargets()

app/vmselect/main.go

@@ -38,9 +38,15 @@ var (
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
 
-	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , "+
+	vmalertProxyURL = flagutil.NewArrayString("vmalert.proxyURL", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , "+
 		"then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules . "+
+		"If multiple URLs are set, then alerting API requests are sent to all of them and the responses are merged. Every returned group, alert and notifier target "+
+		"is marked with the `__vmalert_source` label containing the name of the vmalert it came from - see -vmalert.proxyName. "+
+		"Unavailable vmalerts do not fail the request - they are reported via the `warnings` field in the response. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert")
+	vmalertProxyName = flagutil.NewArrayString("vmalert.proxyName", "Optional name for the vmalert at the corresponding -vmalert.proxyURL. "+
+		"It is used as a value for the `__vmalert_source` label and for routing requests to a particular vmalert via `vmalert_source` query arg. "+
+		"By default the name is set to `vmalert_proxy_N`, where N is the one-based position of the corresponding -vmalert.proxyURL")
 )
 
 var slowQueries = metrics.NewCounter(`vm_slow_queries_total`)
@@ -56,10 +62,8 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
 	maxQueueDuration = vmselectMaxQueueDuration
 	concurrencyLimitCh = make(chan struct{}, maxConcurrentRequests)
 
+	vmalertproxy.Init(*vmalertProxyURL, *vmalertProxyName)
 	initVMUIConfig()
-
-	vmalertproxy.Init(*vmalertProxyURL)
-
 }
 
 // InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
@@ -527,7 +531,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 	}
 	if strings.HasPrefix(path, "/vmalert/") {
 		vmalertRequests.Inc()
-		if len(*vmalertProxyURL) == 0 {
+		if !vmalertproxy.Enabled() {
 			w.WriteHeader(http.StatusBadRequest)
 			w.Header().Set("Content-Type", "application/json")
 			fmt.Fprintf(w, "%s", `{"status":"error","msg":"the '-vmalert.proxyURL' command-line must be configured; `+
@@ -571,7 +575,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/api/v1/rules", "/rules":
 		rulesRequests.Inc()
-		if len(*vmalertProxyURL) > 0 {
+		if vmalertproxy.Enabled() {
 			vmalertproxy.HandleRequest(w, r, path)
 			return true
 		}
@@ -581,7 +585,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/api/v1/alerts", "/alerts":
 		alertsRequests.Inc()
-		if len(*vmalertProxyURL) > 0 {
+		if vmalertproxy.Enabled() {
 			vmalertproxy.HandleRequest(w, r, path)
 			return true
 		}
@@ -591,7 +595,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/api/v1/notifiers", "/notifiers":
 		notifiersRequests.Inc()
-		if len(*vmalertProxyURL) > 0 {
+		if vmalertproxy.Enabled() {
 			vmalertproxy.HandleRequest(w, r, path)
 			return true
 		}
@@ -749,6 +753,9 @@ func initVMUIConfig() {
 		} `json:"license"`
 		VMAlert struct {
 			Enabled bool `json:"enabled"`
+			// Sources contains names of the vmalerts at -vmalert.proxyURL.
+			// vmui uses them for filtering rules by the vmalert they come from.
+			Sources []string `json:"sources,omitempty"`
 		} `json:"vmalert"`
 	}
 	data, err := vmuiFiles.ReadFile("vmui/config.json")
@@ -764,7 +771,11 @@ func initVMUIConfig() {
 		// buildinfo.ShortVersion() may return empty result for builds without tags
 		cfg.Version = buildinfo.Version
 	}
-	cfg.VMAlert.Enabled = len(*vmalertProxyURL) != 0
+	cfg.VMAlert.Enabled = vmalertproxy.Enabled()
+	if names := vmalertproxy.SourceNames(); len(names) > 1 {
+		// A single vmalert needs no filtering by source.
+		cfg.VMAlert.Sources = names
+	}
 	data, err = json.Marshal(&cfg)
 	if err != nil {
 		logger.Fatalf("cannot create vmui config: %s", err)

app/vmselect/vmui/index.html

@@ -37,11 +37,11 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-BiDX4bB6.js"></script>
+  <script type="module" crossorigin src="./assets/index-CLkMlXOw.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-CymA7XYg.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-B7GLMMVh.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmui/packages/vmui/src/api/explore-alerts.ts

@@ -1,5 +1,10 @@
-export const getGroupsUrl = (server: string, search: string, type: string, states: string[], maxGroups: number): string => {
-  return `${server}/vmalert/api/v1/rules?datasource_type=prometheus&search=${encodeURIComponent(search)}&type=${encodeURIComponent(type)}&state=${states.map(encodeURIComponent).join(",")}&group_limit=${maxGroups}&extended_states=true`;
+// vmalertSourceParam routes the request to a single vmalert from -vmalert.proxyURL.
+// An empty source means that the request is sent to all the configured vmalerts.
+const vmalertSourceParam = (source: string): string =>
+  source ? `&vmalert_source=${encodeURIComponent(source)}` : "";
+
+export const getGroupsUrl = (server: string, search: string, type: string, states: string[], maxGroups: number, source: string): string => {
+  return `${server}/vmalert/api/v1/rules?datasource_type=prometheus&search=${encodeURIComponent(search)}&type=${encodeURIComponent(type)}&state=${states.map(encodeURIComponent).join(",")}&group_limit=${maxGroups}&extended_states=true${vmalertSourceParam(source)}`;
 };
 
 export const getItemUrl = (

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/index.tsx

@@ -11,9 +11,12 @@ interface RulesHeaderProps {
   allRuleTypes: string[];
   allStates: string[];
   states: string[];
+  sources: string[];
+  allSources: string[];
   search: string;
   onChangeRuleType: (input: string) => void;
   onChangeStates: (input: string) => void;
+  onChangeSource: (input: string) => void;
   onChangeSearch: (input: string) => void;
 }
 
@@ -22,9 +25,12 @@ const RulesHeader = ({
   allRuleTypes,
   allStates,
   states,
+  sources,
+  allSources,
   search,
   onChangeRuleType,
   onChangeStates,
+  onChangeSource,
   onChangeSearch,
 }: RulesHeaderProps) => {
   const noStateText = useMemo(
@@ -67,6 +73,19 @@ const RulesHeader = ({
             searchable
           />
         </div>
+        {allSources.length > 1 && (
+          <div className="vm-explore-alerts-header__vmalert_source">
+            <Select
+              value={sources}
+              list={allSources}
+              label="vmalert source"
+              placeholder="Please select vmalert source"
+              onChange={onChangeSource}
+              includeAll
+              searchable
+            />
+          </div>
+        )}
         <div className="vm-explore-alerts-header-search">
           <TextField
             label="Search"

app/vmui/packages/vmui/src/components/ExploreAlerts/RulesHeader/style.scss

@@ -21,6 +21,10 @@
     min-width: 150px;
   }
 
+  &__vmalert_source {
+    min-width: 180px;
+  }
+
   &-search {
     flex-grow: 1;
     .vm-text-field__input {

app/vmui/packages/vmui/src/pages/ExploreAlerts/ExploreRules.tsx

@@ -17,11 +17,13 @@ import { getQueryStringValue } from "../../utils/query-string";
 import { getChanges } from "./helpers";
 import debounce from "lodash.debounce";
 import { getStates } from "../../components/ExploreAlerts/helpers";
+import { useAppState } from "../../state/common/StateContext";
 
 const defaultRuleType = getQueryStringValue("type", "") as string;
 const defaultStatesStr = getQueryStringValue("states", "") as string;
 const defaultStates = defaultStatesStr.split("&").filter((s) => s) as string[];
 const defaultSearchInput = getQueryStringValue("search", "") as string;
+const defaultSource = getQueryStringValue("vmalert_source", "") as string;
 const TYPE_STATES: Record<string, string[]> = {
   alert:  ["inactive", "firing", "nomatch", "pending", "unhealthy"],
   record: ["unhealthy", "nomatch", "ok"],
@@ -36,8 +38,10 @@ const ExploreRules: FC = () => {
   const [searchInput, setSearchInput] = useState(defaultSearchInput);
   const [ruleType, setRuleType] = useState(defaultRuleType);
   const [states, setStates] = useState(defaultStates);
+  const [source, setSource] = useState(defaultSource);
   const [modalOpen, setModalOpen] = useState(false);
   const [searchParams, setSearchParams] = useSearchParams();
+  const { appConfig } = useAppState();
 
   useEffect(() => {
     setModalOpen(!!groupId);
@@ -47,6 +51,7 @@ const ExploreRules: FC = () => {
     type: ruleType,
     states: states.join("&"),
     search: searchInput,
+    vmalert_source: source,
     group_id: groupId,
     alert_id: alertId,
     rule_id: ruleId,
@@ -113,19 +118,28 @@ const ExploreRules: FC = () => {
     [ruleType]
   );
   const selectedRuleTypes = [ruleType].filter(Boolean);
+  // allSources is set by vmselect only when more than a single -vmalert.proxyURL is configured.
+  const allSources = useMemo(() => appConfig?.vmalert?.sources || [], [appConfig]);
+  const selectedSources = [source].filter(Boolean);
   useEffect(() => {
     if (!states.every(v => allStates.includes(v))) {
       setStates([]);
     }
   }, [states, allStates]);
+  useEffect(() => {
+    if (source && allSources.length && !allSources.includes(source)) {
+      setSource("");
+    }
+  }, [source, allSources]);
 
   const pageNumInt: number = Math.max(1, parseInt(pageNum, 10) || 1);
   const {
     groups,
     isLoading,
     error,
+    warnings,
     pageInfo,
-  } = useFetchGroups({ blockFetch: modalOpen, search: searchInput, ruleType, states, pageNum: pageNumInt, onPageChange });
+  } = useFetchGroups({ blockFetch: modalOpen, search: searchInput, ruleType, states, source, pageNum: pageNumInt, onPageChange });
 
   const handleChangeStates = useCallback((title: string) => {
     const newParams = new URLSearchParams(searchParams);
@@ -143,6 +157,14 @@ const ExploreRules: FC = () => {
     setRuleType(changes.length && changes.length !== allRuleTypes.length ? changes[0] : "");
   }, [ruleType, searchParams]);
 
+  const handleChangeSource = useCallback((title: string) => {
+    const newParams = new URLSearchParams(searchParams);
+    newParams.set("page_num", "1");
+    setSearchParams(newParams);
+    const changes = getChanges(title, selectedSources);
+    setSource(changes.length && changes.length !== allSources.length ? changes[0] : "");
+  }, [source, searchParams, allSources]);
+
   return (
     <>
       {modalOpen && getModal()}
@@ -153,11 +175,20 @@ const ExploreRules: FC = () => {
             allRuleTypes={allRuleTypes}
             states={states}
             allStates={allStates}
+            sources={selectedSources}
+            allSources={allSources}
             search={searchInput}
             onChangeRuleType={handleChangeRuleType}
             onChangeStates={handleChangeStates}
+            onChangeSource={handleChangeSource}
             onChangeSearch={debounce(handleChangeSearch, 500)}
           />
+          {warnings.map((warning) => (
+            <Alert
+              key={warning}
+              variant="warning"
+            >{warning}</Alert>
+          ))}
           <Pagination
             page={pageInfo.page}
             totalPages={pageInfo.total_pages}

app/vmui/packages/vmui/src/pages/ExploreAlerts/hooks/useFetchGroups.ts

@@ -8,6 +8,9 @@ interface FetchGroupsReturn {
   groups: Group[];
   isLoading: boolean;
   error?: ErrorTypes | string;
+  // warnings is non-empty when some of the vmalerts at -vmalert.proxyURL are unavailable.
+  // The rest of vmalerts still return their groups in this case.
+  warnings: string[];
   pageInfo: PageInfo;
 }
 
@@ -16,6 +19,7 @@ interface FetchGroupsProps {
   search: string;
   ruleType: string;
   states: string[];
+  source: string;
   pageNum: number;
   onPageChange: (num: number) => () => void;
 }
@@ -29,7 +33,7 @@ interface PageInfo {
 
 const MAX_GROUPS = 100;
 
-export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states, onPageChange }: FetchGroupsProps): FetchGroupsReturn => {
+export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states, source, onPageChange }: FetchGroupsProps): FetchGroupsReturn => {
   const { serverUrl } = useAppState();
   const { period } = useTimeState();
 
@@ -42,10 +46,11 @@ export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states,
     total_rules: 0,
   });
   const [error, setError] = useState<ErrorTypes | string>();
+  const [warnings, setWarnings] = useState<string[]>([]);
 
   const fetchUrl = useMemo(
-    () => getGroupsUrl(serverUrl, search, ruleType, states, MAX_GROUPS),
-    [serverUrl, search, ruleType, states],
+    () => getGroupsUrl(serverUrl, search, ruleType, states, MAX_GROUPS, source),
+    [serverUrl, search, ruleType, states, source],
   );
 
   const loaded = !!groups.length || !blockFetch;
@@ -66,6 +71,7 @@ export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states,
             total_groups: resp.total_groups || 0,
             total_rules: resp.total_rules || 0,
           });
+          setWarnings((resp.warnings || []) as string[]);
           setError(undefined);
         } else if (response.status === 400 && resp?.error?.includes("exceeds total amount of pages")) {
           onPageChange(1)();
@@ -83,5 +89,5 @@ export const useFetchGroups = ({ blockFetch, pageNum, search, ruleType, states,
     fetchData().catch(console.error);
   }, [fetchUrl, period, loaded, pageNum]);
 
-  return { groups, isLoading, error, pageInfo };
+  return { groups, isLoading, error, warnings, pageInfo };
 };

app/vmui/packages/vmui/src/pages/ExploreAlerts/hooks/useSetQueryParams.ts

@@ -6,6 +6,7 @@ interface rulesQueryProps {
   type?: string;
   states?: string;
   search?: string;
+  vmalert_source?: string;
   rule_id: string;
   group_id: string;
   alert_id: string;
@@ -15,6 +16,7 @@ export const useRulesSetQueryParams = ({
   type,
   states,
   search,
+  vmalert_source,
   rule_id,
   alert_id,
   group_id,
@@ -26,6 +28,7 @@ export const useRulesSetQueryParams = ({
       type,
       states,
       search,
+      vmalert_source,
       alert_id,
       rule_id,
       group_id,
@@ -38,6 +41,7 @@ export const useRulesSetQueryParams = ({
     type,
     states,
     search,
+    vmalert_source,
     rule_id,
     group_id,
     alert_id,

app/vmui/packages/vmui/src/types/index.ts

@@ -184,6 +184,9 @@ export interface AppConfig {
   };
   vmalert?: {
     enabled: boolean;
+    // sources contains names of the vmalerts configured via -vmalert.proxyURL.
+    // It is set only if more than a single vmalert is configured.
+    sources?: string[];
   };
   version?: string;
 }

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -95,7 +95,7 @@ See also multitenancy [via headers](#multitenancy-via-headers) and [via labels](
 
 ### Multitenancy via headers
 
-With `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag enabled (enabled by default {{% available_from "v1.150.0" %}})
+With `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag enabled (enabled by default {{% available_from "#" %}})
 tenant ID can be specified via HTTP headers `AccountID` and `ProjectID`. This flag needs to be enabled on vminserts and vmselects.
 
 With `--enableMultitenancyViaHeaders` enabled [URL format](#url-format) can be simplified to the following:

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,10 +26,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-## [v1.150.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.150.0)
-
-Release candidate
-
 **Update Note 1:** `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: default value of `-enableMultitenancyViaHeaders` command-line flag has changed from `false` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default. With this change, mentioned components will start supporting URLs with omitted tenant ID in the path: `https://<vmselect>:8481/select/prometheus/api/v1/query` will become a valid URL. To disable multitenancy via headers and simplified URLs set `--enableMultitenancyViaHeaders=false` on vmagent, vminsert and vmselect. 
 
 * SECURITY: upgrade Go builder from Go1.26.5 to Go1.26.6. See [the list of issues addressed in Go1.26.6](https://github.com/golang/go/issues?q=milestone%3AGo1.26.6%20label%3ACherryPickApproved).

docs/victoriametrics/sd_configs.md

@@ -1316,7 +1316,7 @@ The list of discovered Kuma targets is refreshed at the interval, which can be c
 
 ## linode_sd_configs
 
-Linode SD configuration {{% available_from "v1.150.0" %}} allows retrieving scrape targets from [Linode](https://www.linode.com/) instances.
+Linode SD configuration {{% available_from "#" %}} allows retrieving scrape targets from [Linode](https://www.linode.com/) instances.
 The following [Linode API](https://www.linode.com/docs/api/) token scopes are required: `linodes:read_only` and `ips:read_only`.
 
 Configuration example:

docs/victoriametrics/vmagent.md

@@ -642,7 +642,7 @@ specified via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` c
 
 vmagent can write data to multiple distinct tenants if:
 * its `-remoteWrite.url` points to the [VictoriaMetrics cluster multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
-* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` (enabled by default {{% available_from "v1.150.0" %}}) command-line flags are both set
+* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` (enabled by default {{% available_from "#" %}}) command-line flags are both set
 * clients ingest data into vmagent with the tenants specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}
 
 ```mermaid

lib/vmalertproxy/vmalertproxy.go

@@ -1,30 +1,107 @@
 package vmalertproxy
 
 import (
+	"fmt"
 	"net/http"
-	"net/http/httputil"
+	nethttputil "net/http/httputil"
 	"net/url"
 	"strings"
 
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-// Init initializes proxying requests to the given proxyURL when calling HandleRequest.
+// SourceLabel is the meta-label added to every group, alert and notifier target
+// returned by HandleRequest when more than a single -vmalert.proxyURL is configured.
 //
-// Init must be called after flag.Parse(), since it uses command-line flags.
-func Init(proxyURL string) {
-	if len(proxyURL) == 0 {
-		return
-	}
-	pu, err := url.Parse(proxyURL)
-	if err != nil {
-		logger.Fatalf("cannot parse -vmalert.proxyURL=%q: %s", proxyURL, err)
-	}
-	vmalertProxyHost = pu.Host
-	vmalertProxy = httputil.NewSingleHostReverseProxy(pu)
+// It contains the name of the vmalert, which returned the corresponding entity.
+const SourceLabel = "__vmalert_source"
+
+// SourceQueryArg is the name of the query arg, which routes the request
+// to a single vmalert with the given name instead of fanning it out to all
+// the configured -vmalert.proxyURL.
+const SourceQueryArg = "vmalert_source"
+
+type backend struct {
+	// name is the value of SourceLabel added to entities returned by this backend.
+	name string
+
+	// u is the parsed -vmalert.proxyURL.
+	u *url.URL
+
+	rp *nethttputil.ReverseProxy
+
+	requests *metrics.Counter
+	errors   *metrics.Counter
 }
 
-// HandleRequest proxies the given request path to vmalert at proxyURL passed to Init().
+var backends []*backend
+
+// Init initializes proxying requests to the given proxyURLs when calling HandleRequest.
+//
+// proxyNames contains optional names for the vmalert at the corresponding proxyURLs.
+// Missing names are set to `vmalert_proxy_N`, where N is the one-based index of the proxyURL.
+//
+// Init must be called after flag.Parse(), since it uses command-line flags.
+func Init(proxyURLs, proxyNames []string) {
+	backends = nil
+	names := make(map[string]struct{}, len(proxyURLs))
+	for i, proxyURL := range proxyURLs {
+		if len(proxyURL) == 0 {
+			continue
+		}
+		pu, err := url.Parse(proxyURL)
+		if err != nil {
+			logger.Fatalf("cannot parse -vmalert.proxyURL=%q: %s", proxyURL, err)
+		}
+		name := fmt.Sprintf("vmalert_proxy_%d", i+1)
+		if i < len(proxyNames) && len(proxyNames[i]) > 0 {
+			name = proxyNames[i]
+		}
+		if _, ok := names[name]; ok {
+			logger.Fatalf("duplicate name %q for -vmalert.proxyURL #%d; every vmalert must have unique name at -vmalert.proxyName", name, i+1)
+		}
+		names[name] = struct{}{}
+		backends = append(backends, &backend{
+			name:     name,
+			u:        pu,
+			rp:       nethttputil.NewSingleHostReverseProxy(pu),
+			requests: metrics.GetOrCreateCounter(fmt.Sprintf(`vm_vmalert_proxy_requests_total{source=%q}`, name)),
+			errors:   metrics.GetOrCreateCounter(fmt.Sprintf(`vm_vmalert_proxy_request_errors_total{source=%q}`, name)),
+		})
+	}
+	if len(proxyNames) > len(proxyURLs) {
+		logger.Fatalf("-vmalert.proxyName cannot contain more items (%d) than -vmalert.proxyURL (%d)", len(proxyNames), len(proxyURLs))
+	}
+}
+
+// Enabled returns true if at least a single -vmalert.proxyURL is configured.
+func Enabled() bool {
+	return len(backends) > 0
+}
+
+// SourceNames returns names of the configured vmalerts in the order of -vmalert.proxyURL.
+//
+// The returned names are used as values for the SourceLabel meta-label and for the SourceQueryArg query arg.
+func SourceNames() []string {
+	names := make([]string, len(backends))
+	for i, b := range backends {
+		names[i] = b.name
+	}
+	return names
+}
+
+// HandleRequest proxies the given request path to vmalert at proxyURLs passed to Init().
+//
+// If multiple proxyURLs are configured, then requests to the alerting API are sent to all of them
+// and the responses are merged. Every returned entity is marked with the SourceLabel meta-label,
+// so it is possible to determine the vmalert it came from. Unavailable vmalerts don't fail
+// the whole request - they are reported via the `warnings` field of the response instead.
+//
+// Requests to a non-mergeable path (such as vmalert UI) are proxied to the first configured vmalert.
+// Pass SourceQueryArg query arg for proxying the request to the given vmalert only.
 func HandleRequest(w http.ResponseWriter, r *http.Request, path string) {
 	defer func() {
 		err := recover()
@@ -36,11 +113,59 @@ func HandleRequest(w http.ResponseWriter, r *http.Request, path string) {
 		// Forward other panics to the caller.
 		panic(err)
 	}()
+
+	bs := backends
+	if len(bs) == 0 {
+		writeAPIError(w, http.StatusBadRequest, "the '-vmalert.proxyURL' command-line flag must be configured; "+
+			"see https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert")
+		return
+	}
+
+	if name := r.URL.Query().Get(SourceQueryArg); len(name) > 0 {
+		b := getBackendByName(name)
+		if b == nil {
+			writeAPIError(w, http.StatusBadRequest, fmt.Sprintf("unknown %s=%q; make sure it matches -vmalert.proxyName", SourceQueryArg, name))
+			return
+		}
+		b.proxyRequest(w, r, path)
+		return
+	}
+	if len(bs) == 1 {
+		bs[0].proxyRequest(w, r, path)
+		return
+	}
+	if handleFanOut(w, r, path, bs) {
+		return
+	}
+	// The path cannot be merged across multiple vmalerts (e.g. vmalert UI and its static assets).
+	// Proxy it to the first configured vmalert.
+	bs[0].proxyRequest(w, r, path)
+}
+
+func getBackendByName(name string) *backend {
+	for _, b := range backends {
+		if b.name == name {
+			return b
+		}
+	}
+	return nil
+}
+
+// proxyRequest proxies r to b as is.
+func (b *backend) proxyRequest(w http.ResponseWriter, r *http.Request, path string) {
+	b.requests.Inc()
 	req := r.Clone(r.Context())
 	req.URL.Path = path
-	req.Host = vmalertProxyHost
+	req.Host = b.u.Host
 
-	if strings.HasPrefix(r.Header.Get(`User-Agent`), `Grafana`) {
+	q := req.URL.Query()
+	changed := false
+	if q.Has(SourceQueryArg) {
+		// SourceQueryArg is consumed by HandleRequest - do not pass it to vmalert.
+		q.Del(SourceQueryArg)
+		changed = true
+	}
+	if isGrafanaRequest(r) {
 		// Grafana currently supports only Prometheus-style alerts. If other alert types
 		// (e.g. logs or traces) are returned, it may fail with "Error loading alerts".
 		//
@@ -53,16 +178,21 @@ func HandleRequest(w http.ResponseWriter, r *http.Request, path string) {
 		// See:
 		// - https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/329#issuecomment-3847585443
 		// - https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/59
-		q := req.URL.Query()
 		q.Set("datasource_type", "prometheus")
+		changed = true
+	}
+	if changed {
 		req.URL.RawQuery = q.Encode()
 		req.RequestURI = ""
 	}
 
-	vmalertProxy.ServeHTTP(w, req)
+	b.rp.ServeHTTP(w, req)
 }
 
-var (
-	vmalertProxyHost string
-	vmalertProxy     *httputil.ReverseProxy
-)
+func isGrafanaRequest(r *http.Request) bool {
+	return strings.HasPrefix(r.Header.Get(`User-Agent`), `Grafana`)
+}
+
+var proxyClient = &http.Client{
+	Transport: httputil.NewTransport(false, "vm_vmalert_proxy_client"),
+}