Update docs/victoriametrics/vmalert.md

Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
Signed-off-by: Nikolay <nik@victoriametrics.com>

app/vmagent/remotewrite/client.go

@@ -151,17 +151,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		}
 		tr.Proxy = http.ProxyURL(pu)
 	}
+
 	hc := &http.Client{
 		Transport: authCfg.NewRoundTripper(tr),
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
+	rwURL, err := url.Parse(remoteWriteURL)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	retryMaxIntervalFlag := retryMaxTime
 	if retryMaxInterval.String() != "" {
 		retryMaxIntervalFlag = retryMaxInterval
 	}
 	c := &client{
 		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
+		remoteWriteURL:   rwURL.String(),
 		authCfg:          authCfg,
 		awsCfg:           awsCfg,
 		fq:               fq,

app/vmalert/datasource/init.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -94,6 +95,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	hc := &http.Client{Transport: tr}
+	datasourceURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
+	}
+	hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
 
 	if extraParams == nil {
 		extraParams = url.Values{}
@@ -120,9 +127,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 	}
 
 	return &Client{
-		c:                &http.Client{Transport: tr},
+		c:                hc,
 		authCfg:          authCfg,
-		datasourceURL:    strings.TrimSuffix(*addr, "/"),
+		datasourceURL:    strings.TrimSuffix(datasourceURL.String(), "/"),
 		appendTypePrefix: *appendTypePrefix,
 		queryStep:        *queryStep,
 		extraParams:      extraParams,

app/vmalert/remoteread/init.go

@@ -4,12 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 )
 
@@ -76,7 +78,13 @@ func Init() (datasource.QuerierBuilder, error) {
 		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
+	c := &http.Client{Transport: tr}
+	rrURL, err := url.Parse(*addr)
+	if err != nil {
+		logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
+	}
 
+	c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +97,5 @@ func Init() (datasource.QuerierBuilder, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}
-	c := &http.Client{Transport: tr}
-	return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
+	return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
 }

app/vmalert/remotewrite/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"strings"
 	"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	if cfg.Concurrency > 0 {
 		cc = cfg.Concurrency
 	}
+	hc := &http.Client{
+		Timeout:   *sendTimeout,
+		Transport: cfg.Transport,
+	}
+	rwURL, err := url.Parse(cfg.Addr)
+	if err != nil {
+		logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
+	}
+	hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
 	c := &Client{
-		c: &http.Client{
-			Timeout:   *sendTimeout,
-			Transport: cfg.Transport,
-		},
-		addr:          strings.TrimSuffix(cfg.Addr, "/"),
+		c:             hc,
+		addr:          strings.TrimSuffix(rwURL.String(), "/"),
 		authCfg:       cfg.AuthCfg,
 		flushInterval: cfg.FlushInterval,
 		maxBatchSize:  cfg.MaxBatchSize,

dashboards/victoriametrics-cluster.json

@@ -2083,7 +2083,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/victoriametrics.json

@@ -2388,7 +2388,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/victoriametrics-cluster.json

@@ -2084,7 +2084,7 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/victoriametrics.json

@@ -2389,7 +2389,7 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/vmagent.json

@@ -2165,7 +2165,7 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vm/vmalert.json

@@ -1840,7 +1840,7 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vmagent.json

@@ -2164,7 +2164,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

dashboards/vmalert.json

@@ -1839,7 +1839,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSeу major page faults rate panel in Troubleshooting section if this metric continued to be high.",
+          "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\n\n**Lower is better.**\n\nPressure is measured as amount of time within 1sec time window the process was:\n- waiting: at least one thread was blocked on memory.\n- stalled: every thread was blocked on memory (severe pressure).\n\nElevated memory pressure can slowdown the process performance by utilizing more disk IO. Consider increasing amount of available RAM limit or decreasing the load on the process.\n\nSee major page faults rate panel in Troubleshooting section if this metric continued to be high.",
           "fieldConfig": {
             "defaults": {
               "color": {

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,11 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add client side round-robin load-balancing with `DNS` discovery. See [#2388](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2388) and these [vmagent DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmagent/#dns-urls), [vmalert DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmalert/#dns-urls).
+
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
+* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
+
 ## [v1.146.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.146.0)
 
 Released at 2026-06-22
@@ -45,11 +50,11 @@ Released at 2026-06-22
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix potential corruption of remote-write metadata `Unit` values. See [#11120](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11120). Thanks for @fxrlv for the contribution.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/),[vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/),[vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): fix rare unbounded shutdown delay when config reload takes longer than `-configCheckInterval`. See [#11107](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11107). Thanks to @PleasingFungus for contribution.
-* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548d](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): push metrics to configured `-pushmetrics.url` on shutdown when migration fails. Previously, metrics were not pushed if vmctl exited with an error. See [#11081](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11081). Thanks to @zasdaym for contribution.
 * BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): disallow restoring parts outside the configured `-storageDataPath` directory. See [710c920d](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/710c920d6083327042a309e449fae4383617d817).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly apply long tenant filters. Previously, such filters could be truncated, causing tenants to be matched incorrectly. See [#11096](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11096). Thanks for @fxrlv for the contribution.
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix corrupted metrics metadata when a response contains multiple rows. See [#11115](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11115). Thanks for @fxrlv for the contribution.
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8)
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): don't cache empty responses for tenant IDs discovery during [multitenant queries](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenant-reads). This problem was visible during integration tests when multitenant queries were executed before the first ingestion happened. See [#10982](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10982)
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).

docs/victoriametrics/url-examples.md

@@ -622,11 +622,13 @@ curl -Is http://localhost:8428/internal/resetRollupResultCache
 Cluster version of VictoriaMetrics:
 
 ```sh
-curl -Is http://<vmselect>:8481/internal/resetRollupResultCache
+curl -Is http://<vmselect>:8481/internal/resetRollupResultCache?propagate=1
 ```
 
-vmselect will propagate this call to the rest of the vmselects listed in its `-selectNode` cmd-line flag. If this
-flag isn't set, then cache need to be purged from each vmselect individually.
+vmselect will propagate this call to the rest of the vmselects listed in its `-selectNode` cmd-line flag when `propagate=1` argument is set.
+If this flag or the `propagate` argument isn't set, then cache need to be purged from each vmselect individually.
+
+If `-search.resetCacheAuthKey` is set, it will be attached to the propagation request as query argument.
 
 ### TCP and UDP
 

docs/victoriametrics/vmagent.md

@@ -431,6 +431,43 @@ and `-remoteWrite.streamAggr.config`:
 
 There is also the `-promscrape.configCheckInterval` command-line flag, which can be used to automatically reload configs from the updated `-promscrape.config` file.
 
+## DNS URLs
+
+If `vmagent` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses
+via [DNS A records](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1). The port from the original URL is appended to each discovered IP address.
+Each discovered IP address is used for round-robin balancing of write requests.
+
+DNS URLs are supported in the following places:
+
+* In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
+  `192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428/api/v1/write` is automatically resolved into
+  `-remoteWrite.url=http://192.168.1.15:8428/api/v1/write`.
+
+DNS URLs are useful when client-side HTTP load balancing is needed. A good example
+is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
+which returns multiple IP addresses for a single hostname.
+
+### DNS URLs and HTTPS
+
+When a `dns+` URL uses the `https` scheme, `vmagent` connects to the discovered
+IP addresses directly. This affects [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security)
+in two ways:
+
+* No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) is sent in the TLS handshake,
+  since the connection target is an IP address rather than a hostname.
+* The server certificate is verified against the IP address, so the verification fails
+  unless the certificate contains the corresponding
+  [IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
+
+To use `dns+` URLs with HTTPS, pass the original hostname via the `-remoteWrite.tlsServerName`
+command-line flag. It is used both as SNI and as the name the server certificate
+is verified against:
+
+```sh
+-remoteWrite.url=https://dns+victoria-metrics:8428/api/v1/write
+-remoteWrite.tlsServerName=victoria-metrics
+```
+
 ## SRV URLs
 
 If `vmagent` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
@@ -441,7 +478,7 @@ SRV URLs are supported in the following places:
 * In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
   `victoria-metrics-host:8428` TCP address, then `-remoteWrite.url=http://srv+victoria-metrics/api/v1/write` is automatically resolved into
   `-remoteWrite.url=http://victoria-metrics-host:8428/api/v1/write`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmagent`
-  uses a randomly chosen address for each connection it establishes to the remote storage.
+   performs per request round-robin load-balancing.
 
 * In scrape target addresses aka `__address__` label. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#how-to-modify-scrape-urls-in-targets) for details.
 

docs/victoriametrics/vmalert.md

@@ -1470,6 +1470,59 @@ alert_relabel_configs:
 
 The configuration file can be [hot-reloaded](#hot-config-reload).
 
+## DNS URLs
+
+If `vmalert` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses via DNS A/AAAA records.
+via [DNS A records](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1). The port from the original URL is appended to each discovered IP address.
+Each discovered IP address is used for round-robin balancing of write requests.
+
+DNS URLs are supported in the following places:
+
+* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
+  `192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428` is automatically resolved into
+  `-remoteWrite.url=http://192.168.1.15:8428`.
+
+DNS URLs are useful when client-side HTTP load balancing is needed. A good example
+is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
+which returns multiple IP addresses for a single hostname.
+
+### DNS URLs and HTTPS
+
+When a `dns+` URL uses the `https` scheme, `vmalert` connects to the discovered
+IP addresses directly. No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication)
+is sent in the TLS handshake, and the server certificate is verified against the IP address,
+which fails unless the certificate contains the corresponding
+[IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
+
+To use `dns+` URLs with HTTPS, pass the original hostname via the corresponding
+`tlsServerName` command-line flag - `-datasource.tlsServerName`, `-remoteRead.tlsServerName`
+or `-remoteWrite.tlsServerName`. It is used both as SNI and as the name the server
+certificate is verified against:
+
+```sh
+-datasource.url=https://dns+victoria-metrics:8428
+-datasource.tlsServerName=victoria-metrics
+```
+
+Alternatively, issue server certificates with IP SAN entries for every backend IP address.
+Avoid `tlsInsecureSkipVerify` flags for working around this, since they disable
+server certificate verification completely.
+
+## SRV URLs
+
+If `vmalert` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
+record into TCP address with hostname and TCP port, and then use the resulting URL when it needs to connect to it.
+
+SRV URLs are supported in the following places:
+
+* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
+  `victoria-metrics-host:8085`, then `-remoteWrite.url=http://srv+victoria-metrics:8428` is automatically resolved into
+  `-remoteWrite.url=http://victoria-metrics-host:8085`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmalert`
+   performs per request round-robin load-balancing.
+
+SRV URLs are useful when HTTP services run on different TCP ports or when their TCP ports can change over time (for instance, after a restart).
+
+
 ## Contributing
 
 `vmalert` is mostly designed and built by VictoriaMetrics community.

lib/httputil/transport_lb.go

@@ -0,0 +1,231 @@
+package httputil
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+)
+
+// NewLoadBalancerTransport returns new RoundTripper that performs round-robin HTTP requests loadbalancing
+// based on discovered backends for the given url host
+// and update url with load-balancing prefix
+//
+// It returns origin transport and url if load-balancing is not needed for given url
+func NewLoadBalancerTransport(origin http.RoundTripper, originURL *url.URL) (http.RoundTripper, *url.URL) {
+
+	modifiedURL := *originURL
+	var discoverFunc func(context.Context, string, string) ([]*backend, error)
+	switch {
+	case strings.HasPrefix(originURL.Host, "dns+"):
+		modifiedURL.Host = modifiedURL.Host[4:]
+		discoverFunc = discoverDNSBackends
+	case strings.HasPrefix(originURL.Host, "srv+"):
+		modifiedURL.Host = modifiedURL.Host[4:]
+		discoverFunc = discoverSRVBackends
+	default:
+		return origin, originURL
+	}
+	host, port, err := net.SplitHostPort(modifiedURL.Host)
+	if err != nil {
+		host = modifiedURL.Host
+		port = "80"
+		if modifiedURL.Scheme == "https" {
+			port = "443"
+		}
+	}
+	t := &loadbalancerTransport{
+		tr:           origin,
+		host:         host,
+		port:         port,
+		discoverFunc: discoverFunc,
+	}
+	t.discoverBackends()
+	return t, &modifiedURL
+}
+
+type loadbalancerTransport struct {
+	tr   http.RoundTripper
+	host string
+	port string
+
+	discoverFunc func(context.Context, string, string) ([]*backend, error)
+
+	nextDiscoveryDeadline atomic.Uint64
+	discovering           atomic.Bool
+	dbs                   atomic.Pointer[discoveredBackends]
+}
+
+type discoveredBackends struct {
+	backends []*backend
+	// n is an atomic counter, which is used for balancing load among available backends.
+	n atomic.Uint64
+}
+
+func (dbs *discoveredBackends) getBackend() *backend {
+	if len(dbs.backends) == 1 {
+		// fast path
+		return dbs.backends[0]
+	}
+	for range len(dbs.backends) {
+		idx := dbs.n.Add(1)
+		b := dbs.backends[idx%uint64(len(dbs.backends))]
+		if b.isBroken() {
+			continue
+		}
+		return b
+	}
+
+	return dbs.backends[0]
+
+}
+
+type backend struct {
+	addr           string
+	brokenDeadline atomic.Uint64
+}
+
+func (b *backend) isBroken() bool {
+	bd := b.brokenDeadline.Load()
+	if bd == 0 {
+		return false
+	}
+	ct := fasttime.UnixTimestamp()
+	return ct < bd
+}
+
+// RoundTrip implements http.RoundTripper interface
+func (lb *loadbalancerTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	dbs := lb.getBackends()
+	if dbs == nil || len(dbs.backends) == 0 {
+		return nil, fmt.Errorf("no backends found for hostname=%q", lb.host)
+	}
+
+	maxRetries := len(dbs.backends)
+	var lastErr error
+	for range maxRetries {
+		b := dbs.getBackend()
+		r2 := r.Clone(r.Context())
+		if r.GetBody != nil {
+			body, err := r.GetBody()
+			if err != nil {
+				return nil, err
+			}
+			r2.Body = body
+		}
+		r2.URL.Host = b.addr
+		if r2.Host == "" {
+			r2.Host = r.URL.Host
+		}
+		resp, err := lb.tr.RoundTrip(r2)
+		if err != nil {
+			const brokenDuration = 10 * time.Second
+			ct := fasttime.UnixTimestamp()
+			brokenDeadline := ct + uint64(brokenDuration.Seconds())
+			b.brokenDeadline.Store(brokenDeadline)
+			var dnsErr *net.DNSError
+			// perform a single retry for in case of trivial error
+			// or dns lookup error for srv discovery
+			if !netutil.IsTrivialNetworkError(err) && (errors.As(err, &dnsErr) && !dnsErr.IsNotFound) {
+				return nil, err
+			}
+			// perform the same check for retry as http.Request.isReplayable does
+			canRetry := r.Body == nil || r.Body == http.NoBody || r.GetBody != nil
+			if !canRetry {
+				return nil, err
+			}
+			lastErr = err
+			continue
+		}
+		return resp, err
+	}
+	return nil, fmt.Errorf("all backends are unavailable: %w", lastErr)
+}
+
+func (lb *loadbalancerTransport) getBackends() *discoveredBackends {
+	ct := fasttime.UnixTimestamp()
+	deadline := lb.nextDiscoveryDeadline.Load()
+	if ct < deadline || !lb.discovering.CompareAndSwap(false, true) {
+		return lb.dbs.Load()
+	}
+	lb.discoverBackends()
+	return lb.dbs.Load()
+}
+
+func (lb *loadbalancerTransport) discoverBackends() {
+	const discoveryInterval = 5 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer func() {
+		cancel()
+		ct := fasttime.UnixTimestamp()
+		nextDeadline := ct + uint64(discoveryInterval.Seconds())
+		lb.nextDiscoveryDeadline.Store(nextDeadline)
+		lb.discovering.Store(false)
+	}()
+	backends, err := lb.discoverFunc(ctx, lb.host, lb.port)
+	if err != nil {
+		logger.Errorf("cannot discover backends: %s, retry in %s", err, discoveryInterval)
+		return
+	}
+	rand.Shuffle(len(backends), func(i, j int) {
+		backends[i], backends[j] = backends[j], backends[i]
+	})
+	dbs := &discoveredBackends{
+		backends: backends,
+	}
+	lb.dbs.Store(dbs)
+}
+
+func discoverDNSBackends(ctx context.Context, host, port string) ([]*backend, error) {
+	addrs, err := netutil.Resolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return nil, fmt.Errorf("failed to lookupIPAddr for host: %q: %w", host, err)
+	}
+	backends := make([]*backend, 0, len(addrs))
+	for _, addr := range addrs {
+		if !netutil.TCP6Enabled() {
+			ip, ok := netip.AddrFromSlice(addr.IP)
+			if !ok {
+				logger.Panicf("BUG: cannot build netip Addr from slice addr: %q", addr.IP.String())
+			}
+			if !ip.Unmap().Is4() {
+				continue
+			}
+		}
+		ip := addr.IP.String()
+		if len(port) > 0 {
+			ip = net.JoinHostPort(ip, port)
+		}
+		backends = append(backends, &backend{addr: ip})
+	}
+	return backends, nil
+}
+
+func discoverSRVBackends(ctx context.Context, host, port string) ([]*backend, error) {
+	_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", host)
+	if err != nil {
+		return nil, fmt.Errorf("failed to LookupSRV records for host: %q: %w", host, err)
+	}
+	backends := make([]*backend, 0, len(addrs))
+	for _, addr := range addrs {
+		hostPort := port
+		if addr.Port > 0 {
+			hostPort = strconv.FormatUint(uint64(addr.Port), 10)
+		}
+		hostAddr := net.JoinHostPort(addr.Target, hostPort)
+		backends = append(backends, &backend{addr: hostAddr})
+	}
+	return backends, nil
+}

lib/httputil/transport_lb_test.go

@@ -0,0 +1,127 @@
+package httputil
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"sync"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
+)
+
+type testRemoteServer struct {
+	mu              sync.Mutex
+	requestsPerHost map[string]int
+
+	totalRequests int
+	firstError    error
+}
+
+func (trs *testRemoteServer) RoundTrip(r *http.Request) (*http.Response, error) {
+	trs.mu.Lock()
+	if trs.firstError != nil && trs.totalRequests == 0 {
+		err := trs.firstError
+		trs.firstError = nil
+		trs.totalRequests++
+		trs.mu.Unlock()
+		return nil, err
+	}
+	trs.totalRequests++
+
+	if trs.requestsPerHost == nil {
+		trs.requestsPerHost = make(map[string]int)
+	}
+	trs.requestsPerHost[r.URL.Host]++
+	trs.mu.Unlock()
+
+	return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil
+}
+
+type testDNSResolver struct {
+	ips []net.IPAddr
+}
+
+func (tdr *testDNSResolver) LookupSRV(_ context.Context, _, _, name string) (cname string, addrs []*net.SRV, err error) {
+	return "", nil, fmt.Errorf("unexpected LookupMX call for name=%q", name)
+}
+func (tdr *testDNSResolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) {
+	return tdr.ips, nil
+}
+
+func (tdr *testDNSResolver) LookupMX(_ context.Context, name string) ([]*net.MX, error) {
+	return nil, fmt.Errorf("unexpected LookupMX call for name=%q", name)
+}
+
+func TestLoadbalancerTransport(t *testing.T) {
+	f := func(discoveredIPs []string, trs *testRemoteServer) {
+		t.Helper()
+
+		parsedIPs := make([]net.IPAddr, 0, len(discoveredIPs))
+		for _, dIP := range discoveredIPs {
+			pIP, err := netip.ParseAddr(dIP)
+			if err != nil {
+				t.Fatalf("cannot parse IP=%q: %s", dIP, err)
+			}
+			parsedIPs = append(parsedIPs, net.IPAddr{IP: pIP.AsSlice()})
+		}
+		tdr := &testDNSResolver{ips: parsedIPs}
+		originResolver := netutil.Resolver
+		defer func() { netutil.Resolver = originResolver }()
+
+		netutil.Resolver = tdr
+		requestURL, err := url.Parse("http://dns+vmsingle.example.com:8429/api/v1/write")
+		if err != nil {
+			t.Fatalf("cannot parse url: %s", err)
+		}
+		lbt, requestURL := NewLoadBalancerTransport(trs, requestURL)
+		if len(discoveredIPs) == 0 {
+			r, err := http.NewRequest(http.MethodGet, requestURL.String(), nil)
+			if err != nil {
+				t.Fatalf("cannot create http request: %s", err)
+			}
+			_, err = lbt.RoundTrip(r)
+			if err == nil {
+				t.Fatalf("expected no backends found error")
+			}
+			return
+		}
+		expectedRequestsPerHost := 2
+		for range len(discoveredIPs) * expectedRequestsPerHost {
+			r, err := http.NewRequest(http.MethodGet, requestURL.String(), nil)
+			if err != nil {
+				t.Fatalf("cannot create http request: %s", err)
+			}
+			resp, err := lbt.RoundTrip(r)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			resp.Body.Close()
+		}
+		requestsPerHost := trs.requestsPerHost
+
+		for _, dIP := range discoveredIPs {
+			expectedHostPort := net.JoinHostPort(dIP, "8429")
+			gotRequestsPerHost, ok := requestsPerHost[expectedHostPort]
+			if !ok {
+				t.Fatalf("not found expected backend request for: %q", expectedHostPort)
+			}
+			if gotRequestsPerHost != expectedRequestsPerHost {
+				t.Fatalf("unexpected requests per host:%q %d:%d (-;+)", expectedHostPort, expectedRequestsPerHost, gotRequestsPerHost)
+			}
+		}
+	}
+	trs := testRemoteServer{}
+	f([]string{"1.1.1.1"}, &trs)
+
+	trs = testRemoteServer{}
+	f([]string{"1.1.1.1", "2.2.2.2", "5.5.5.5"}, &trs)
+
+	// empty backends, expecting error
+	trs = testRemoteServer{}
+	f([]string{}, &trs)
+
+}

lib/promrelabel/debug.qtpl

@@ -36,10 +36,12 @@ function submitRelabelDebugForm(e) {
     <div class="container-fluid">
         <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/" target="_blank">Relabeling docs</a>{% space %}
 
-        {% if targetURL != "" %}
-          <a href="metric-relabel-debug{% if targetID != "" %}?id={%s targetID %}{% endif %}">Metric relabel debug</a>
-        {% else %}
-          <a href="target-relabel-debug{% if targetID != "" %}?id={%s targetID %}{% endif %}">Target relabel debug</a>
+        {% if targetID != "" %}
+          {% if targetURL != "" %}
+            <a href="metric-relabel-debug?id={%s targetID %}">Metric relabel debug</a>
+          {% else %}
+            <a href="target-relabel-debug?id={%s targetID %}">Target relabel debug</a>
+          {% endif %}
         {% endif %}
 
         <br>

lib/promrelabel/debug.qtpl.go

@@ -80,425 +80,417 @@ func StreamRelabelDebugStepsHTML(qw422016 *qt422016.Writer, targetURL, targetID
 //line lib/promrelabel/debug.qtpl:37
 	qw422016.N().S(` `)
 //line lib/promrelabel/debug.qtpl:39
-	if targetURL != "" {
-//line lib/promrelabel/debug.qtpl:39
-		qw422016.N().S(`<a href="metric-relabel-debug`)
+	if targetID != "" {
 //line lib/promrelabel/debug.qtpl:40
-		if targetID != "" {
+		if targetURL != "" {
 //line lib/promrelabel/debug.qtpl:40
-			qw422016.N().S(`?id=`)
-//line lib/promrelabel/debug.qtpl:40
-			qw422016.E().S(targetID)
-//line lib/promrelabel/debug.qtpl:40
-		}
-//line lib/promrelabel/debug.qtpl:40
-		qw422016.N().S(`">Metric relabel debug</a>`)
+			qw422016.N().S(`<a href="metric-relabel-debug?id=`)
 //line lib/promrelabel/debug.qtpl:41
-	} else {
-//line lib/promrelabel/debug.qtpl:41
-		qw422016.N().S(`<a href="target-relabel-debug`)
-//line lib/promrelabel/debug.qtpl:42
-		if targetID != "" {
-//line lib/promrelabel/debug.qtpl:42
-			qw422016.N().S(`?id=`)
-//line lib/promrelabel/debug.qtpl:42
 			qw422016.E().S(targetID)
+//line lib/promrelabel/debug.qtpl:41
+			qw422016.N().S(`">Metric relabel debug</a>`)
 //line lib/promrelabel/debug.qtpl:42
-		}
+		} else {
 //line lib/promrelabel/debug.qtpl:42
-		qw422016.N().S(`">Target relabel debug</a>`)
+			qw422016.N().S(`<a href="target-relabel-debug?id=`)
 //line lib/promrelabel/debug.qtpl:43
+			qw422016.E().S(targetID)
+//line lib/promrelabel/debug.qtpl:43
+			qw422016.N().S(`">Target relabel debug</a>`)
+//line lib/promrelabel/debug.qtpl:44
+		}
+//line lib/promrelabel/debug.qtpl:45
 	}
-//line lib/promrelabel/debug.qtpl:43
+//line lib/promrelabel/debug.qtpl:45
 	qw422016.N().S(`<br>`)
-//line lib/promrelabel/debug.qtpl:46
+//line lib/promrelabel/debug.qtpl:48
 	if err != nil {
-//line lib/promrelabel/debug.qtpl:47
+//line lib/promrelabel/debug.qtpl:49
 		htmlcomponents.StreamErrorNotification(qw422016, err)
-//line lib/promrelabel/debug.qtpl:48
+//line lib/promrelabel/debug.qtpl:50
 	}
-//line lib/promrelabel/debug.qtpl:48
+//line lib/promrelabel/debug.qtpl:50
 	qw422016.N().S(`<div class="m-3"><form method="POST" onsubmit="submitRelabelDebugForm(event)">`)
-//line lib/promrelabel/debug.qtpl:52
+//line lib/promrelabel/debug.qtpl:54
 	streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs)
-//line lib/promrelabel/debug.qtpl:53
+//line lib/promrelabel/debug.qtpl:55
 	if targetID != "" {
-//line lib/promrelabel/debug.qtpl:53
+//line lib/promrelabel/debug.qtpl:55
 		qw422016.N().S(`<input type="hidden" name="id" value="`)
-//line lib/promrelabel/debug.qtpl:54
+//line lib/promrelabel/debug.qtpl:56
 		qw422016.E().S(targetID)
-//line lib/promrelabel/debug.qtpl:54
+//line lib/promrelabel/debug.qtpl:56
 		qw422016.N().S(`" />`)
-//line lib/promrelabel/debug.qtpl:55
+//line lib/promrelabel/debug.qtpl:57
 	}
-//line lib/promrelabel/debug.qtpl:55
+//line lib/promrelabel/debug.qtpl:57
 	qw422016.N().S(`<input type="submit" value="Submit" class="btn btn-primary m-1" />`)
-//line lib/promrelabel/debug.qtpl:57
+//line lib/promrelabel/debug.qtpl:59
 	if targetID != "" {
-//line lib/promrelabel/debug.qtpl:57
+//line lib/promrelabel/debug.qtpl:59
 		qw422016.N().S(`<button type="button" onclick="location.href='?id=`)
-//line lib/promrelabel/debug.qtpl:58
+//line lib/promrelabel/debug.qtpl:60
 		qw422016.E().S(targetID)
-//line lib/promrelabel/debug.qtpl:58
+//line lib/promrelabel/debug.qtpl:60
 		qw422016.N().S(`'" class="btn btn-secondary m-1">Reset</button>`)
-//line lib/promrelabel/debug.qtpl:59
+//line lib/promrelabel/debug.qtpl:61
 	}
-//line lib/promrelabel/debug.qtpl:59
+//line lib/promrelabel/debug.qtpl:61
 	qw422016.N().S(`</form></div><div class="row"><main class="col-12">`)
-//line lib/promrelabel/debug.qtpl:65
+//line lib/promrelabel/debug.qtpl:67
 	streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
-//line lib/promrelabel/debug.qtpl:65
+//line lib/promrelabel/debug.qtpl:67
 	qw422016.N().S(`</main></div></div></body></html>`)
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 }
 
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 func WriteRelabelDebugStepsHTML(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 }
 
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	WriteRelabelDebugStepsHTML(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 	return qs422016
-//line lib/promrelabel/debug.qtpl:71
+//line lib/promrelabel/debug.qtpl:73
 }
 
-//line lib/promrelabel/debug.qtpl:73
+//line lib/promrelabel/debug.qtpl:75
 func streamrelabelDebugFormInputs(qw422016 *qt422016.Writer, metric, relabelConfigs string) {
-//line lib/promrelabel/debug.qtpl:73
+//line lib/promrelabel/debug.qtpl:75
 	qw422016.N().S(`<div>Relabel configs:<br/><textarea name="relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">`)
-//line lib/promrelabel/debug.qtpl:76
+//line lib/promrelabel/debug.qtpl:78
 	qw422016.E().S(relabelConfigs)
-//line lib/promrelabel/debug.qtpl:76
+//line lib/promrelabel/debug.qtpl:78
 	qw422016.N().S(`</textarea></div><div>Labels:<br/><textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">`)
-//line lib/promrelabel/debug.qtpl:81
+//line lib/promrelabel/debug.qtpl:83
 	qw422016.E().S(metric)
-//line lib/promrelabel/debug.qtpl:81
+//line lib/promrelabel/debug.qtpl:83
 	qw422016.N().S(`</textarea></div>`)
-//line lib/promrelabel/debug.qtpl:83
-}
-
-//line lib/promrelabel/debug.qtpl:83
-func writerelabelDebugFormInputs(qq422016 qtio422016.Writer, metric, relabelConfigs string) {
-//line lib/promrelabel/debug.qtpl:83
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:83
-	streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs)
-//line lib/promrelabel/debug.qtpl:83
-	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:83
-}
-
-//line lib/promrelabel/debug.qtpl:83
-func relabelDebugFormInputs(metric, relabelConfigs string) string {
-//line lib/promrelabel/debug.qtpl:83
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:83
-	writerelabelDebugFormInputs(qb422016, metric, relabelConfigs)
-//line lib/promrelabel/debug.qtpl:83
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:83
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:83
-	return qs422016
-//line lib/promrelabel/debug.qtpl:83
+//line lib/promrelabel/debug.qtpl:85
 }
 
 //line lib/promrelabel/debug.qtpl:85
+func writerelabelDebugFormInputs(qq422016 qtio422016.Writer, metric, relabelConfigs string) {
+//line lib/promrelabel/debug.qtpl:85
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:85
+	streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs)
+//line lib/promrelabel/debug.qtpl:85
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:85
+}
+
+//line lib/promrelabel/debug.qtpl:85
+func relabelDebugFormInputs(metric, relabelConfigs string) string {
+//line lib/promrelabel/debug.qtpl:85
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:85
+	writerelabelDebugFormInputs(qb422016, metric, relabelConfigs)
+//line lib/promrelabel/debug.qtpl:85
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:85
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:85
+	return qs422016
+//line lib/promrelabel/debug.qtpl:85
+}
+
+//line lib/promrelabel/debug.qtpl:87
 func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep, targetURL, targetID string) {
-//line lib/promrelabel/debug.qtpl:86
+//line lib/promrelabel/debug.qtpl:88
 	if len(dss) > 0 {
-//line lib/promrelabel/debug.qtpl:86
+//line lib/promrelabel/debug.qtpl:88
 		qw422016.N().S(`<div class="m-3"><b>Original labels:</b> <samp>`)
-//line lib/promrelabel/debug.qtpl:88
+//line lib/promrelabel/debug.qtpl:90
 		streammustFormatLabels(qw422016, dss[0].In)
-//line lib/promrelabel/debug.qtpl:88
+//line lib/promrelabel/debug.qtpl:90
 		qw422016.N().S(`</samp></div>`)
-//line lib/promrelabel/debug.qtpl:90
+//line lib/promrelabel/debug.qtpl:92
 	}
-//line lib/promrelabel/debug.qtpl:90
+//line lib/promrelabel/debug.qtpl:92
 	qw422016.N().S(`<table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col" style="width: 5%">Step</th><th scope="col" style="width: 25%">Relabeling Rule</th><th scope="col" style="width: 35%">Input Labels</th><th scope="col" stile="width: 35%">Output labels</a></tr></thead><tbody>`)
-//line lib/promrelabel/debug.qtpl:101
-	for i, ds := range dss {
 //line lib/promrelabel/debug.qtpl:103
+	for i, ds := range dss {
+//line lib/promrelabel/debug.qtpl:105
 		inLabels, inErr := promutil.NewLabelsFromString(ds.In)
 		outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
 		changedLabels := getChangedLabelNames(inLabels, outLabels)
 
-//line lib/promrelabel/debug.qtpl:106
+//line lib/promrelabel/debug.qtpl:108
 		qw422016.N().S(`<tr><td>`)
-//line lib/promrelabel/debug.qtpl:108
+//line lib/promrelabel/debug.qtpl:110
 		qw422016.N().D(i)
-//line lib/promrelabel/debug.qtpl:108
+//line lib/promrelabel/debug.qtpl:110
 		qw422016.N().S(`</td><td><b><pre class="m-2">`)
-//line lib/promrelabel/debug.qtpl:109
+//line lib/promrelabel/debug.qtpl:111
 		qw422016.E().S(ds.Rule)
-//line lib/promrelabel/debug.qtpl:109
+//line lib/promrelabel/debug.qtpl:111
 		qw422016.N().S(`</pre></b></td><td>`)
-//line lib/promrelabel/debug.qtpl:111
+//line lib/promrelabel/debug.qtpl:113
 		if inErr == nil {
-//line lib/promrelabel/debug.qtpl:111
+//line lib/promrelabel/debug.qtpl:113
 			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="deleted and updated labels highlighted in red">`)
-//line lib/promrelabel/debug.qtpl:113
+//line lib/promrelabel/debug.qtpl:115
 			streamlabelsWithHighlight(qw422016, inLabels, changedLabels, "#D15757")
-//line lib/promrelabel/debug.qtpl:113
+//line lib/promrelabel/debug.qtpl:115
 			qw422016.N().S(`</div>`)
-//line lib/promrelabel/debug.qtpl:115
+//line lib/promrelabel/debug.qtpl:117
 		} else {
-//line lib/promrelabel/debug.qtpl:115
+//line lib/promrelabel/debug.qtpl:117
 			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing input labels"><pre>`)
-//line lib/promrelabel/debug.qtpl:117
-			qw422016.E().S(inErr.Error())
-//line lib/promrelabel/debug.qtpl:117
-			qw422016.N().S(`</pre></div>`)
 //line lib/promrelabel/debug.qtpl:119
-			break
-//line lib/promrelabel/debug.qtpl:120
-		}
-//line lib/promrelabel/debug.qtpl:120
-		qw422016.N().S(`</td><td>`)
-//line lib/promrelabel/debug.qtpl:123
-		if outErr == nil {
-//line lib/promrelabel/debug.qtpl:123
-			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="added and updated labels highlighted in blue">`)
-//line lib/promrelabel/debug.qtpl:125
-			streamlabelsWithHighlight(qw422016, outLabels, changedLabels, "#4495e0")
-//line lib/promrelabel/debug.qtpl:125
-			qw422016.N().S(`</div>`)
-//line lib/promrelabel/debug.qtpl:127
-		} else {
-//line lib/promrelabel/debug.qtpl:127
-			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing output labels"><pre>`)
-//line lib/promrelabel/debug.qtpl:129
-			qw422016.E().S(outErr.Error())
-//line lib/promrelabel/debug.qtpl:129
+			qw422016.E().S(inErr.Error())
+//line lib/promrelabel/debug.qtpl:119
 			qw422016.N().S(`</pre></div>`)
-//line lib/promrelabel/debug.qtpl:131
+//line lib/promrelabel/debug.qtpl:121
 			break
-//line lib/promrelabel/debug.qtpl:132
+//line lib/promrelabel/debug.qtpl:122
 		}
-//line lib/promrelabel/debug.qtpl:132
-		qw422016.N().S(`</td></tr>`)
-//line lib/promrelabel/debug.qtpl:135
-	}
-//line lib/promrelabel/debug.qtpl:135
-	qw422016.N().S(`</tbody></table>`)
-//line lib/promrelabel/debug.qtpl:138
-	if len(dss) > 0 {
-//line lib/promrelabel/debug.qtpl:138
-		qw422016.N().S(`<div class="m-3"><b>Resulting labels:</b> <samp>`)
-//line lib/promrelabel/debug.qtpl:140
-		streammustFormatLabels(qw422016, dss[len(dss)-1].Out)
-//line lib/promrelabel/debug.qtpl:140
-		qw422016.N().S(`</samp>`)
-//line lib/promrelabel/debug.qtpl:141
-		if targetURL != "" {
-//line lib/promrelabel/debug.qtpl:141
-			qw422016.N().S(`<div><b>Target URL:</b>`)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(` `)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(`<a href="`)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.E().S(targetURL)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(`" target="_blank">`)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.E().S(targetURL)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(`</a>`)
-//line lib/promrelabel/debug.qtpl:144
-			if targetID != "" {
-//line lib/promrelabel/debug.qtpl:145
-				qw422016.N().S(` `)
-//line lib/promrelabel/debug.qtpl:145
-				qw422016.N().S(`(<a href="target_response?id=`)
-//line lib/promrelabel/debug.qtpl:146
-				qw422016.E().S(targetID)
-//line lib/promrelabel/debug.qtpl:146
-				qw422016.N().S(`" target="_blank" title="click to fetch target response on behalf of the scraper">response</a>)`)
-//line lib/promrelabel/debug.qtpl:147
-			}
-//line lib/promrelabel/debug.qtpl:147
+//line lib/promrelabel/debug.qtpl:122
+		qw422016.N().S(`</td><td>`)
+//line lib/promrelabel/debug.qtpl:125
+		if outErr == nil {
+//line lib/promrelabel/debug.qtpl:125
+			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="added and updated labels highlighted in blue">`)
+//line lib/promrelabel/debug.qtpl:127
+			streamlabelsWithHighlight(qw422016, outLabels, changedLabels, "#4495e0")
+//line lib/promrelabel/debug.qtpl:127
 			qw422016.N().S(`</div>`)
-//line lib/promrelabel/debug.qtpl:149
+//line lib/promrelabel/debug.qtpl:129
+		} else {
+//line lib/promrelabel/debug.qtpl:129
+			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing output labels"><pre>`)
+//line lib/promrelabel/debug.qtpl:131
+			qw422016.E().S(outErr.Error())
+//line lib/promrelabel/debug.qtpl:131
+			qw422016.N().S(`</pre></div>`)
+//line lib/promrelabel/debug.qtpl:133
+			break
+//line lib/promrelabel/debug.qtpl:134
 		}
-//line lib/promrelabel/debug.qtpl:149
-		qw422016.N().S(`</div>`)
-//line lib/promrelabel/debug.qtpl:151
+//line lib/promrelabel/debug.qtpl:134
+		qw422016.N().S(`</td></tr>`)
+//line lib/promrelabel/debug.qtpl:137
 	}
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:137
+	qw422016.N().S(`</tbody></table>`)
+//line lib/promrelabel/debug.qtpl:140
+	if len(dss) > 0 {
+//line lib/promrelabel/debug.qtpl:140
+		qw422016.N().S(`<div class="m-3"><b>Resulting labels:</b> <samp>`)
+//line lib/promrelabel/debug.qtpl:142
+		streammustFormatLabels(qw422016, dss[len(dss)-1].Out)
+//line lib/promrelabel/debug.qtpl:142
+		qw422016.N().S(`</samp>`)
+//line lib/promrelabel/debug.qtpl:143
+		if targetURL != "" {
+//line lib/promrelabel/debug.qtpl:143
+			qw422016.N().S(`<div><b>Target URL:</b>`)
+//line lib/promrelabel/debug.qtpl:145
+			qw422016.N().S(` `)
+//line lib/promrelabel/debug.qtpl:145
+			qw422016.N().S(`<a href="`)
+//line lib/promrelabel/debug.qtpl:145
+			qw422016.E().S(targetURL)
+//line lib/promrelabel/debug.qtpl:145
+			qw422016.N().S(`" target="_blank">`)
+//line lib/promrelabel/debug.qtpl:145
+			qw422016.E().S(targetURL)
+//line lib/promrelabel/debug.qtpl:145
+			qw422016.N().S(`</a>`)
+//line lib/promrelabel/debug.qtpl:146
+			if targetID != "" {
+//line lib/promrelabel/debug.qtpl:147
+				qw422016.N().S(` `)
+//line lib/promrelabel/debug.qtpl:147
+				qw422016.N().S(`(<a href="target_response?id=`)
+//line lib/promrelabel/debug.qtpl:148
+				qw422016.E().S(targetID)
+//line lib/promrelabel/debug.qtpl:148
+				qw422016.N().S(`" target="_blank" title="click to fetch target response on behalf of the scraper">response</a>)`)
+//line lib/promrelabel/debug.qtpl:149
+			}
+//line lib/promrelabel/debug.qtpl:149
+			qw422016.N().S(`</div>`)
+//line lib/promrelabel/debug.qtpl:151
+		}
+//line lib/promrelabel/debug.qtpl:151
+		qw422016.N().S(`</div>`)
+//line lib/promrelabel/debug.qtpl:153
+	}
+//line lib/promrelabel/debug.qtpl:154
 }
 
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep, targetURL, targetID string) {
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 }
 
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) string {
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	writerelabelDebugSteps(qb422016, dss, targetURL, targetID)
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 	return qs422016
-//line lib/promrelabel/debug.qtpl:152
+//line lib/promrelabel/debug.qtpl:154
 }
 
-//line lib/promrelabel/debug.qtpl:154
+//line lib/promrelabel/debug.qtpl:156
 func StreamRelabelDebugStepsJSON(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
-//line lib/promrelabel/debug.qtpl:154
+//line lib/promrelabel/debug.qtpl:156
 	qw422016.N().S(`{`)
-//line lib/promrelabel/debug.qtpl:156
-	if err != nil {
-//line lib/promrelabel/debug.qtpl:156
-		qw422016.N().S(`"status": "error","error":`)
 //line lib/promrelabel/debug.qtpl:158
-		qw422016.N().Q(fmt.Sprintf("Error: %s", err))
-//line lib/promrelabel/debug.qtpl:159
-	} else {
+	if err != nil {
+//line lib/promrelabel/debug.qtpl:158
+		qw422016.N().S(`"status": "error","error":`)
 //line lib/promrelabel/debug.qtpl:160
+		qw422016.N().Q(fmt.Sprintf("Error: %s", err))
+//line lib/promrelabel/debug.qtpl:161
+	} else {
+//line lib/promrelabel/debug.qtpl:162
 		var hasError bool
 
-//line lib/promrelabel/debug.qtpl:160
+//line lib/promrelabel/debug.qtpl:162
 		qw422016.N().S(`"status": "success","steps": [`)
-//line lib/promrelabel/debug.qtpl:163
-		for i, ds := range dss {
 //line lib/promrelabel/debug.qtpl:165
+		for i, ds := range dss {
+//line lib/promrelabel/debug.qtpl:167
 			inLabels, inErr := promutil.NewLabelsFromString(ds.In)
 			outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
 			changedLabels := getChangedLabelNames(inLabels, outLabels)
 
-//line lib/promrelabel/debug.qtpl:168
+//line lib/promrelabel/debug.qtpl:170
 			qw422016.N().S(`{"inLabels":`)
-//line lib/promrelabel/debug.qtpl:170
+//line lib/promrelabel/debug.qtpl:172
 			qw422016.N().Q(labelsWithHighlight(inLabels, changedLabels, "#D15757"))
-//line lib/promrelabel/debug.qtpl:170
+//line lib/promrelabel/debug.qtpl:172
 			qw422016.N().S(`,"outLabels":`)
-//line lib/promrelabel/debug.qtpl:171
+//line lib/promrelabel/debug.qtpl:173
 			qw422016.N().Q(labelsWithHighlight(outLabels, changedLabels, "#4495e0"))
-//line lib/promrelabel/debug.qtpl:171
+//line lib/promrelabel/debug.qtpl:173
 			qw422016.N().S(`,"rule":`)
-//line lib/promrelabel/debug.qtpl:172
+//line lib/promrelabel/debug.qtpl:174
 			qw422016.N().Q(ds.Rule)
-//line lib/promrelabel/debug.qtpl:172
+//line lib/promrelabel/debug.qtpl:174
 			qw422016.N().S(`,"errors": {`)
-//line lib/promrelabel/debug.qtpl:174
-			if inErr != nil {
-//line lib/promrelabel/debug.qtpl:174
-				qw422016.N().S(`"inLabels":`)
-//line lib/promrelabel/debug.qtpl:175
-				qw422016.N().Q(`<span style="color: #D15757">` + inErr.Error() + `</span>`)
-//line lib/promrelabel/debug.qtpl:175
-				if outErr != nil {
-//line lib/promrelabel/debug.qtpl:175
-					qw422016.N().S(`,`)
-//line lib/promrelabel/debug.qtpl:175
-				}
 //line lib/promrelabel/debug.qtpl:176
-				hasError = true
-
+			if inErr != nil {
+//line lib/promrelabel/debug.qtpl:176
+				qw422016.N().S(`"inLabels":`)
 //line lib/promrelabel/debug.qtpl:177
-			} else {
+				qw422016.N().Q(`<span style="color: #D15757">` + inErr.Error() + `</span>`)
+//line lib/promrelabel/debug.qtpl:177
+				if outErr != nil {
+//line lib/promrelabel/debug.qtpl:177
+					qw422016.N().S(`,`)
+//line lib/promrelabel/debug.qtpl:177
+				}
 //line lib/promrelabel/debug.qtpl:178
-			}
-//line lib/promrelabel/debug.qtpl:179
-			if outErr != nil {
-//line lib/promrelabel/debug.qtpl:179
-				qw422016.N().S(`"outLabels":`)
-//line lib/promrelabel/debug.qtpl:180
-				qw422016.N().Q(`<span style="color: #D15757">` + outErr.Error() + `</span>`)
-//line lib/promrelabel/debug.qtpl:181
 				hasError = true
 
-//line lib/promrelabel/debug.qtpl:182
+//line lib/promrelabel/debug.qtpl:179
+			} else {
+//line lib/promrelabel/debug.qtpl:180
 			}
+//line lib/promrelabel/debug.qtpl:181
+			if outErr != nil {
+//line lib/promrelabel/debug.qtpl:181
+				qw422016.N().S(`"outLabels":`)
 //line lib/promrelabel/debug.qtpl:182
+				qw422016.N().Q(`<span style="color: #D15757">` + outErr.Error() + `</span>`)
+//line lib/promrelabel/debug.qtpl:183
+				hasError = true
+
+//line lib/promrelabel/debug.qtpl:184
+			}
+//line lib/promrelabel/debug.qtpl:184
 			qw422016.N().S(`}}`)
-//line lib/promrelabel/debug.qtpl:185
+//line lib/promrelabel/debug.qtpl:187
 			if i != len(dss)-1 {
-//line lib/promrelabel/debug.qtpl:185
+//line lib/promrelabel/debug.qtpl:187
 				qw422016.N().S(`,`)
-//line lib/promrelabel/debug.qtpl:185
+//line lib/promrelabel/debug.qtpl:187
 			}
-//line lib/promrelabel/debug.qtpl:186
+//line lib/promrelabel/debug.qtpl:188
 		}
-//line lib/promrelabel/debug.qtpl:186
+//line lib/promrelabel/debug.qtpl:188
 		qw422016.N().S(`]`)
-//line lib/promrelabel/debug.qtpl:188
+//line lib/promrelabel/debug.qtpl:190
 		if len(dss) > 0 && !hasError {
-//line lib/promrelabel/debug.qtpl:188
+//line lib/promrelabel/debug.qtpl:190
 			qw422016.N().S(`,"originalLabels":`)
-//line lib/promrelabel/debug.qtpl:190
-			qw422016.N().Q(mustFormatLabels(dss[0].In))
-//line lib/promrelabel/debug.qtpl:190
-			qw422016.N().S(`,"resultingLabels":`)
-//line lib/promrelabel/debug.qtpl:191
-			qw422016.N().Q(mustFormatLabels(dss[len(dss)-1].Out))
 //line lib/promrelabel/debug.qtpl:192
+			qw422016.N().Q(mustFormatLabels(dss[0].In))
+//line lib/promrelabel/debug.qtpl:192
+			qw422016.N().S(`,"resultingLabels":`)
+//line lib/promrelabel/debug.qtpl:193
+			qw422016.N().Q(mustFormatLabels(dss[len(dss)-1].Out))
+//line lib/promrelabel/debug.qtpl:194
 		}
-//line lib/promrelabel/debug.qtpl:193
+//line lib/promrelabel/debug.qtpl:195
 	}
-//line lib/promrelabel/debug.qtpl:193
+//line lib/promrelabel/debug.qtpl:195
 	qw422016.N().S(`}`)
-//line lib/promrelabel/debug.qtpl:195
-}
-
-//line lib/promrelabel/debug.qtpl:195
-func WriteRelabelDebugStepsJSON(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
-//line lib/promrelabel/debug.qtpl:195
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:195
-	StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
-//line lib/promrelabel/debug.qtpl:195
-	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:195
-}
-
-//line lib/promrelabel/debug.qtpl:195
-func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
-//line lib/promrelabel/debug.qtpl:195
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:195
-	WriteRelabelDebugStepsJSON(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
-//line lib/promrelabel/debug.qtpl:195
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:195
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:195
-	return qs422016
-//line lib/promrelabel/debug.qtpl:195
+//line lib/promrelabel/debug.qtpl:197
 }
 
 //line lib/promrelabel/debug.qtpl:197
-func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
+func WriteRelabelDebugStepsJSON(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
+//line lib/promrelabel/debug.qtpl:197
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:197
+	StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
+//line lib/promrelabel/debug.qtpl:197
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:197
+}
+
+//line lib/promrelabel/debug.qtpl:197
+func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
+//line lib/promrelabel/debug.qtpl:197
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:197
+	WriteRelabelDebugStepsJSON(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
+//line lib/promrelabel/debug.qtpl:197
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:197
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:197
+	return qs422016
+//line lib/promrelabel/debug.qtpl:197
+}
+
 //line lib/promrelabel/debug.qtpl:199
+func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
+//line lib/promrelabel/debug.qtpl:201
 	labelsList := labels.GetLabels()
 	metricName := ""
 	for i, label := range labelsList {
@@ -509,153 +501,153 @@ func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Label
 		}
 	}
 
-//line lib/promrelabel/debug.qtpl:209
+//line lib/promrelabel/debug.qtpl:211
 	if metricName != "" {
-//line lib/promrelabel/debug.qtpl:210
-		if _, ok := highlight["__name__"]; ok {
-//line lib/promrelabel/debug.qtpl:210
-			qw422016.N().S(`<span style="font-weight:bold;color:`)
-//line lib/promrelabel/debug.qtpl:211
-			qw422016.E().S(color)
-//line lib/promrelabel/debug.qtpl:211
-			qw422016.N().S(`">`)
-//line lib/promrelabel/debug.qtpl:211
-			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:211
-			qw422016.N().S(`</span>`)
 //line lib/promrelabel/debug.qtpl:212
-		} else {
+		if _, ok := highlight["__name__"]; ok {
+//line lib/promrelabel/debug.qtpl:212
+			qw422016.N().S(`<span style="font-weight:bold;color:`)
+//line lib/promrelabel/debug.qtpl:213
+			qw422016.E().S(color)
+//line lib/promrelabel/debug.qtpl:213
+			qw422016.N().S(`">`)
 //line lib/promrelabel/debug.qtpl:213
 			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:214
-		}
-//line lib/promrelabel/debug.qtpl:215
-		if len(labelsList) == 0 {
-//line lib/promrelabel/debug.qtpl:215
-			return
-//line lib/promrelabel/debug.qtpl:215
-		}
-//line lib/promrelabel/debug.qtpl:216
-	}
-//line lib/promrelabel/debug.qtpl:216
-	qw422016.N().S(`{`)
-//line lib/promrelabel/debug.qtpl:218
-	for i, label := range labelsList {
-//line lib/promrelabel/debug.qtpl:219
-		if _, ok := highlight[label.Name]; ok {
-//line lib/promrelabel/debug.qtpl:219
-			qw422016.N().S(`<span style="font-weight:bold;color:`)
-//line lib/promrelabel/debug.qtpl:220
-			qw422016.E().S(color)
-//line lib/promrelabel/debug.qtpl:220
-			qw422016.N().S(`">`)
-//line lib/promrelabel/debug.qtpl:220
-			qw422016.E().S(label.Name)
-//line lib/promrelabel/debug.qtpl:220
-			qw422016.N().S(`=`)
-//line lib/promrelabel/debug.qtpl:220
-			qw422016.E().Q(label.Value)
-//line lib/promrelabel/debug.qtpl:220
+//line lib/promrelabel/debug.qtpl:213
 			qw422016.N().S(`</span>`)
-//line lib/promrelabel/debug.qtpl:221
+//line lib/promrelabel/debug.qtpl:214
 		} else {
+//line lib/promrelabel/debug.qtpl:215
+			qw422016.E().S(metricName)
+//line lib/promrelabel/debug.qtpl:216
+		}
+//line lib/promrelabel/debug.qtpl:217
+		if len(labelsList) == 0 {
+//line lib/promrelabel/debug.qtpl:217
+			return
+//line lib/promrelabel/debug.qtpl:217
+		}
+//line lib/promrelabel/debug.qtpl:218
+	}
+//line lib/promrelabel/debug.qtpl:218
+	qw422016.N().S(`{`)
+//line lib/promrelabel/debug.qtpl:220
+	for i, label := range labelsList {
+//line lib/promrelabel/debug.qtpl:221
+		if _, ok := highlight[label.Name]; ok {
+//line lib/promrelabel/debug.qtpl:221
+			qw422016.N().S(`<span style="font-weight:bold;color:`)
+//line lib/promrelabel/debug.qtpl:222
+			qw422016.E().S(color)
+//line lib/promrelabel/debug.qtpl:222
+			qw422016.N().S(`">`)
 //line lib/promrelabel/debug.qtpl:222
 			qw422016.E().S(label.Name)
 //line lib/promrelabel/debug.qtpl:222
 			qw422016.N().S(`=`)
 //line lib/promrelabel/debug.qtpl:222
 			qw422016.E().Q(label.Value)
+//line lib/promrelabel/debug.qtpl:222
+			qw422016.N().S(`</span>`)
 //line lib/promrelabel/debug.qtpl:223
-		}
+		} else {
 //line lib/promrelabel/debug.qtpl:224
+			qw422016.E().S(label.Name)
+//line lib/promrelabel/debug.qtpl:224
+			qw422016.N().S(`=`)
+//line lib/promrelabel/debug.qtpl:224
+			qw422016.E().Q(label.Value)
+//line lib/promrelabel/debug.qtpl:225
+		}
+//line lib/promrelabel/debug.qtpl:226
 		if i < len(labelsList)-1 {
-//line lib/promrelabel/debug.qtpl:224
+//line lib/promrelabel/debug.qtpl:226
 			qw422016.N().S(`,`)
-//line lib/promrelabel/debug.qtpl:224
+//line lib/promrelabel/debug.qtpl:226
 			qw422016.N().S(` `)
-//line lib/promrelabel/debug.qtpl:224
+//line lib/promrelabel/debug.qtpl:226
 		}
-//line lib/promrelabel/debug.qtpl:225
+//line lib/promrelabel/debug.qtpl:227
 	}
-//line lib/promrelabel/debug.qtpl:225
+//line lib/promrelabel/debug.qtpl:227
 	qw422016.N().S(`}`)
-//line lib/promrelabel/debug.qtpl:227
-}
-
-//line lib/promrelabel/debug.qtpl:227
-func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
-//line lib/promrelabel/debug.qtpl:227
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:227
-	streamlabelsWithHighlight(qw422016, labels, highlight, color)
-//line lib/promrelabel/debug.qtpl:227
-	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:227
-}
-
-//line lib/promrelabel/debug.qtpl:227
-func labelsWithHighlight(labels *promutil.Labels, highlight map[string]struct{}, color string) string {
-//line lib/promrelabel/debug.qtpl:227
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:227
-	writelabelsWithHighlight(qb422016, labels, highlight, color)
-//line lib/promrelabel/debug.qtpl:227
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:227
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:227
-	return qs422016
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:229
 }
 
 //line lib/promrelabel/debug.qtpl:229
+func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
+//line lib/promrelabel/debug.qtpl:229
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:229
+	streamlabelsWithHighlight(qw422016, labels, highlight, color)
+//line lib/promrelabel/debug.qtpl:229
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:229
+}
+
+//line lib/promrelabel/debug.qtpl:229
+func labelsWithHighlight(labels *promutil.Labels, highlight map[string]struct{}, color string) string {
+//line lib/promrelabel/debug.qtpl:229
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:229
+	writelabelsWithHighlight(qb422016, labels, highlight, color)
+//line lib/promrelabel/debug.qtpl:229
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:229
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:229
+	return qs422016
+//line lib/promrelabel/debug.qtpl:229
+}
+
+//line lib/promrelabel/debug.qtpl:231
 func streammustFormatLabels(qw422016 *qt422016.Writer, s string) {
-//line lib/promrelabel/debug.qtpl:230
+//line lib/promrelabel/debug.qtpl:232
 	labels, err := promutil.NewLabelsFromString(s)
 
-//line lib/promrelabel/debug.qtpl:231
-	if err != nil {
-//line lib/promrelabel/debug.qtpl:231
-		qw422016.N().S(`<span style="color: red" title="error parsing labels:`)
-//line lib/promrelabel/debug.qtpl:232
-		qw422016.E().S(err.Error())
-//line lib/promrelabel/debug.qtpl:232
-		qw422016.N().S(`">`)
-//line lib/promrelabel/debug.qtpl:232
-		qw422016.E().S("error parsing labels: " + err.Error())
-//line lib/promrelabel/debug.qtpl:232
-		qw422016.N().S(`</span>`)
 //line lib/promrelabel/debug.qtpl:233
-	} else {
+	if err != nil {
+//line lib/promrelabel/debug.qtpl:233
+		qw422016.N().S(`<span style="color: red" title="error parsing labels:`)
 //line lib/promrelabel/debug.qtpl:234
-		streamlabelsWithHighlight(qw422016, labels, nil, "")
+		qw422016.E().S(err.Error())
+//line lib/promrelabel/debug.qtpl:234
+		qw422016.N().S(`">`)
+//line lib/promrelabel/debug.qtpl:234
+		qw422016.E().S("error parsing labels: " + err.Error())
+//line lib/promrelabel/debug.qtpl:234
+		qw422016.N().S(`</span>`)
 //line lib/promrelabel/debug.qtpl:235
+	} else {
+//line lib/promrelabel/debug.qtpl:236
+		streamlabelsWithHighlight(qw422016, labels, nil, "")
+//line lib/promrelabel/debug.qtpl:237
 	}
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 }
 
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 func writemustFormatLabels(qq422016 qtio422016.Writer, s string) {
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	streammustFormatLabels(qw422016, s)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 }
 
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 func mustFormatLabels(s string) string {
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	writemustFormatLabels(qb422016, s)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 	return qs422016
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:238
 }

lib/streamaggr/rate.go

@@ -108,7 +108,7 @@ func (av *rateAggrValue) pushSample(c aggrConfig, sample *pushSample, key string
 	}
 	if ok {
 		state = sv.getState(av.isGreen)
-		if sample.timestamp < state.timestamp {
+		if sample.timestamp < state.timestamp || sample.timestamp < sv.prevTimestamp {
 			// Skip out of order sample
 			return
 		}
@@ -143,9 +143,6 @@ func (av *rateAggrValue) flush(c aggrConfig, ctx *flushCtx, key string, isLast b
 			putRateAggrSharedValue(sv)
 			continue
 		}
-		if sv.prevTimestamp == 0 {
-			continue
-		}
 		state = sv.getState(av.isGreen)
 		if state.timestamp > 0 {
 			d := float64(state.timestamp-sv.prevTimestamp) / 1000

lib/streamaggr/streamaggr_synctest_test.go

@@ -789,6 +789,24 @@ foo:1m_by_cde_rate_sum{cde="1"} 0.125
   outputs: [rate_sum, rate_avg]
 `, "11111")
 
+	// test rate_sum with out of order samples
+	f([]string{`
+foo 1
+`, `
+foo 61
+`, `
+foo 31 -70
+foo 91
+`, `
+foo 121
+`}, time.Minute, `foo:1m_rate_sum 1
+foo:1m_rate_sum 0.5
+foo:1m_rate_sum 0.5
+`, `
+- interval: 1m
+  outputs: [rate_sum]
+`, "11111")
+
 	// test rate_sum and rate_avg with different staleness intervals
 	f([]string{`
 foo{abc="123", cde="1"} 1