wip

The commit automates several steps so they can be copy-pasted and
executed without any manual adjustments.

On success, it prints a message starting with “SUCCESS:” to make it
easier to distinguish from failures.

It also introduces the CURR_TAG and NEXT_TAG environment variables,
which are used by the automated commands.

.github/workflows/build.yml

@@ -47,6 +47,8 @@ jobs:
             arch: arm
           - os: linux
             arch: ppc64le
+          - os: linux
+            arch: s390x
           - os: darwin
             arch: amd64
           - os: darwin

Makefile

@@ -125,6 +125,15 @@ vmutils-linux-ppc64le: \
 	vmrestore-linux-ppc64le \
 	vmctl-linux-ppc64le
 
+vmutils-linux-s390x: \
+	vmagent-linux-s390x \
+	vmalert-linux-s390x \
+	vmalert-tool-linux-s390x \
+	vmauth-linux-s390x \
+	vmbackup-linux-s390x \
+	vmrestore-linux-s390x \
+	vmctl-linux-s390x
+
 vmutils-darwin-amd64: \
 	vmagent-darwin-amd64 \
 	vmalert-darwin-amd64 \
@@ -257,6 +266,7 @@ release-victoria-metrics: \
 	release-victoria-metrics-linux-amd64 \
 	release-victoria-metrics-linux-arm \
 	release-victoria-metrics-linux-arm64 \
+	release-victoria-metrics-linux-s390x \
 	release-victoria-metrics-darwin-amd64 \
 	release-victoria-metrics-darwin-arm64 \
 	release-victoria-metrics-freebsd-amd64 \
@@ -275,6 +285,9 @@ release-victoria-metrics-linux-arm:
 release-victoria-metrics-linux-arm64:
 	GOOS=linux GOARCH=arm64 $(MAKE) release-victoria-metrics-goos-goarch
 
+release-victoria-metrics-linux-s390x:
+	GOOS=linux GOARCH=s390x $(MAKE) release-victoria-metrics-goos-goarch
+
 release-victoria-metrics-darwin-amd64:
 	GOOS=darwin GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
@@ -314,6 +327,7 @@ release-vmutils: \
 	release-vmutils-linux-amd64 \
 	release-vmutils-linux-arm64 \
 	release-vmutils-linux-arm \
+	release-vmutils-linux-s390x \
 	release-vmutils-darwin-amd64 \
 	release-vmutils-darwin-arm64 \
 	release-vmutils-freebsd-amd64 \
@@ -332,6 +346,9 @@ release-vmutils-linux-arm64:
 release-vmutils-linux-arm:
 	GOOS=linux GOARCH=arm $(MAKE) release-vmutils-goos-goarch
 
+release-vmutils-linux-s390x:
+	GOOS=linux GOARCH=s390x $(MAKE) release-vmutils-goos-goarch
+
 release-vmutils-darwin-amd64:
 	GOOS=darwin GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 

app/victoria-metrics/Makefile

@@ -27,6 +27,9 @@ victoria-metrics-linux-ppc64le-prod:
 victoria-metrics-linux-386-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-386
 
+victoria-metrics-linux-s390x-prod:
+	APP_NAME=victoria-metrics $(MAKE) app-via-docker-linux-s390x
+
 victoria-metrics-darwin-amd64-prod:
 	APP_NAME=victoria-metrics $(MAKE) app-via-docker-darwin-amd64
 

app/vmagent/Makefile

@@ -27,6 +27,9 @@ vmagent-linux-ppc64le-prod:
 vmagent-linux-386-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-linux-386
 
+vmagent-linux-s390x-prod:
+	APP_NAME=vmagent $(MAKE) app-via-docker-linux-s390x
+
 vmagent-darwin-amd64-prod:
 	APP_NAME=vmagent $(MAKE) app-via-docker-darwin-amd64
 

app/vmagent/main.go

@@ -74,7 +74,7 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config and /remotewrite-.*-config pages. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
@@ -111,7 +111,6 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	flagutil.ApplySecretFlags()
 	remotewrite.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
@@ -253,6 +252,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"metric-relabel-debug", "debug metric relabeling"},
 			{"api/v1/targets", "advanced information about discovered targets in JSON format"},
 			{"config", "-promscrape.config contents"},
+			{"remotewrite-relabel-config", "-remoteWrite.relabelConfig contents"},
+			{"remotewrite-url-relabel-config", "-remoteWrite.urlRelabelConfig contents"},
 			{"metrics", "available service metrics"},
 			{"flags", "command-line flags"},
 			{"-/reload", "reload configuration"},
@@ -478,6 +479,42 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promscrape.WriteConfigData(&bb)
 		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
 		return true
+	case "/remotewrite-relabel-config":
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+			return true
+		}
+		remoteWriteRelabelConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		remotewrite.WriteRelabelConfigData(w)
+		return true
+	case "/api/v1/status/remotewrite-relabel-config":
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+			return true
+		}
+		remoteWriteStatusRelabelConfigRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		var bb bytesutil.ByteBuffer
+		remotewrite.WriteRelabelConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
+		return true
+	case "/remotewrite-url-relabel-config":
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+			return true
+		}
+		remoteWriteURLRelabelConfigRequests.Inc()
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		remotewrite.WriteURLRelabelConfigData(w)
+		return true
+	case "/api/v1/status/remotewrite-url-relabel-config":
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+			return true
+		}
+		remoteWriteStatusURLRelabelConfigRequests.Inc()
+		w.Header().Set("Content-Type", "application/json")
+		var bb bytesutil.ByteBuffer
+		remotewrite.WriteURLRelabelConfigData(&bb)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
+		return true
 	case "/prometheus/-/reload", "/-/reload":
 		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
@@ -748,6 +785,12 @@ var (
 	promscrapeConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/config"}`)
 	promscrapeStatusConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/config"}`)
 
+	remoteWriteRelabelConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/remotewrite-relabel-config"}`)
+	remoteWriteStatusRelabelConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/remotewrite-relabel-config"}`)
+
+	remoteWriteURLRelabelConfigRequests       = metrics.NewCounter(`vmagent_http_requests_total{path="/remotewrite-url-relabel-config"}`)
+	remoteWriteStatusURLRelabelConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/remotewrite-url-relabel-config"}`)
+
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
 )
 

app/vmagent/remotewrite/relabel.go

@@ -3,15 +3,18 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"io"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"go.yaml.in/yaml/v3"
 
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -32,9 +35,12 @@ var (
 		"See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels")
 )
 
-var labelsGlobal []prompb.Label
-
 var (
+	labelsGlobal []prompb.Label
+
+	remoteWriteRelabelConfigData    atomic.Pointer[[]byte]
+	remoteWriteURLRelabelConfigData atomic.Pointer[[]interface{}]
+
 	relabelConfigReloads      *metrics.Counter
 	relabelConfigReloadErrors *metrics.Counter
 	relabelConfigSuccess      *metrics.Gauge
@@ -67,6 +73,42 @@ func initRelabelConfigs() {
 	}
 }
 
+// WriteRelabelConfigData writes -remoteWrite.relabelConfig contents to w
+func WriteRelabelConfigData(w io.Writer) {
+	p := remoteWriteRelabelConfigData.Load()
+	if p == nil {
+		// Nothing to write to w
+		return
+	}
+	_, _ = w.Write(*p)
+}
+
+// WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
+func WriteURLRelabelConfigData(w io.Writer) {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		// Nothing to write to w
+		return
+	}
+	type urlRelabelCfg struct {
+		Url           string      `yaml:"url"`
+		RelabelConfig interface{} `yaml:"relabel_config"`
+	}
+	var cs []urlRelabelCfg
+	for i, url := range *remoteWriteURLs {
+		cfgData := (*p)[i]
+		if !*showRemoteWriteURL {
+			url = fmt.Sprintf("%d:secret-url", i+1)
+		}
+		cs = append(cs, urlRelabelCfg{
+			Url:           url,
+			RelabelConfig: cfgData,
+		})
+	}
+	d, _ := yaml.Marshal(cs)
+	_, _ = w.Write(d)
+}
+
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {
@@ -90,28 +132,42 @@ func reloadRelabelConfigs() {
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
-		global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
+		global, rawCfg, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err)
 		}
+		remoteWriteRelabelConfigData.Store(&rawCfg)
 		rcs.global = global
 	}
 	if len(*relabelConfigPaths) > len(*remoteWriteURLs) {
 		return nil, fmt.Errorf("too many -remoteWrite.urlRelabelConfig args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
 	}
+
+	var urlRelabelCfgs []interface{}
 	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
-			// Skip empty relabel config.
+			urlRelabelCfgs = append(urlRelabelCfgs, nil)
 			continue
 		}
-		prc, err := promrelabel.LoadRelabelConfigs(path)
+		prc, rawCfg, err := promrelabel.LoadRelabelConfigs(path)
 		if err != nil {
 			return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err)
 		}
 		rcs.perURL[i] = prc
+
+		var parsedCfg interface{}
+		_ = yaml.Unmarshal(rawCfg, &parsedCfg)
+		urlRelabelCfgs = append(urlRelabelCfgs, parsedCfg)
 	}
+	if len(*remoteWriteURLs) > len(*relabelConfigPaths) {
+		// fill the urlRelabelCfgs with empty relabel configs if not set
+		for i := len(*relabelConfigPaths); i < len(*remoteWriteURLs); i++ {
+			urlRelabelCfgs = append(urlRelabelCfgs, nil)
+		}
+	}
+	remoteWriteURLRelabelConfigData.Store(&urlRelabelCfgs)
 	return &rcs, nil
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
 	"github.com/VictoriaMetrics/metrics"
@@ -485,6 +486,9 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 			matchIdxs.B = sas.Push(tssBlock, matchIdxs.B)
 			if !*streamAggrGlobalKeepInput {
 				tssBlock = dropAggregatedSeries(tssBlock, matchIdxs.B, *streamAggrGlobalDropInput)
+			} else if *streamAggrGlobalDropInput {
+				// if both keep_input and drop_input are true, we keep only the aggregated series
+				tssBlock = dropUnaggregatedSeries(tssBlock, matchIdxs.B)
 			}
 			matchIdxsPool.Put(matchIdxs)
 		}
@@ -988,7 +992,17 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 				tss = append(*v, tss...)
 			}
 			tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
+		} else if rwctx.streamAggrDropInput {
+			// if both keep_input and drop_input are true, we keep only the aggregated series
+			if rctx == nil {
+				rctx = getRelabelCtx()
+				// Make a copy of tss before dropping aggregated series
+				v = tssPool.Get().(*[]prompb.TimeSeries)
+				tss = append(*v, tss...)
+			}
+			tss = dropUnaggregatedSeries(tss, matchIdxs.B)
 		}
+
 		matchIdxsPool.Put(matchIdxs)
 	}
 	if rwctx.deduplicator != nil {
@@ -1011,9 +1025,10 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
 	return false
 }
 
-var matchIdxsPool bytesutil.ByteBufferPool
+var matchIdxsPool slicesutil.BufferPool[uint32]
 
-func dropAggregatedSeries(src []prompb.TimeSeries, matchIdxs []byte, dropInput bool) []prompb.TimeSeries {
+// dropAggregatedSeries drops matched series, also the unmatched if dropInput is true.
+func dropAggregatedSeries(src []prompb.TimeSeries, matchIdxs []uint32, dropInput bool) []prompb.TimeSeries {
 	dst := src[:0]
 	if !dropInput {
 		for i, match := range matchIdxs {
@@ -1028,6 +1043,20 @@ func dropAggregatedSeries(src []prompb.TimeSeries, matchIdxs []byte, dropInput b
 	return dst
 }
 
+// dropUnaggregatedSeries drops unmatched series.
+func dropUnaggregatedSeries(src []prompb.TimeSeries, matchIdxs []uint32) []prompb.TimeSeries {
+	dst := src[:0]
+	for i, match := range matchIdxs {
+		if match == 0 {
+			continue
+		}
+		dst = append(dst, src[i])
+	}
+	tail := src[len(dst):]
+	clear(tail)
+	return dst
+}
+
 func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompb.TimeSeries) {
 	if rwctx.tryPushTimeSeriesInternal(tss) {
 		return

app/vmagent/remotewrite/remotewrite_test.go

@@ -10,6 +10,8 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -57,8 +59,8 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 	f(10)
 }
 
-func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
+func TestRemoteWriteContext_TryPushTimeSeries(t *testing.T) {
+	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string, expectedRowsPushedAfterRelabel, expectedPushedSample int) {
 		t.Helper()
 		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
 		if err != nil {
@@ -71,10 +73,16 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		}
 		allRelabelConfigs.Store(rcs)
 
+		path := "fast-queue-write-test"
+		fs.MustRemoveDir(path)
+		fq := persistentqueue.MustOpenFastQueue(path, "test", 100, 0, false)
+		defer fs.MustRemoveDir(path)
+		defer fq.MustClose()
+
 		pss := make([]*pendingSeries, 1)
 		isVMProto := &atomic.Bool{}
 		isVMProto.Store(true)
-		pss[0] = newPendingSeries(nil, isVMProto, 0, 100)
+		pss[0] = newPendingSeries(fq, isVMProto, 0, 100)
 		rwctx := &remoteWriteCtx{
 			idx:                    0,
 			streamAggrKeepInput:    keepInput,
@@ -83,6 +91,8 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsPushedAfterRelabel: metrics.GetOrCreateCounter(`foo`),
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
+		defer metrics.UnregisterAllMetrics()
+
 		if dedupInterval > 0 {
 			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, enableWindows, dedupInterval, nil, "dedup-global")
 		}
@@ -104,23 +114,27 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		inputTss := prometheus.MustParsePromMetrics(input, offsetMsecs)
 		expectedTss := make([]prompb.TimeSeries, len(inputTss))
 
-		// copy inputTss to make sure it is not mutated during TryPush call
+		// check inputTss is not modified after TryPushTimeSeries
 		copy(expectedTss, inputTss)
 		if !rwctx.TryPushTimeSeries(inputTss, false) {
 			t.Fatalf("cannot push samples to rwctx")
 		}
 
+		if int(rwctx.rowsPushedAfterRelabel.Get()) != expectedRowsPushedAfterRelabel {
+			t.Fatalf("unexpected number of rows after relabel; got %d; want %d", rwctx.rowsPushedAfterRelabel.Get(), expectedRowsPushedAfterRelabel)
+		}
+
+		if len(pss[0].wr.tss) != expectedPushedSample {
+			t.Fatalf("unexpected number of pushed samples; got %d; want %d", len(pss[0].wr.tss), expectedPushedSample)
+		}
+
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)
 		}
 	}
 
-	f(`
-- interval: 1m
-  outputs: [sum_samples]
-- interval: 2m
-  outputs: [count_series]
-`, `
+	// relabeling
+	f(``, `
 - action: keep
   source_labels: [env]
   regex: "dev"
@@ -129,53 +143,66 @@ metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
 metric{env="bar"} 25
-`)
+`, 2, 2)
+
+	// relabeling + aggregation
+	f(`
+- match: '{env="dev"}'
+  interval: 1m
+  outputs: [sum_samples]
+`, `
+- action: keep
+  source_labels: [env]
+  regex: ".*"
+`, false, 0, false, false, `
+metric{env="dev"} 10
+metric{env="bar"} 20
+metric{env="dev"} 15
+metric{env="bar"} 25
+`, 4, 2)
+
+	// aggregation + keepInput
+	f(`
+- match: '{env="dev"}'
+  interval: 1m
+  outputs: [sum_samples]
+`, ``, false, 0, true, false, `
+metric{env="dev"} 10
+metric{env="bar"} 20
+metric{env="dev"} 15
+metric{env="bar"} 25
+`, 4, 4)
+
+	// aggregation + dropInput
+	f(`
+- match: '{env="dev"}'
+  interval: 1m
+  outputs: [sum_samples]
+`, ``, false, 0, false, true, `
+metric{env="dev"} 10
+metric{env="bar"} 20
+metric{env="dev"} 15
+metric{env="bar"} 25
+`, 4, 0)
+
+	// aggregation + keepInput + dropInput
+	f(`
+- match: '{env="dev"}'
+  interval: 1m
+  outputs: [sum_samples]
+`, ``, false, 0, true, true, `
+metric{env="dev"} 10
+metric{env="bar"} 20
+metric{env="bar"} 25
+`, 3, 1)
+
+	// aggregation + deduplication
 	f(``, ``, true, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="foo"} 20
 metric{env="dev"} 15
 metric{env="foo"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, false, false, `
-metric{env="dev"} 10
-metric{env="bar"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, true, false, `
-metric{env="test"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, false, true, `
-metric{env="foo"} 10
-metric{env="dev"} 20
-metric{env="foo"} 15
-metric{env="dev"} 25
-`)
-	f(``, `
-- action: keep
-  source_labels: [env]
-  regex: "dev"
-`, true, time.Hour, true, true, `
-metric{env="dev"} 10
-metric{env="test"} 20
-metric{env="dev"} 15
-metric{env="bar"} 25
-`)
+`, 4, 0)
 }
 
 func TestShardAmountRemoteWriteCtx(t *testing.T) {

app/vmagent/remotewrite/streamaggr.go

@@ -18,12 +18,12 @@ var (
 	streamAggrGlobalConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
 		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
-	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation "+
-		"with -streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
-		"are written to remote storages write. See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+	streamAggrGlobalKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep input samples that match any rule in "+
+		"-streamAggr.config. By default, matched raw samples are aggregated and dropped, while unmatched samples "+
+		"are written to the remote storage. See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
+	streamAggrGlobalDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop input samples that not matching any rule in "+
+		"-streamAggr.config. By default, only matched raw samples are dropped, while unmatched samples "+
+		"are written to the remote storage. See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrGlobalDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval on "+
 		"aggregator before optional aggregation with -streamAggr.config . "+
 		"See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
@@ -43,11 +43,11 @@ var (
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
 		"See also -remoteWrite.streamAggr.keepInput, -remoteWrite.streamAggr.dropInput and -remoteWrite.streamAggr.dedupInterval")
-	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
+	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop input samples that not matching any rule in "+
+		"the corresponding -remoteWrite.streamAggr.config. By default, only matched raw samples are dropped, while unmatched samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep all the input samples after the aggregation "+
-		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. By default, only aggregates samples are dropped, while the remaining samples "+
+	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep input samples that match any rule in "+
+		"the corresponding -remoteWrite.streamAggr.config. By default, matched raw samples are aggregated and dropped, while unmatched samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation "+
 		"with -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. See also -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")

app/vmalert-tool/Makefile

@@ -27,6 +27,9 @@ vmalert-tool-linux-ppc64le-prod:
 vmalert-tool-linux-386-prod:
 	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-386
 
+vmalert-tool-linux-s390x-prod:
+	APP_NAME=vmalert-tool $(MAKE) app-via-docker-linux-s390x
+
 vmalert-tool-darwin-amd64-prod:
 	APP_NAME=vmalert-tool $(MAKE) app-via-docker-darwin-amd64
 

app/vmalert-tool/unittest/unittest.go

@@ -132,7 +132,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 		}
 		labels[s[:n]] = s[n+1:]
 	}
-	_, err = notifier.Init(labels, externalURL)
+	err = notifier.Init(labels, externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init notifier: %v", err)
 	}
@@ -379,7 +379,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 			if len(g.Rules) == 0 {
 				continue
 			}
-			errs := g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, rw, ts)
+			errs := g.ExecOnce(context.Background(), rw, ts)
 			for err := range errs {
 				if err != nil {
 					checkErrs = append(checkErrs, fmt.Errorf("\nfailed to exec group: %q, time: %s, err: %w", g.Name,

app/vmalert/Makefile

@@ -27,6 +27,9 @@ vmalert-linux-ppc64le-prod:
 vmalert-linux-386-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-linux-386
 
+vmalert-linux-s390x-prod:
+	APP_NAME=vmalert $(MAKE) app-via-docker-linux-s390x
+
 vmalert-darwin-amd64-prod:
 	APP_NAME=vmalert $(MAKE) app-via-docker-darwin-amd64
 

app/vmalert/datasource/client.go

@@ -179,11 +179,11 @@ func (c *Client) Query(ctx context.Context, query string, ts time.Time) (Result,
 	var parseFn func(resp *http.Response) (Result, error)
 	switch c.dataSourceType {
 	case datasourcePrometheus:
-		parseFn = parsePrometheusResponse
+		parseFn = parsePrometheusInstantResponse
 	case datasourceGraphite:
 		parseFn = parseGraphiteResponse
 	case datasourceVLogs:
-		parseFn = parseVLogsResponse
+		parseFn = parseVLogsInstantResponse
 	default:
 		logger.Panicf("BUG: unsupported datasource type %q to parse query response", c.dataSourceType)
 	}
@@ -239,9 +239,9 @@ func (c *Client) QueryRange(ctx context.Context, query string, start, end time.T
 	var parseFn func(resp *http.Response) (Result, error)
 	switch c.dataSourceType {
 	case datasourcePrometheus:
-		parseFn = parsePrometheusResponse
+		parseFn = parsePrometheusRangeResponse
 	case datasourceVLogs:
-		parseFn = parseVLogsResponse
+		parseFn = parseVLogsRangeResponse
 	default:
 		logger.Panicf("BUG: unsupported datasource type %q to parse query range response", c.dataSourceType)
 	}

app/vmalert/datasource/client_prom.go

@@ -172,17 +172,26 @@ const (
 	rtVector, rtMatrix, rScalar = "vector", "matrix", "scalar"
 )
 
-func parsePrometheusResponse(resp *http.Response) (res Result, err error) {
+func parsePromResponse(resp *http.Response) (*promResponse, error) {
 	r := &promResponse{}
-	if err = json.NewDecoder(resp.Body).Decode(r); err != nil {
-		return res, fmt.Errorf("failed to decode response: %w", err)
+	if err := json.NewDecoder(resp.Body).Decode(r); err != nil {
+		return nil, fmt.Errorf("failed to decode response: %w", err)
 	}
 	if r.Status == statusError {
-		return res, fmt.Errorf("response error %q: %s", r.ErrorType, r.Error)
+		return nil, fmt.Errorf("response error %q: %s", r.ErrorType, r.Error)
 	}
 	if r.Status != statusSuccess {
-		return res, fmt.Errorf("unknown response status %q", r.Status)
+		return nil, fmt.Errorf("unknown response status %q", r.Status)
 	}
+	return r, nil
+}
+
+func parsePrometheusInstantResponse(resp *http.Response) (res Result, err error) {
+	r, err := parsePromResponse(resp)
+	if err != nil {
+		return res, fmt.Errorf("failed to parse response: %w", err)
+	}
+
 	var parseFn func() ([]Metric, error)
 	switch r.Data.ResultType {
 	case rtVector:
@@ -191,12 +200,6 @@ func parsePrometheusResponse(resp *http.Response) (res Result, err error) {
 			return res, fmt.Errorf("unmarshal err %w; \n %#v", err, string(r.Data.Result))
 		}
 		parseFn = pi.metrics
-	case rtMatrix:
-		var pr promRange
-		if err := json.Unmarshal(r.Data.Result, &pr.Result); err != nil {
-			return res, err
-		}
-		parseFn = pr.metrics
 	case rScalar:
 		var ps promScalar
 		if err := json.Unmarshal(r.Data.Result, &ps); err != nil {
@@ -206,7 +209,6 @@ func parsePrometheusResponse(resp *http.Response) (res Result, err error) {
 	default:
 		return res, fmt.Errorf("unknown result type %q", r.Data.ResultType)
 	}
-
 	ms, err := parseFn()
 	if err != nil {
 		return res, err
@@ -222,6 +224,34 @@ func parsePrometheusResponse(resp *http.Response) (res Result, err error) {
 	return res, nil
 }
 
+func parsePrometheusRangeResponse(resp *http.Response) (res Result, err error) {
+	r, err := parsePromResponse(resp)
+	if err != nil {
+		return res, fmt.Errorf("failed to parse response: %w", err)
+	}
+	if r.Data.ResultType != rtMatrix {
+		return res, fmt.Errorf("unexpected result type %q; expected result type %q", r.Data.ResultType, rtMatrix)
+	}
+
+	var pr promRange
+	if err := json.Unmarshal(r.Data.Result, &pr.Result); err != nil {
+		return res, err
+	}
+	ms, err := pr.metrics()
+	if err != nil {
+		return res, err
+	}
+	res = Result{Data: ms, IsPartial: r.IsPartial}
+	if r.Stats.SeriesFetched != nil {
+		intV, err := strconv.Atoi(*r.Stats.SeriesFetched)
+		if err != nil {
+			return res, fmt.Errorf("failed to convert stats.seriesFetched to int: %w", err)
+		}
+		res.SeriesFetched = &intV
+	}
+	return res, nil
+}
+
 func (c *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	if c.appendTypePrefix {
 		r.URL.Path += "/prometheus"

app/vmalert/datasource/client_test.go

@@ -65,21 +65,23 @@ func TestVMInstantQuery(t *testing.T) {
 		case 3:
 			w.Write([]byte(`{"status":"unknown"}`))
 		case 4:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix"}}`))
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector"}}`))
 		case 5:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"vm_rows"},"values":[[1583786142,"13763"]]}]}}`))
 		case 6:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_rows","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
 		case 7:
-			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]},"stats":{"seriesFetched": "42"}}`))
+			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		case 8:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"scalar","result":[1583786142, "1"]},"stats":{"seriesFetched": "42"}}`))
+		case 9:
 			w.Write([]byte(`{"status":"success", "isPartial":true, "data":{"resultType":"scalar","result":[1583786142, "1"]}}`))
 		}
 	})
 	mux.HandleFunc("/render", func(w http.ResponseWriter, _ *http.Request) {
 		c++
 		switch c {
-		case 9:
+		case 10:
 			w.Write([]byte(`[{"target":"constantLine(10)","tags":{"name":"constantLine(10)"},"datapoints":[[10,1611758343],[10,1611758373],[10,1611758403]]}]`))
 		}
 	})
@@ -102,9 +104,9 @@ func TestVMInstantQuery(t *testing.T) {
 			t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
-		case 10:
-			w.Write([]byte("[]"))
 		case 11:
+			w.Write([]byte("[]"))
+		case 12:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"total","foo":"bar"},"value":[1583786142,"13763"]},{"metric":{"__name__":"total","foo":"baz"},"value":[1583786140,"2000"]}]}}`))
 		}
 	})
@@ -123,6 +125,7 @@ func TestVMInstantQuery(t *testing.T) {
 	ts := time.Now()
 
 	expErr := func(query, err string) {
+		t.Helper()
 		_, _, gotErr := pq.Query(ctx, query, ts)
 		if gotErr == nil {
 			t.Fatalf("expected %q got nil", err)
@@ -137,8 +140,9 @@ func TestVMInstantQuery(t *testing.T) {
 	expErr(vmQuery, "response error")               // 2
 	expErr(vmQuery, "unknown response status")      // 3
 	expErr(vmQuery, "unexpected end of JSON input") // 4
+	expErr(vmQuery, "unknown result type")          // 5
 
-	res, _, err := pq.Query(ctx, vmQuery, ts) // 5 - vector
+	res, _, err := pq.Query(ctx, vmQuery, ts) // 6 - vector
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -159,7 +163,7 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	metricsEqual(t, res.Data, expected)
 
-	res, req, err := pq.Query(ctx, vmQuery, ts) // 6 - scalar
+	res, req, err := pq.Query(ctx, vmQuery, ts) // 7 - scalar
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -184,7 +188,7 @@ func TestVMInstantQuery(t *testing.T) {
 			res.SeriesFetched)
 	}
 
-	res, _, err = pq.Query(ctx, vmQuery, ts) // 7 - scalar with stats
+	res, _, err = pq.Query(ctx, vmQuery, ts) // 8 - scalar with stats
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -205,7 +209,7 @@ func TestVMInstantQuery(t *testing.T) {
 			*res.SeriesFetched)
 	}
 
-	res, _, err = pq.Query(ctx, vmQuery, ts) // 8
+	res, _, err = pq.Query(ctx, vmQuery, ts) // 9
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -216,7 +220,7 @@ func TestVMInstantQuery(t *testing.T) {
 	// test graphite
 	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
 
-	res, _, err = gq.Query(ctx, queryRender, ts) // 9 - graphite
+	res, _, err = gq.Query(ctx, queryRender, ts) // 10 - graphite
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -236,9 +240,9 @@ func TestVMInstantQuery(t *testing.T) {
 	vlogs := datasourceVLogs
 	pq = s.BuildWithParams(QuerierParams{DataSourceType: string(vlogs), EvaluationInterval: 15 * time.Second})
 
-	expErr(vlogsQuery, "error parsing response") // 10
+	expErr(vlogsQuery, "error parsing response") // 11
 
-	res, _, err = pq.Query(ctx, vlogsQuery, ts) // 11
+	res, _, err = pq.Query(ctx, vlogsQuery, ts) // 12
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
@@ -390,6 +394,8 @@ func TestVMRangeQuery(t *testing.T) {
 		switch c {
 		case 0:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"vm_rows"},"values":[[1583786142,"13763"]]}]}}`))
+		case 1:
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[1583786142, "1"]}}`))
 		}
 	})
 	mux.HandleFunc("/select/logsql/stats_query_range", func(w http.ResponseWriter, r *http.Request) {
@@ -422,7 +428,7 @@ func TestVMRangeQuery(t *testing.T) {
 			t.Fatalf("expected 'step' query param to be 60s; got %q instead", step)
 		}
 		switch c {
-		case 1:
+		case 2:
 			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"__name__":"total"},"values":[[1583786142,"10"]]}]}}`))
 		}
 	})
@@ -446,13 +452,13 @@ func TestVMRangeQuery(t *testing.T) {
 
 	start, end := time.Now().Add(-time.Minute), time.Now()
 
-	res, err := pq.QueryRange(ctx, vmQuery, start, end)
+	res, err := pq.QueryRange(ctx, vmQuery, start, end) // case 0
 	if err != nil {
 		t.Fatalf("unexpected %s", err)
 	}
 	m := res.Data
 	if len(m) != 1 {
-		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
+		t.Fatalf("expected 1 metric got %d in %+v", len(m), m)
 	}
 	expected := Metric{
 		Labels:     []prompb.Label{{Value: "vm_rows", Name: "__name__"}},
@@ -463,6 +469,9 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("unexpected metric %+v want %+v", m[0], expected)
 	}
 
+	_, err = pq.QueryRange(ctx, vmQuery, start, end) // case 1
+	expectError(t, err, "unexpected result type")
+
 	// test unsupported graphite
 	gq := s.BuildWithParams(QuerierParams{DataSourceType: string(datasourceGraphite)})
 

app/vmalert/datasource/client_vlogs.go

@@ -40,8 +40,28 @@ func (c *Client) setVLogsRangeReqParams(r *http.Request, query string, start, en
 	c.setReqParams(r, query)
 }
 
-func parseVLogsResponse(resp *http.Response) (res Result, err error) {
-	res, err = parsePrometheusResponse(resp)
+func parseVLogsInstantResponse(resp *http.Response) (res Result, err error) {
+	res, err = parsePrometheusInstantResponse(resp)
+	if err != nil {
+		return Result{}, err
+	}
+	for i := range res.Data {
+		m := &res.Data[i]
+		for j := range m.Labels {
+			// reserve the stats func result name with a new label `stats_result` instead of dropping it,
+			// since there could be multiple stats results in a single query, for instance:
+			// 	_time:5m | stats quantile(0.5, request_duration_seconds) p50, quantile(0.9, request_duration_seconds) p90
+			if m.Labels[j].Name == "__name__" {
+				m.Labels[j].Name = "stats_result"
+				break
+			}
+		}
+	}
+	return
+}
+
+func parseVLogsRangeResponse(resp *http.Response) (res Result, err error) {
+	res, err = parsePrometheusRangeResponse(resp)
 	if err != nil {
 		return Result{}, err
 	}

app/vmalert/main.go

@@ -76,7 +76,7 @@ absolute path to all .tpl files in root.
 		`Link to VMUI: -external.alert.source='vmui/#/?g0.expr={{.Expr|queryEscape}}'. `+
 		`If empty 'vmalert/alert?group_id={{.GroupID}}&alert_id={{.AlertID}}' is used.`)
 	externalLabels = flagutil.NewArrayString("external.label", "Optional label in the form 'Name=value' to add to all generated recording rules and alerts. "+
-		"In case of conflicts, original labels are kept with prefix `exported_`.")
+		"In case of conflicts, original labels are kept with prefix 'exported_'.")
 
 	dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmalert. The rules file are validated. The -rule flag must be specified.")
 )
@@ -90,7 +90,6 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	flagutil.ApplySecretFlags()
 	remoteread.InitSecretFlags()
 	remotewrite.InitSecretFlags()
 	datasource.InitSecretFlags()
@@ -227,14 +226,13 @@ func newManager(ctx context.Context) (*manager, error) {
 		labels[s[:n]] = s[n+1:]
 	}
 
-	nts, err := notifier.Init(labels, *externalURL)
+	err = notifier.Init(labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
 	manager := &manager{
 		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: q,
-		notifiers:      nts,
 		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)

app/vmalert/main_test.go

@@ -96,9 +96,10 @@ groups:
 		querierBuilder: &datasource.FakeQuerier{},
 		groups:         make(map[uint64]*rule.Group),
 		labels:         map[string]string{},
-		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 		rw:             &remotewrite.Client{},
 	}
+	_, cleanup := notifier.InitFakeNotifier()
+	defer cleanup()
 
 	syncCh := make(chan struct{})
 	sighupCh := procutil.NewSighupChan()

app/vmalert/manager.go

@@ -16,7 +16,6 @@ import (
 // manager controls group states
 type manager struct {
 	querierBuilder datasource.QuerierBuilder
-	notifiers      func() []notifier.Notifier
 
 	rw remotewrite.RWClient
 	// remote read builder.
@@ -94,17 +93,16 @@ func (m *manager) close() {
 }
 
 func (m *manager) startGroup(ctx context.Context, g *rule.Group, restore bool) error {
-	m.wg.Add(1)
 	id := g.GetID()
 	g.Init()
-	go func() {
-		defer m.wg.Done()
+	m.wg.Go(func() {
 		if restore {
-			g.Start(ctx, m.notifiers, m.rw, m.rr)
+			g.Start(ctx, m.rw, m.rr)
 		} else {
-			g.Start(ctx, m.notifiers, m.rw, nil)
+			g.Start(ctx, m.rw, nil)
 		}
-	}()
+	})
+
 	m.groups[id] = g
 	return nil
 }
@@ -131,7 +129,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	if rrPresent && m.rw == nil {
 		return fmt.Errorf("config contains recording rules but `-remoteWrite.url` isn't set")
 	}
-	if arPresent && m.notifiers == nil {
+	if arPresent && notifier.GetTargets() == nil {
 		return fmt.Errorf("config contains alerting rules but neither `-notifier.url` nor `-notifier.config` nor `-notifier.blackhole` aren't set")
 	}
 
@@ -168,15 +166,15 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	if len(toUpdate) > 0 {
 		var wg sync.WaitGroup
 		for _, item := range toUpdate {
-			wg.Add(1)
-			// cancel evaluation so the Update will be applied as fast as possible.
-			// it is important to call InterruptEval before the update, because cancel fn
-			// can be re-assigned during the update.
-			item.old.InterruptEval()
-			go func(oldGroup *rule.Group, newGroup *rule.Group) {
-				oldGroup.UpdateWith(newGroup)
-				wg.Done()
-			}(item.old, item.new)
+			oldG := item.old
+			newG := item.new
+			wg.Go(func() {
+				// cancel evaluation so the Update will be applied as fast as possible.
+				// it is important to call InterruptEval before the update, because cancel fn
+				// can be re-assigned during the update.
+				oldG.InterruptEval()
+				oldG.UpdateWith(newG)
+			})
 		}
 		wg.Wait()
 	}

app/vmalert/manager_test.go

@@ -40,10 +40,11 @@ func TestManagerEmptyRulesDir(t *testing.T) {
 // execution of configuration update.
 // Should be executed with -race flag
 func TestManagerUpdateConcurrent(t *testing.T) {
+	_, cleanup := notifier.InitFakeNotifier()
+	defer cleanup()
 	m := &manager{
 		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: &datasource.FakeQuerier{},
-		notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 	}
 	paths := []string{
 		"config/testdata/dir/rules0-good.rules",
@@ -127,8 +128,9 @@ func TestManagerUpdate_Success(t *testing.T) {
 		m := &manager{
 			groups:         make(map[uint64]*rule.Group),
 			querierBuilder: &datasource.FakeQuerier{},
-			notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
 		}
+		_, cleanup := notifier.InitFakeNotifier()
+		defer cleanup()
 
 		cfgInit := loadCfg(t, []string{initPath}, true, true)
 		if err := m.update(ctx, cfgInit, false); err != nil {
@@ -277,7 +279,8 @@ func TestManagerUpdate_Failure(t *testing.T) {
 			rw:             rw,
 		}
 		if notifiers != nil {
-			m.notifiers = func() []notifier.Notifier { return notifiers }
+			_, cleanup := notifier.InitFakeNotifier()
+			defer cleanup()
 		}
 		err := m.update(context.Background(), []config.Group{cfg}, false)
 		if err == nil {

app/vmalert/notifier/alert_test.go

@@ -20,7 +20,7 @@ func TestAlertExecTemplate(t *testing.T) {
 	)
 	extLabels["cluster"] = extCluster
 	extLabels["dc"] = extDC
-	_, err := Init(extLabels, extURL)
+	err := Init(extLabels, extURL)
 	checkErr(t, err)
 
 	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {

app/vmalert/notifier/alertmanager.go

@@ -77,10 +77,13 @@ func (am *AlertManager) LastError() string {
 }
 
 // Send an alert or resolve message
-func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
+func (am *AlertManager) Send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, headers map[string]string) error {
+	if len(alerts) != len(alertLabels) {
+		return fmt.Errorf("mismatched number of alerts and label sets after global alert relabeling")
+	}
 	am.metrics.alertsSent.Add(len(alerts))
 	startTime := time.Now()
-	err := am.send(ctx, alerts, headers)
+	err := am.send(ctx, alerts, alertLabels, headers)
 	am.metrics.alertsSendDuration.UpdateDuration(startTime)
 	if err != nil {
 		am.metrics.alertsSendErrors.Add(len(alerts))
@@ -91,12 +94,15 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[st
 	return err
 }
 
-func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error {
+func (am *AlertManager) send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, headers map[string]string) error {
 	b := &bytes.Buffer{}
 	alertsToSend := make([]Alert, 0, len(alerts))
 	lblss := make([][]prompb.Label, 0, len(alerts))
-	for _, a := range alerts {
-		lbls := a.applyRelabelingIfNeeded(am.relabelConfigs)
+	for i, a := range alerts {
+		lbls := alertLabels[i]
+		if am.relabelConfigs != nil {
+			lbls = am.relabelConfigs.Apply(lbls, 0)
+		}
 		if len(lbls) == 0 {
 			continue
 		}

app/vmalert/notifier/alertmanager_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
@@ -145,11 +146,11 @@ func TestAlertManager_Send(t *testing.T) {
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
+	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, [][]prompb.Label{{{Name: "a", Value: "b"}}}, nil); err == nil {
 		t.Fatalf("expected connection error got nil")
 	}
 
-	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
+	if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, [][]prompb.Label{{{Name: "a", Value: "b"}}}, nil); err == nil {
 		t.Fatalf("expected wrong http code error got nil")
 	}
 
@@ -160,7 +161,7 @@ func TestAlertManager_Send(t *testing.T) {
 		End:         time.Now().UTC(),
 		Labels:      map[string]string{"alertname": "alert0"},
 		Annotations: map[string]string{"a": "b", "c": "d"},
-	}}, map[string]string{headerKey: "bar"}); err != nil {
+	}}, [][]prompb.Label{{{Name: "alertname", Value: "alert0"}}}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
@@ -174,7 +175,7 @@ func TestAlertManager_Send(t *testing.T) {
 			Name:   "alert2",
 			Labels: map[string]string{"rule": "test", "tenant": "1"},
 		},
-	}, map[string]string{headerKey: "bar"}); err != nil {
+	}, [][]prompb.Label{{{Name: "rule", Value: "test"}, {Name: "tenant", Value: "0"}}, {{Name: "rule", Value: "test"}, {Name: "tenant", Value: "1"}}}, map[string]string{headerKey: "bar"}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
@@ -187,7 +188,7 @@ func TestAlertManager_Send(t *testing.T) {
 			Name:   "alert2",
 			Labels: map[string]string{},
 		},
-	}, map[string]string{}); err != nil {
+	}, [][]prompb.Label{{{Name: "rule", Value: "test"}}, {{}}}, map[string]string{}); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 

app/vmalert/notifier/config.go

@@ -27,15 +27,9 @@ type Config struct {
 	// PathPrefix is added to URL path before adding alertManagerPath value
 	PathPrefix string `yaml:"path_prefix,omitempty"`
 
-	// ConsulSDConfigs contains list of settings for service discovery via Consul
-	// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
-	ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
-	// DNSSDConfigs contains list of settings for service discovery via DNS.
-	// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
-	DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
-
-	// StaticConfigs contains list of static targets
-	StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"`
+	ConsulSDConfigs []ConsulSDConfigs `yaml:"consul_sd_configs,omitempty"`
+	DNSSDConfigs    []DNSSDConfigs    `yaml:"dns_sd_configs,omitempty"`
+	StaticConfigs   []StaticConfig    `yaml:"static_configs,omitempty"`
 
 	// HTTPClientConfig contains HTTP configuration for Notifier clients
 	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
@@ -62,14 +56,29 @@ type Config struct {
 	parsedAlertRelabelConfigs *promrelabel.ParsedConfigs
 }
 
-// StaticConfig contains list of static targets in the following form:
+// staticConfig contains list of static targets in the following form:
 //
 //	targets:
 //	[ - '<host>' ]
 type StaticConfig struct {
 	Targets []string `yaml:"targets"`
 	// HTTPClientConfig contains HTTP configuration for the Targets
-	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
+	HTTPClientConfig    promauth.HTTPClientConfig   `yaml:",inline"`
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
+}
+
+// ConsulSDConfigs contains list of settings for service discovery via Consul,
+// see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config
+type ConsulSDConfigs struct {
+	consul.SDConfig     `yaml:",inline"`
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
+}
+
+// DNSSDConfigs contains list of settings for service discovery via DNS,
+// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config
+type DNSSDConfigs struct {
+	dns.SDConfig        `yaml:",inline"`
+	AlertRelabelConfigs []promrelabel.RelabelConfig `yaml:"alert_relabel_configs,omitempty"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -95,6 +104,31 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(any) error) error {
 	}
 	cfg.parsedAlertRelabelConfigs = arCfg
 
+	for _, s := range cfg.StaticConfigs {
+		if len(s.AlertRelabelConfigs) > 0 {
+			_, err := promrelabel.ParseRelabelConfigs(s.AlertRelabelConfigs)
+			if err != nil {
+				return fmt.Errorf("failed to parse alert_relabel_configs in static_config: %w", err)
+			}
+		}
+	}
+	for _, s := range cfg.ConsulSDConfigs {
+		if len(s.AlertRelabelConfigs) > 0 {
+			_, err := promrelabel.ParseRelabelConfigs(s.AlertRelabelConfigs)
+			if err != nil {
+				return fmt.Errorf("failed to parse alert_relabel_configs in consul_sd_config: %w", err)
+			}
+		}
+	}
+	for _, s := range cfg.DNSSDConfigs {
+		if len(s.AlertRelabelConfigs) > 0 {
+			_, err := promrelabel.ParseRelabelConfigs(s.AlertRelabelConfigs)
+			if err != nil {
+				return fmt.Errorf("failed to parse alert_relabel_configs in dns_sd_config: %w", err)
+			}
+		}
+	}
+
 	b, err := yaml.Marshal(cfg)
 	if err != nil {
 		return fmt.Errorf("failed to marshal configuration for checksum: %w", err)

app/vmalert/notifier/config_test.go

@@ -35,4 +35,6 @@ func TestParseConfig_Failure(t *testing.T) {
 
 	f("testdata/unknownFields.bad.yaml", "unknown field")
 	f("non-existing-file", "error reading")
+	f("testdata/consul.bad.yaml", "failed to parse alert_relabel_configs in consul_sd_config")
+	f("testdata/dns.bad.yaml", "failed to parse alert relabeling config")
 }

app/vmalert/notifier/config_watcher.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
@@ -28,11 +29,7 @@ type configWatcher struct {
 	targets   map[TargetType][]Target
 }
 
-func newWatcher(path string, gen AlertURLGenerator) (*configWatcher, error) {
-	cfg, err := parseConfig(path)
-	if err != nil {
-		return nil, err
-	}
+func newWatcher(cfg *Config, gen AlertURLGenerator) (*configWatcher, error) {
 	cw := &configWatcher{
 		cfg:       cfg,
 		wg:        sync.WaitGroup{},
@@ -88,18 +85,15 @@ func (cw *configWatcher) reload(path string) error {
 	return cw.start()
 }
 
-func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn getLabels) error {
-	targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
+func (cw *configWatcher) add(typeK TargetType, interval time.Duration, targetsFn getTargets) error {
+	targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
 	for _, err := range errors {
 		return fmt.Errorf("failed to init notifier for %q: %w", typeK, err)
 	}
 
 	cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 
-	cw.wg.Add(1)
-	go func() {
-		defer cw.wg.Done()
-
+	cw.wg.Go(func() {
 		ticker := time.NewTicker(interval)
 		defer ticker.Stop()
 
@@ -109,62 +103,77 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, labelsFn
 				return
 			case <-ticker.C:
 			}
-			targetMetadata, errors := getTargetMetadata(labelsFn, cw.cfg)
+			targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
 			for _, err := range errors {
 				logger.Errorf("failed to init notifier for %q: %w", typeK, err)
 			}
 			cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 		}
-	}()
+	})
 	return nil
 }
 
-func getTargetMetadata(labelsFn getLabels, cfg *Config) (map[string]*promutil.Labels, []error) {
-	metaLabels, err := labelsFn()
+type targetMetadata struct {
+	*promutil.Labels
+	alertRelabelConfigs *promrelabel.ParsedConfigs
+}
+
+func getTargetMetadata(targetsFn getTargets, cfg *Config) (map[string]targetMetadata, []error) {
+	metaLabelsList, alertRelabelCfgs, err := targetsFn()
 	if err != nil {
 		return nil, []error{fmt.Errorf("failed to get labels: %w", err)}
 	}
-	targetMetadata := make(map[string]*promutil.Labels, len(metaLabels))
+	targetMts := make(map[string]targetMetadata, len(metaLabelsList))
 	var errors []error
 	duplicates := make(map[string]struct{})
-	for _, labels := range metaLabels {
-		target := labels.Get("__address__")
-		u, processedLabels, err := parseLabels(target, labels, cfg)
-		if err != nil {
-			errors = append(errors, err)
-			continue
-		}
-		if len(u) == 0 {
-			continue
-		}
-		if _, ok := duplicates[u]; ok { // check for duplicates
-			if !*suppressDuplicateTargetErrors {
-				logger.Errorf("skipping duplicate target with identical address %q; "+
-					"make sure service discovery and relabeling is set up properly; "+
-					"original labels: %s; resulting labels: %s",
-					u, labels, processedLabels)
+	for i := range metaLabelsList {
+		metaLabels := metaLabelsList[i]
+		alertRelabelCfg := alertRelabelCfgs[i]
+		for _, labels := range metaLabels {
+			target := labels.Get("__address__")
+			u, processedLabels, err := parseLabels(target, labels, cfg)
+			if err != nil {
+				errors = append(errors, err)
+				continue
+			}
+			if len(u) == 0 {
+				continue
+			}
+			// check for duplicated targets
+			// targets with same address but different alert_relabel_configs are still considered duplicates since it's mostly due to misconfiguration and could cause duplicated notifications.
+			if _, ok := duplicates[u]; ok {
+				if !*suppressDuplicateTargetErrors {
+					logger.Errorf("skipping duplicate target with identical address %q; "+
+						"make sure service discovery and relabeling is set up properly; "+
+						"original labels: %s; resulting labels: %s",
+						u, labels, processedLabels)
+				}
+				continue
+			}
+			duplicates[u] = struct{}{}
+			targetMts[u] = targetMetadata{
+				Labels:              processedLabels,
+				alertRelabelConfigs: alertRelabelCfg,
 			}
-			continue
 		}
-		duplicates[u] = struct{}{}
-		targetMetadata[u] = processedLabels
 	}
-	return targetMetadata, errors
+	return targetMts, errors
 }
 
-type getLabels func() ([]*promutil.Labels, error)
+type getTargets func() ([][]*promutil.Labels, []*promrelabel.ParsedConfigs, error)
 
 func (cw *configWatcher) start() error {
 	if len(cw.cfg.StaticConfigs) > 0 {
 		var targets []Target
-		for _, cfg := range cw.cfg.StaticConfigs {
+		for i, cfg := range cw.cfg.StaticConfigs {
+			alertRelabelConfig, _ := promrelabel.ParseRelabelConfigs(cw.cfg.StaticConfigs[i].AlertRelabelConfigs)
 			httpCfg := mergeHTTPClientConfigs(cw.cfg.HTTPClientConfig, cfg.HTTPClientConfig)
 			for _, target := range cfg.Targets {
 				address, labels, err := parseLabels(target, nil, cw.cfg)
 				if err != nil {
 					return fmt.Errorf("failed to parse labels for target %q: %w", target, err)
 				}
-				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, cw.cfg.parsedAlertRelabelConfigs, cw.cfg.Timeout.Duration())
+				notifier, err := NewAlertManager(address, cw.genFn, httpCfg, alertRelabelConfig, cw.cfg.Timeout.Duration())
 				if err != nil {
 					return fmt.Errorf("failed to init alertmanager for addr %q: %w", address, err)
 				}
@@ -178,17 +187,20 @@ func (cw *configWatcher) start() error {
 	}
 
 	if len(cw.cfg.ConsulSDConfigs) > 0 {
-		err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([]*promutil.Labels, error) {
-			var labels []*promutil.Labels
+		err := cw.add(TargetConsul, *consul.SDCheckInterval, func() ([][]*promutil.Labels, []*promrelabel.ParsedConfigs, error) {
+			var labels [][]*promutil.Labels
+			var alertRelabelConfigs []*promrelabel.ParsedConfigs
 			for i := range cw.cfg.ConsulSDConfigs {
+				alertRelabelConfig, _ := promrelabel.ParseRelabelConfigs(cw.cfg.ConsulSDConfigs[i].AlertRelabelConfigs)
 				sdc := &cw.cfg.ConsulSDConfigs[i]
 				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
 				if err != nil {
-					return nil, fmt.Errorf("got labels err: %w", err)
+					return nil, nil, fmt.Errorf("got labels err: %w", err)
 				}
-				labels = append(labels, targetLabels...)
+				labels = append(labels, targetLabels)
+				alertRelabelConfigs = append(alertRelabelConfigs, alertRelabelConfig)
 			}
-			return labels, nil
+			return labels, alertRelabelConfigs, nil
 		})
 		if err != nil {
 			return fmt.Errorf("failed to start consulSD discovery: %w", err)
@@ -196,17 +208,21 @@ func (cw *configWatcher) start() error {
 	}
 
 	if len(cw.cfg.DNSSDConfigs) > 0 {
-		err := cw.add(TargetDNS, *dns.SDCheckInterval, func() ([]*promutil.Labels, error) {
-			var labels []*promutil.Labels
+		err := cw.add(TargetDNS, *dns.SDCheckInterval, func() ([][]*promutil.Labels, []*promrelabel.ParsedConfigs, error) {
+			var labels [][]*promutil.Labels
+			var alertRelabelConfigs []*promrelabel.ParsedConfigs
 			for i := range cw.cfg.DNSSDConfigs {
+				alertRelabelConfig, _ := promrelabel.ParseRelabelConfigs(cw.cfg.DNSSDConfigs[i].AlertRelabelConfigs)
 				sdc := &cw.cfg.DNSSDConfigs[i]
 				targetLabels, err := sdc.GetLabels(cw.cfg.baseDir)
 				if err != nil {
-					return nil, fmt.Errorf("got labels err: %w", err)
+					return nil, nil, fmt.Errorf("got labels err: %w", err)
 				}
-				labels = append(labels, targetLabels...)
+				labels = append(labels, targetLabels)
+				alertRelabelConfigs = append(alertRelabelConfigs, alertRelabelConfig)
+
 			}
-			return labels, nil
+			return labels, alertRelabelConfigs, nil
 		})
 		if err != nil {
 			return fmt.Errorf("failed to start DNSSD discovery: %w", err)
@@ -240,30 +256,30 @@ func (cw *configWatcher) setTargets(key TargetType, targets []Target) {
 	cw.targetsMu.Unlock()
 }
 
-func (cw *configWatcher) updateTargets(key TargetType, targetMetadata map[string]*promutil.Labels, cfg *Config, genFn AlertURLGenerator) {
+func (cw *configWatcher) updateTargets(key TargetType, targetMts map[string]targetMetadata, cfg *Config, genFn AlertURLGenerator) {
 	cw.targetsMu.Lock()
 	defer cw.targetsMu.Unlock()
 	oldTargets := cw.targets[key]
 	var updatedTargets []Target
 	for _, ot := range oldTargets {
-		if _, ok := targetMetadata[ot.Addr()]; !ok {
+		if _, ok := targetMts[ot.Addr()]; !ok {
 			// if target not exists in currentTargets, close it
 			ot.Close()
 		} else {
 			updatedTargets = append(updatedTargets, ot)
-			delete(targetMetadata, ot.Addr())
+			delete(targetMts, ot.Addr())
 		}
 	}
 	// create new resources for the new targets
-	for addr, labels := range targetMetadata {
-		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, cfg.parsedAlertRelabelConfigs, cfg.Timeout.Duration())
+	for addr, metadata := range targetMts {
+		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, metadata.alertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
 			logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
 			continue
 		}
 		updatedTargets = append(updatedTargets, Target{
 			Notifier: am,
-			Labels:   labels,
+			Labels:   metadata.Labels,
 		})
 	}
 

app/vmalert/notifier/config_watcher_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -28,7 +29,11 @@ static_configs:
       - localhost:9093
       - localhost:9094
 `)
-	cw, err := newWatcher(f.Name(), nil)
+	cfg, err := parseConfig(f.Name())
+	if err != nil {
+		t.Fatalf("failed to parse config: %s", err)
+	}
+	cw, err := newWatcher(cfg, nil)
 	if err != nil {
 		t.Fatalf("failed to start config watcher: %s", err)
 	}
@@ -83,33 +88,64 @@ consul_sd_configs:
   - server: %s
     services:
       - alertmanager
-`, consulSDServer.URL))
+  - server: %s
+    services:
+      - alertmanager
+    alert_relabel_configs:
+    - target_label: "foo"
+      replacement: "tar"
+`, consulSDServer.URL, consulSDServer.URL))
 
-	cw, err := newWatcher(consulSDFile.Name(), nil)
+	cfg, err := parseConfig(consulSDFile.Name())
+	if err != nil {
+		t.Fatalf("failed to parse config: %s", err)
+	}
+	cw, err := newWatcher(cfg, nil)
 	if err != nil {
 		t.Fatalf("failed to start config watcher: %s", err)
 	}
 	defer cw.mustStop()
 
-	if len(cw.notifiers()) != 2 {
-		t.Fatalf("expected to get 2 notifiers; got %d", len(cw.notifiers()))
+	if len(cw.notifiers()) != 3 {
+		t.Fatalf("expected to get 3 notifiers; got %d", len(cw.notifiers()))
 	}
 
 	expAddr1 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService1)
 	expAddr2 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService2)
+	expAddr3 := fmt.Sprintf("https://%s/proxy/api/v2/alerts", fakeConsulService3)
 
-	n1, n2 := cw.notifiers()[0], cw.notifiers()[1]
+	n1, n2, n3 := cw.notifiers()[0], cw.notifiers()[1], cw.notifiers()[2]
 	if n1.Addr() != expAddr1 {
 		t.Fatalf("exp address %q; got %q", expAddr1, n1.Addr())
 	}
 	if n2.Addr() != expAddr2 {
 		t.Fatalf("exp address %q; got %q", expAddr2, n2.Addr())
 	}
+	if n3.Addr() != expAddr3 {
+		t.Fatalf("exp address %q; got %q", expAddr3, n3.Addr())
+	}
+
+	if n1.(*AlertManager).relabelConfigs.String() != "" {
+		t.Fatalf("unexpected relabel configs: %q", n1.(*AlertManager).relabelConfigs.String())
+	}
+	if n2.(*AlertManager).relabelConfigs.String() != "" {
+		t.Fatalf("unexpected relabel configs: %q", n2.(*AlertManager).relabelConfigs.String())
+	}
+	if n3.(*AlertManager).relabelConfigs.String() != "- target_label: foo\n  replacement: tar\n" {
+		t.Fatalf("unexpected relabel configs: %q", n3.(*AlertManager).relabelConfigs.String())
+	}
 
 	f := func() bool { return len(cw.notifiers()) == 1 }
 	if !waitFor(f, time.Second) {
 		t.Fatalf("expected to get 1 notifiers; got %d", len(cw.notifiers()))
 	}
+	n3 = cw.notifiers()[0]
+	if n3.Addr() != expAddr3 {
+		t.Fatalf("exp address %q; got %q", expAddr3, n3.Addr())
+	}
+	if n3.(*AlertManager).relabelConfigs.String() != "- target_label: foo\n  replacement: tar\n" {
+		t.Fatalf("unexpected relabel configs: %q", n3.(*AlertManager).relabelConfigs.String())
+	}
 }
 
 // TestConfigWatcherReloadConcurrent supposed to test concurrent
@@ -164,7 +200,11 @@ consul_sd_configs:
 		"unknownFields.bad.yaml",
 	}
 
-	cw, err := newWatcher(paths[0], nil)
+	cfg, err := parseConfig(paths[0])
+	if err != nil {
+		t.Fatalf("failed to parse config: %s", err)
+	}
+	cw, err := newWatcher(cfg, nil)
 	if err != nil {
 		t.Fatalf("failed to start config watcher: %s", err)
 	}
@@ -202,10 +242,11 @@ func checkErr(t *testing.T, err error) {
 const (
 	fakeConsulService1 = "127.0.0.1:9093"
 	fakeConsulService2 = "127.0.0.1:9095"
+	fakeConsulService3 = "127.0.0.1:9097"
 )
 
 func newFakeConsulServer() *httptest.Server {
-	requestCount := 0
+	var requestCount atomic.Int32
 	mux := http.NewServeMux()
 	mux.HandleFunc("/v1/agent/self", func(rw http.ResponseWriter, _ *http.Request) {
 		rw.Write([]byte(`{"Config": {"Datacenter": "dc1"}}`))
@@ -220,7 +261,7 @@ func newFakeConsulServer() *httptest.Server {
 }`))
 	})
 	mux.HandleFunc("/v1/health/service/alertmanager", func(rw http.ResponseWriter, _ *http.Request) {
-		if requestCount == 0 {
+		if requestCount.Load() == 0 {
 			rw.Header().Set("X-Consul-Index", "1")
 			rw.Write([]byte(`
 [
@@ -360,7 +401,7 @@ func newFakeConsulServer() *httptest.Server {
     }
 ]`))
 		}
-		requestCount++
+		requestCount.Add(1)
 	})
 
 	return httptest.NewServer(mux)

app/vmalert/notifier/faker.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"sync"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
 
 // FakeNotifier is a mock notifier
@@ -15,6 +17,19 @@ type FakeNotifier struct {
 	counter int
 }
 
+// InitFakeNotifier initializes global notifier to FakeNotifier,
+// and returns a cleanup function to restore the original getActiveNotifiers.
+func InitFakeNotifier() (*FakeNotifier, func()) {
+	originalGetActiveNotifiers := getActiveNotifiers
+	fn := &FakeNotifier{}
+	getActiveNotifiers = func() []Notifier {
+		return []Notifier{fn}
+	}
+	return fn, func() {
+		getActiveNotifiers = originalGetActiveNotifiers
+	}
+}
+
 // Close does nothing
 func (*FakeNotifier) Close() {}
 
@@ -27,7 +42,7 @@ func (*FakeNotifier) LastError() string {
 func (*FakeNotifier) Addr() string { return "" }
 
 // Send sets alerts and increases counter
-func (fn *FakeNotifier) Send(_ context.Context, alerts []Alert, _ map[string]string) error {
+func (fn *FakeNotifier) Send(_ context.Context, alerts []Alert, _ [][]prompb.Label, _ map[string]string) error {
 	fn.Lock()
 	defer fn.Unlock()
 	fn.counter += len(alerts)

app/vmalert/notifier/init.go

@@ -1,17 +1,22 @@
 package notifier
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 )
 
@@ -96,11 +101,25 @@ func InitAlertURLGeneratorFn(externalURL *url.URL, externalAlertSource string, v
 	return nil
 }
 
-// cw holds a configWatcher for configPath configuration file
-// configWatcher provides a list of Notifier objects discovered
-// from static config or via service discovery.
-// cw is not nil only if configPath is provided.
-var cw *configWatcher
+var (
+	// getActiveNotifiers returns the current list of Notifier objects.
+	getActiveNotifiers func() []Notifier
+	// globalRelabelCfg stores the parsed alert relabeling config from the config file if there is
+	globalRelabelCfg *promrelabel.ParsedConfigs
+
+	// cw holds a configWatcher for configPath configuration file
+	// configWatcher provides a list of Notifier objects discovered
+	// from static config or via service discovery.
+	// cw is not nil only if configPath is provided.
+	cw *configWatcher
+
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
 
 // Reload checks the changes in configPath configuration file
 // and applies changes if any.
@@ -111,66 +130,62 @@ func Reload() error {
 	return cw.reload(*configPath)
 }
 
-var staticNotifiersFn func() []Notifier
-
-var (
-	// externalLabels is a global variable for holding external labels configured via flags
-	// It is supposed to be inited via Init function only.
-	externalLabels map[string]string
-	// externalURL is a global variable for holding external URL value configured via flag
-	// It is supposed to be inited via Init function only.
-	externalURL string
-)
-
-// Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
 //   - configuration via flags (for backward compatibility). Is always static
 //     and don't support live reloads.
 //   - configuration via file. Supports live reloads and service discovery.
 //
 // Init returns an error if both mods are used.
-func Init(extLabels map[string]string, extURL string) (func() []Notifier, error) {
+func Init(extLabels map[string]string, extURL string) error {
 	externalURL = extURL
 	externalLabels = extLabels
 	_, err := url.Parse(externalURL)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse external URL: %w", err)
+		return fmt.Errorf("failed to parse external URL: %w", err)
 	}
 
 	if *blackHole {
 		if len(*addrs) > 0 || *configPath != "" {
-			return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
+			return fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
 		}
 		notifier := newBlackHoleNotifier()
-		staticNotifiersFn = func() []Notifier {
+		getActiveNotifiers = func() []Notifier {
 			return []Notifier{notifier}
 		}
-		return staticNotifiersFn, nil
+		return nil
 	}
 
 	if *configPath == "" && len(*addrs) == 0 {
-		return nil, nil
+		return nil
 	}
 	if *configPath != "" && len(*addrs) > 0 {
-		return nil, fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
+		return fmt.Errorf("only one of -notifier.config or -notifier.url flags must be specified")
 	}
 
 	if len(*addrs) > 0 {
 		notifiers, err := notifiersFromFlags(AlertURLGeneratorFn)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create notifier from flag values: %w", err)
+			return fmt.Errorf("failed to create notifier from flag values: %w", err)
 		}
-		staticNotifiersFn = func() []Notifier {
+		getActiveNotifiers = func() []Notifier {
 			return notifiers
 		}
-		return staticNotifiersFn, nil
+		return nil
 	}
 
-	cw, err = newWatcher(*configPath, AlertURLGeneratorFn)
+	cfg, err := parseConfig(*configPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to init config watcher: %w", err)
+		return err
 	}
-	return cw.notifiers, nil
+	if cfg.AlertRelabelConfigs != nil {
+		globalRelabelCfg = cfg.parsedAlertRelabelConfigs
+	}
+	cw, err = newWatcher(cfg, AlertURLGeneratorFn)
+	if err != nil {
+		return fmt.Errorf("failed to init config watcher: %w", err)
+	}
+	getActiveNotifiers = cw.notifiers
+	return nil
 }
 
 // InitSecretFlags must be called after flag.Parse and before any logging
@@ -245,23 +260,57 @@ const (
 
 // GetTargets returns list of static or discovered targets
 // via notifier configuration.
+//
+// Must be called after Init.
 func GetTargets() map[TargetType][]Target {
-	var targets = make(map[TargetType][]Target)
-
-	if staticNotifiersFn != nil {
-		for _, ns := range staticNotifiersFn() {
-			targets[TargetStatic] = append(targets[TargetStatic], Target{
-				Notifier: ns,
-			})
-		}
+	if getActiveNotifiers == nil {
+		return nil
 	}
-
+	var targets = make(map[TargetType][]Target)
+	// use cached targets from configWatcher instead of getActiveNotifiers for the extra target labels
 	if cw != nil {
 		cw.targetsMu.RLock()
 		for key, ns := range cw.targets {
 			targets[key] = append(targets[key], ns...)
 		}
 		cw.targetsMu.RUnlock()
+		return targets
+	}
+
+	// static notifiers don't have labels
+	for _, ns := range getActiveNotifiers() {
+		targets[TargetStatic] = append(targets[TargetStatic], Target{
+			Notifier: ns,
+		})
 	}
 	return targets
 }
+
+// Send sends alerts to all active notifiers
+func Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) *vmalertutil.ErrGroup {
+	alertsToSend := make([]Alert, 0, len(alerts))
+	lblss := make([][]prompb.Label, 0, len(alerts))
+	// apply global relabel config first without modifying original alerts in alerts
+	for _, a := range alerts {
+		lbls := a.applyRelabelingIfNeeded(globalRelabelCfg)
+		if len(lbls) == 0 {
+			continue
+		}
+		alertsToSend = append(alertsToSend, a)
+		lblss = append(lblss, lbls)
+	}
+
+	errGr := new(vmalertutil.ErrGroup)
+	wg := sync.WaitGroup{}
+	activeNotifiers := getActiveNotifiers()
+	for i := range activeNotifiers {
+		nt := activeNotifiers[i]
+		wg.Go(func() {
+			if err := nt.Send(ctx, alertsToSend, lblss, notifierHeaders); err != nil {
+				errGr.Add(fmt.Errorf("failed to send alerts to addr %q: %w", nt.Addr(), err))
+			}
+		})
+	}
+	wg.Wait()
+	return errGr
+}

app/vmalert/notifier/init_test.go

@@ -1,11 +1,17 @@
 package notifier
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"net/url"
+	"os"
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 )
 
 func TestInit(t *testing.T) {
@@ -14,14 +20,13 @@ func TestInit(t *testing.T) {
 
 	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
 
-	fn, err := Init(nil, "")
+	err := Init(nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
 
-	nfs := fn()
-	if len(nfs) != 2 {
-		t.Fatalf("expected to get 2 notifiers; got %d", len(nfs))
+	if len(getActiveNotifiers()) != 2 {
+		t.Fatalf("expected to get 2 notifiers; got %d", len(getActiveNotifiers()))
 	}
 
 	targets := GetTargets()
@@ -54,7 +59,7 @@ func TestInitNegative(t *testing.T) {
 		*configPath = path
 		*addrs = flagutil.ArrayString{addr}
 		*blackHole = bh
-		if _, err := Init(nil, ""); err == nil {
+		if err := Init(nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
 		}
 	}
@@ -71,14 +76,13 @@ func TestBlackHole(t *testing.T) {
 
 	*blackHole = true
 
-	fn, err := Init(nil, "")
+	err := Init(nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
 
-	nfs := fn()
-	if len(nfs) != 1 {
-		t.Fatalf("expected to get 1 notifier; got %d", len(nfs))
+	if len(getActiveNotifiers()) != 1 {
+		t.Fatalf("expected to get 1 notifier; got %d", len(getActiveNotifiers()))
 	}
 
 	targets := GetTargets()
@@ -120,3 +124,85 @@ func TestGetAlertURLGenerator(t *testing.T) {
 		t.Fatalf("unexpected url want %s, got %s", exp, AlertURLGeneratorFn(testAlert))
 	}
 }
+
+func TestSendAlerts(t *testing.T) {
+	oldAlertURLGeneratorFn := AlertURLGeneratorFn
+	defer func() { AlertURLGeneratorFn = oldAlertURLGeneratorFn }()
+	AlertURLGeneratorFn = func(alert Alert) string {
+		return ""
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
+		t.Fatalf("should not be called")
+	})
+	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
+		var a []struct {
+			Labels map[string]string `json:"labels"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
+			t.Fatalf("can not unmarshal data into alert %s", err)
+		}
+		if len(a) != 2 {
+			t.Fatalf("expected 2 alert in array got %d", len(a))
+		}
+		if len(a[0].Labels) != 4 {
+			t.Fatalf("expected 4 labels got %d", len(a[0].Labels))
+		}
+		if a[0].Labels["env"] != "prod" {
+			t.Fatalf("expected env label to be prod during relabeling, got %s", a[0].Labels["env"])
+		}
+		if a[0].Labels["c"] != "baz" {
+			t.Fatalf("expected c label to be baz during relabeling, got %s", a[0].Labels["c"])
+		}
+		if len(a[1].Labels) != 1 {
+			t.Fatalf("expected 1 labels got %d", len(a[1].Labels))
+		}
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fs.MustRemovePath(f.Name())
+
+	rawConfig := `
+static_configs:
+  - targets:
+      - %s
+    alert_relabel_configs:
+    - source_labels: [b]
+      target_label: "c"
+alert_relabel_configs:
+  - source_labels: [a]
+    target_label: "b"
+  - target_label: "env"
+    replacement: "prod"
+`
+	config := fmt.Sprintf(rawConfig, srv.URL+alertManagerPath)
+	writeToFile(f.Name(), config)
+
+	oldConfigPath := configPath
+	defer func() { configPath = oldConfigPath }()
+	*configPath = f.Name()
+	err = Init(nil, "")
+	if err != nil {
+		t.Fatalf("unexpected error when parse notifier config: %s", err)
+	}
+
+	firingAlerts := []Alert{
+		{
+			Name:   "alert1",
+			Labels: map[string]string{"a": "baz"},
+		},
+		{
+			Name:   "alert2",
+			Labels: map[string]string{},
+		},
+	}
+	errG := Send(context.Background(), firingAlerts, nil)
+	if errG.Err() != nil {
+		t.Fatalf("unexpected error when sending alerts: %s", err)
+	}
+}

app/vmalert/notifier/notifier.go

@@ -1,13 +1,17 @@
 package notifier
 
-import "context"
+import (
+	"context"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+)
 
 // Notifier is a common interface for alert manager provider
 type Notifier interface {
 	// Send sends the given list of alerts.
 	// Returns an error if fails to send the alerts.
 	// Must unblock if the given ctx is cancelled.
-	Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error
+	Send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, notifierHeaders map[string]string) error
 	// Addr returns address where alerts are sent.
 	Addr() string
 	// LastError returns error, that occured during last attempt to send data

app/vmalert/notifier/notifier_blackhole.go

@@ -1,6 +1,10 @@
 package notifier
 
-import "context"
+import (
+	"context"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+)
 
 // blackHoleNotifier is a Notifier stub, used when no notifications need
 // to be sent.
@@ -10,7 +14,7 @@ type blackHoleNotifier struct {
 }
 
 // Send will send no notifications, but increase the metric.
-func (bh *blackHoleNotifier) Send(_ context.Context, alerts []Alert, _ map[string]string) error { //nolint:revive
+func (bh *blackHoleNotifier) Send(_ context.Context, alerts []Alert, _ [][]prompb.Label, _ map[string]string) error { //nolint:revive
 	bh.metrics.alertsSent.Add(len(alerts))
 	return nil
 }

app/vmalert/notifier/notifier_blackhole_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	metricset "github.com/VictoriaMetrics/metrics"
 )
 
@@ -16,7 +17,7 @@ func TestBlackHoleNotifier_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, nil); err != nil {
+	}}, [][]prompb.Label{{}}, nil); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
@@ -34,7 +35,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, nil); err != nil {
+	}}, [][]prompb.Label{{}}, nil); err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 

app/vmalert/notifier/testdata/consul.bad.yaml

@@ -0,0 +1,19 @@
+consul_sd_configs:
+  - server: localhost:8500
+    scheme: http
+    services:
+      - alertmanager
+    alert_relabel_configs:
+    - action: keep
+      source_labels: [env]
+      regex: "prod"
+  - server: localhost:8500
+    services:
+      - consul
+    alert_relabel_configs:
+    - action: keep
+      source_labels: [env]
+      regex: "(abc"
+alert_relabel_configs:
+  - target_label: "foo"
+    replacement: "aaa"

app/vmalert/notifier/testdata/dns.bad.yaml

@@ -0,0 +1,13 @@
+dns_sd_configs:
+  - names:
+      - cloudflare.com
+    type: 'A'
+    port: 9093
+relabel_configs:
+  - source_labels: [__meta_dns_name]
+    replacement: '${1}'
+    target_label: dns_name
+alert_relabel_configs:
+  - action: keep
+    source_labels: [env]
+    regex: "(abc"

app/vmalert/notifier/testdata/mixed.good.yaml

@@ -2,12 +2,19 @@ static_configs:
   - targets:
       - localhost:9093
       - localhost:9095
-
+    alert_relabel_configs:
+      - action: keep
+        source_labels: [env]
+        regex: "static"
 consul_sd_configs:
   - server: localhost:8500
     scheme: http
     services:
       - alertmanager
+    alert_relabel_configs:
+      - action: keep
+        source_labels: [env]
+        regex: "consul"
   - server: localhost:8500
     services:
       - consul
@@ -17,6 +24,10 @@ dns_sd_configs:
       - cloudflare.com
     type: 'A'
     port: 9093
+    alert_relabel_configs:
+      - action: keep
+        source_labels: [env]
+        regex: "dns"
 
 relabel_configs:
   - source_labels: [__meta_consul_tags]
@@ -25,4 +36,4 @@ relabel_configs:
     target_label: __scheme__
   - source_labels: [__meta_dns_name]
     replacement: '${1}'
-    target_label: dns_name
+    target_label: dns_name

app/vmalert/notifier/testdata/static.good.yaml

@@ -1,22 +1,14 @@
-headers:
-  - 'CustomHeader: foo'
-
 static_configs:
   - targets:
-      - localhost:9093
-      - localhost:9095
-      - https://localhost:9093/test/api/v2/alerts
-    basic_auth:
-      username: foo
-      password: bar
+      - http://192.168.0.101:9093
+    alert_relabel_configs:
+    - target_label: "foo"
+      replacement: "aaa"
 
   - targets:
-      - localhost:9096
-      - localhost:9097
-    basic_auth:
-      username: foo
-      password: baz
+      - http://192.168.0.101:9093
+    alert_relabel_configs:
+    - target_label: "foo"
+      replacement: "ccc"
+
 
-alert_relabel_configs:
-  - target_label: "foo"
-    replacement: "aaa"

app/vmalert/remoteread/init.go

@@ -14,9 +14,9 @@ import (
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect."+
-		"Remote read is used to restore alerts state."+
-		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with MetricsQL. It can be single node VictoriaMetrics or vmselect. "+
+		"Remote read is used to restore alerts state. "+
+		"This configuration makes sense only if vmalert was configured with '-remoteWrite.url' before and has been successfully persisted its state. "+
 		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
 

app/vmalert/remotewrite/client.go

@@ -173,9 +173,8 @@ func (c *Client) run(ctx context.Context) {
 
 		cancel()
 	}
-	c.wg.Add(1)
-	go func() {
-		defer c.wg.Done()
+
+	c.wg.Go(func() {
 		defer ticker.Stop()
 		for {
 			select {
@@ -197,7 +196,7 @@ func (c *Client) run(ctx context.Context) {
 				}
 			}
 		}
-	}()
+	})
 }
 
 var (

app/vmalert/rule/alerting_test.go

@@ -827,12 +827,9 @@ func TestGroup_Restore(t *testing.T) {
 		fg := NewGroup(config.Group{Name: "TestRestore", Rules: rules}, fqr, time.Second, nil)
 		fg.Init()
 		wg := sync.WaitGroup{}
-		wg.Add(1)
-		go func() {
-			nts := func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} }
-			fg.Start(context.Background(), nts, nil, fqr)
-			wg.Done()
-		}()
+		wg.Go(func() {
+			fg.Start(context.Background(), nil, fqr)
+		})
 		fg.Close()
 		wg.Wait()
 

app/vmalert/rule/group.go

@@ -18,7 +18,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 )
@@ -39,6 +38,8 @@ var (
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
 	remoteReadLookBack     = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries. "+
 		"For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+	maxStartDelay = flag.Duration("group.maxStartDelay", 5*time.Minute, "Defines the max delay before starting the group evaluation. Group's start is artificially delayed for random duration on interval"+
+		" [0..min(--group.maxStartDelay, group.interval)]. This helps smoothing out the load on the configured datasource, so evaluations aren't executed too close to each other.")
 )
 
 // Group is an entity for grouping rules
@@ -330,13 +331,13 @@ func (g *Group) Init() {
 }
 
 // Start starts group's evaluation
-func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, rr datasource.QuerierBuilder) {
+func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 	evalTS := time.Now()
 	// sleep random duration to spread group rules evaluation
-	// over time to reduce the load on datasource.
+	// over maxStartDelay to reduce the load on datasource.
 	if !SkipRandSleepOnGroupStart {
-		sleepBeforeStart := delayBeforeStart(evalTS, g.GetID(), g.Interval, g.EvalOffset)
+		sleepBeforeStart := g.delayBeforeStart(evalTS, *maxStartDelay)
 		g.infof("will start in %v", sleepBeforeStart)
 
 		sleepTimer := time.NewTimer(sleepBeforeStart)
@@ -368,7 +369,6 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 
 	e := &executor{
 		Rw:              rw,
-		Notifiers:       nts,
 		notifierHeaders: g.NotifierHeaders,
 	}
 
@@ -475,20 +475,31 @@ func (g *Group) UpdateWith(newGroup *Group) {
 	g.updateCh <- newGroup
 }
 
-// if offset is specified, delayBeforeStart returns a duration to help aligning timestamp with offset;
-// otherwise, it returns a random duration between [0..interval] based on group key.
-func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *time.Duration) time.Duration {
-	if offset != nil {
-		currentOffsetPoint := ts.Truncate(interval).Add(*offset)
+// delayBeforeStart returns duration for delaying the evaluation start
+// based on given ts and Group settings. The delay can't exceed maxDelay.
+// maxDelay is ignored if g.EvalOffset != nil.
+//
+// Delaying is important to smooth out the load on the datasource when all groups start at the same time.
+// delayBeforeStart calculates delay based on Group ID, so all groups will start at different moments of time.
+func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Duration {
+	if g.EvalOffset != nil {
+		// if offset is specified, ignore the maxDelay and return a duration aligned with offset
+		currentOffsetPoint := ts.Truncate(g.Interval).Add(*g.EvalOffset)
 		if currentOffsetPoint.Before(ts) {
 			// wait until the next offset point
-			return currentOffsetPoint.Add(interval).Sub(ts)
+			return currentOffsetPoint.Add(g.Interval).Sub(ts)
 		}
 		return currentOffsetPoint.Sub(ts)
 	}
 
+	// otherwise, return a random duration between [0..min(interval, maxDelay)] based on group ID
+	interval := g.Interval
+	if interval > maxDelay {
+		// artificially limit interval, so groups with big intervals could start sooner.
+		interval = maxDelay
+	}
 	var randSleep time.Duration
-	randSleep = time.Duration(float64(interval) * (float64(key) / (1 << 64)))
+	randSleep = time.Duration(float64(interval) * (float64(g.GetID()) / (1 << 64)))
 	sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())
 	if randSleep < sleepOffset {
 		randSleep += interval
@@ -550,15 +561,13 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 	if !disableProgressBar {
 		bar = pb.StartNew(iterations * len(g.Rules))
 	}
-	for _, r := range g.Rules {
+	for i := range g.Rules {
+		rule := g.Rules[i]
 		sem <- struct{}{}
-		wg.Add(1)
-		go func(r Rule, ri rangeIterator) {
-			// pass ri as a copy, so it can be modified within the replayRuleRange
-			res <- replayRuleRange(r, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
+		wg.Go(func() {
+			res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
 			<-sem
-			wg.Done()
-		}(r, ri)
+		})
 	}
 
 	wg.Wait()
@@ -588,10 +597,10 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 	res := make(chan int, int(ri.end.Sub(ri.start)/ri.step)+1)
 	for ri.next() {
 		sem <- struct{}{}
-		wg.Add(1)
-
-		go func(s, e time.Time) {
-			n, err := replayRule(r, s, e, rw, replayRuleRetryAttempts)
+		start := ri.s
+		end := ri.e
+		wg.Go(func() {
+			n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
 			if err != nil {
 				logger.Fatalf("rule %q: %s", r, err)
 			}
@@ -600,8 +609,7 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 			}
 			res <- n
 			<-sem
-			wg.Done()
-		}(ri.s, ri.e)
+		})
 	}
 	wg.Wait()
 	close(res)
@@ -615,10 +623,9 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
 }
 
 // ExecOnce evaluates all the rules under group for once with given timestamp.
-func (g *Group) ExecOnce(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, evalTS time.Time) chan error {
+func (g *Group) ExecOnce(ctx context.Context, rw remotewrite.RWClient, evalTS time.Time) chan error {
 	e := &executor{
 		Rw:              rw,
-		Notifiers:       nts,
 		notifierHeaders: g.NotifierHeaders,
 	}
 	if len(g.Rules) < 1 {
@@ -693,7 +700,6 @@ func (g *Group) getEvalDelay() time.Duration {
 
 // executor contains group's notify and rw configs
 type executor struct {
-	Notifiers       func() []notifier.Notifier
 	notifierHeaders map[string]string
 
 	Rw remotewrite.RWClient
@@ -714,14 +720,13 @@ func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.T
 	sem := make(chan struct{}, concurrency)
 	go func() {
 		wg := sync.WaitGroup{}
-		for _, r := range rules {
+		for i := range rules {
+			rule := rules[i]
 			sem <- struct{}{}
-			wg.Add(1)
-			go func(r Rule) {
-				res <- e.exec(ctx, r, ts, resolveDuration, limit)
+			wg.Go(func() {
+				res <- e.exec(ctx, rule, ts, resolveDuration, limit)
 				<-sem
-				wg.Done()
-			}(r)
+			})
 		}
 		wg.Wait()
 		close(res)
@@ -775,17 +780,6 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		return nil
 	}
 
-	wg := sync.WaitGroup{}
-	errGr := new(vmalertutil.ErrGroup)
-	for _, nt := range e.Notifiers() {
-		wg.Add(1)
-		go func(nt notifier.Notifier) {
-			if err := nt.Send(ctx, alerts, e.notifierHeaders); err != nil {
-				errGr.Add(fmt.Errorf("rule %q: failed to send alerts to addr %q: %w", r, nt.Addr(), err))
-			}
-			wg.Done()
-		}(nt)
-	}
-	wg.Wait()
+	errGr := notifier.Send(ctx, alerts, e.notifierHeaders)
 	return errGr.Err()
 }

app/vmalert/rule/group_test.go

@@ -262,7 +262,7 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 		updateCh: make(chan *Group),
 	}
 	g.Init()
-	go g.Start(context.Background(), nil, nil, nil)
+	go g.Start(context.Background(), nil, nil)
 
 	rule1 := AlertingRule{
 		Name: "jobDown",
@@ -346,7 +346,8 @@ func TestGroupStart(t *testing.T) {
 	}
 
 	fs := &datasource.FakeQuerier{}
-	fn := &notifier.FakeNotifier{}
+	fn, cleanup := notifier.InitFakeNotifier()
+	defer cleanup()
 
 	const evalInterval = time.Millisecond
 	g := NewGroup(groups[0], fs, evalInterval, map[string]string{"cluster": "east-1"})
@@ -395,7 +396,7 @@ func TestGroupStart(t *testing.T) {
 	fs.Add(m2)
 	g.Init()
 	go func() {
-		g.Start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil, fs)
+		g.Start(context.Background(), nil, fs)
 		close(finished)
 	}()
 
@@ -472,15 +473,10 @@ func TestFaultyNotifier(t *testing.T) {
 	r := newTestAlertingRule("instant", 0)
 	r.q = fq
 
-	fn := &notifier.FakeNotifier{}
-	e := &executor{
-		Notifiers: func() []notifier.Notifier {
-			return []notifier.Notifier{
-				&notifier.FaultyNotifier{},
-				fn,
-			}
-		},
-	}
+	fn, cleanup := notifier.InitFakeNotifier()
+	defer cleanup()
+
+	e := &executor{}
 	delay := 5 * time.Second
 	ctx, cancel := context.WithTimeout(context.Background(), delay)
 	defer cancel()
@@ -553,7 +549,7 @@ func TestCloseWithEvalInterruption(t *testing.T) {
 	g := NewGroup(groups[0], fq, evalInterval, nil)
 	g.Init()
 
-	go g.Start(context.Background(), nil, nil, nil)
+	go g.Start(context.Background(), nil, nil)
 
 	time.Sleep(evalInterval * 20)
 
@@ -571,9 +567,10 @@ func TestCloseWithEvalInterruption(t *testing.T) {
 
 func TestGroupStartDelay(t *testing.T) {
 	g := &Group{}
+	g.id = uint64(math.MaxUint64 / 10)
 	// interval of 5min and key generate a static delay of 30s
 	g.Interval = time.Minute * 5
-	key := uint64(math.MaxUint64 / 10)
+	maxDelay := time.Minute * 5
 
 	f := func(atS, expS string) {
 		t.Helper()
@@ -585,7 +582,7 @@ func TestGroupStartDelay(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		delay := delayBeforeStart(at, key, g.Interval, g.EvalOffset)
+		delay := g.delayBeforeStart(at, maxDelay)
 		gotStart := at.Add(delay)
 		if expTS != gotStart {
 			t.Fatalf("expected to get %v; got %v instead", expTS, gotStart)
@@ -606,6 +603,15 @@ func TestGroupStartDelay(t *testing.T) {
 	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
 	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
 	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
+
+	maxDelay = time.Minute * 1
+	g.EvalOffset = nil
+
+	// test group with maxDelay, and offset disabled
+	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:00:06.000+00:00")
+	f("2023-01-01T00:00:01.000+00:00", "2023-01-01T00:00:06.000+00:00")
+	f("2023-01-01T00:00:06.100+00:00", "2023-01-01T00:01:06.000+00:00")
+	f("2023-01-01T00:00:11.000+00:00", "2023-01-01T00:01:06.000+00:00")
 }
 
 func TestGetPrometheusReqTimestamp(t *testing.T) {

app/vmalert/static/css/custom.css

@@ -34,11 +34,12 @@ body {
   padding-top: 4.5rem;
 }
 
-.group-items {
+.vm-group {
   cursor: pointer;
   padding: 5px;
   margin-top: 5px;
   position: relative;
+  display: none;
 }
 
 .btn svg, .dropdown-item svg {
@@ -55,14 +56,22 @@ body {
   height: 38px;
 }
 
-.group-items:not(:has(.sub-item:not(.d-none))) {
-  display: none !important;
+.vm-item:not(.vm-found) {
+  display: none;
 }
 
-.group-items:hover {
+.vm-group:has(.vm-item:is(.vm-found)), .vm-group:is(.vm-found) {
+  display: flex;
+}
+
+.vm-group:hover {
   background-color: #f8f9fa!important;
 }
 
+.vm-group:is(.vm-found) .vm-item {
+  display: table-row;
+}
+
 .table {
   table-layout: fixed;
 }
@@ -111,3 +120,9 @@ textarea.curl-area {
 .w-60 {
   width: 60%;
 }
+
+.annotations {
+  white-space: pre-wrap;
+  color: gray;
+  word-wrap: break-word;
+}

app/vmalert/static/js/custom.js

@@ -65,32 +65,34 @@ function getParamURL(key) {
     return url.searchParams.get(key)
 }
 
+function matchText(search, item) {
+    const text = item.innerText.toLowerCase();
+    return text.indexOf(search) >= 0;
+}
+
 function filterRules(searchPhrase) {
-    document.querySelectorAll('.sub-items').forEach((rules) => {
-        let found = false;
-        rules.querySelectorAll('.sub-item').forEach((rule) => {
-            if (searchPhrase) {
-                const ruleName = rule.innerText.toLowerCase();
-                const matches = []
-                const hasValue = ruleName.indexOf(searchPhrase) >= 0;
-                rule.querySelectorAll('.label').forEach((label) => {
-                    const text = label.innerText.toLowerCase();
-                    if (text.indexOf(searchPhrase) >= 0) {
-                        matches.push(text);
-                    }
-                });
-                if (!matches.length && !hasValue) {
-                    rule.classList.add('d-none');
-                    return;
-                }
+    document.querySelectorAll('.vm-group').forEach((group) => {
+        if (!searchPhrase) {
+            group.classList.add('vm-found');
+            return;
+        }
+        for (const item of group.querySelectorAll('.vm-group-search')) {
+            if (matchText(searchPhrase, item)) {
+                group.classList.add('vm-found');
+                return;
             }
-            rule.classList.remove('d-none');
-            found = true;
-        });
-        if (found && searchPhrase || !searchPhrase) {
-            rules.classList.remove('d-none');
-        } else {
-            rules.classList.add('d-none');
+        }
+        group.classList.remove('vm-found');
+        for (const item of group.querySelectorAll('.vm-item')) {
+            if (matchText(searchPhrase, item)) {
+                item.classList.add('vm-found');
+                continue;
+            }
+            if (Array.from(item.querySelectorAll('.label')).find(l => matchText(searchPhrase, l))) {
+                item.classList.add('vm-found');
+                continue;
+            }
+            item.classList.remove('vm-found');
         }
     });
 }

app/vmalert/templates/template.go

@@ -485,6 +485,12 @@ func templateFuncs() textTpl.FuncMap {
 
 		/* Helpers */
 
+		// now returns the Unix timestamp in seconds at the time of the template evaluation.
+		// For example: {{ (now | toTime).Sub $activeAt }} will return the duration the alert has been active.
+		"now": func() float64 {
+			return float64(time.Now().Unix())
+		},
+
 		// Converts a list of objects to a map with keys arg0, arg1 etc.
 		// This is intended to allow multiple arguments to be passed to templates.
 		"args": func(args ...any) map[string]any {

app/vmalert/web.qtpl

@@ -114,14 +114,17 @@
         {%= Controls(prefix, currentIcon, currentText, icons, filters, true) %}
         {%  if len(groups) > 0 %}
             {% for _, g := range groups %}
-                <div id="group-{%s g.ID %}" class="d-flex w-100 border-0 flex-column group-items{% if g.Unhealthy > 0 %} alert-danger{% endif %}">
+                <div id="group-{%s g.ID %}" class="w-100 border-0 flex-column vm-group{% if g.Unhealthy > 0 %} alert-danger{% endif %}">
                     <span class="d-flex justify-content-between">
-                        <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
+                        <a
+                            class="vm-group-search"
+                            href="#group-{%s g.ID %}"
+                        >{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s) #</a>
                         <span
                             class="flex-grow-1 d-flex justify-content-end"
                             role="button"
                             data-bs-toggle="collapse"
-                            data-bs-target="#sub-{%s g.ID %}"
+                            data-bs-target="#item-{%s g.ID %}"
                         >
                             <span class="d-flex gap-2">
                                 {% if g.Unhealthy > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.Unhealthy %}</span> {% endif %}
@@ -134,9 +137,9 @@
                         class="d-flex flex-column row-gap-2 mb-2"
                         role="button"
                         data-bs-toggle="collapse"
-                        data-bs-target="#sub-{%s g.ID %}"
+                        data-bs-target="#item-{%s g.ID %}"
                     >
-                        <span class="fs-6 text-start w-100 fw-lighter">{%s g.File %}</span>
+                        <span class="fs-6 text-start vm-group-search w-100 fw-lighter">{%s g.File %}</span>
                         {% if len(g.Params) > 0 %}
                             <span class="fs-6 text-start w-100 d-flex justify-content-between fw-lighter">
                                 <span>Extra params</span>
@@ -158,7 +161,7 @@
                             </span>
                         {% endif %}
                     </span>
-                    <div class="collapse sub-items" id="sub-{%s g.ID %}">
+                    <div class="collapse" id="item-{%s g.ID %}">
                         <table class="table table-striped table-hover table-sm">
                             <thead>
                                 <tr>
@@ -169,7 +172,7 @@
                             </thead>
                             <tbody>
                                 {% for _, r := range g.Rules %}
-                                    <tr class="sub-item{% if r.LastError != "" %} alert-danger{% endif %}">
+                                    <tr class="vm-item{% if r.LastError != "" %} alert-danger{% endif %}">
                                         <td>
                                             <div class="row">
                                                 <div class="col-12 mb-2">
@@ -206,7 +209,12 @@
                                             </div>
                                         </td>
                                         <td class="text-center">{%d r.LastSamples %}</td>
-                                        <td class="text-center">{%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago</td>
+                                        <td class="text-center">{% if r.LastEvaluation.IsZero() %}
+                                             Never
+                                         {% else %}
+                                            {%f.3 time.Since(r.LastEvaluation).Seconds() %}s ago
+                                         {% endif %}
+                                        </td>
                                     </tr>
                                 {% endfor %}
                             </tbody>
@@ -241,14 +249,14 @@
                  }
                  sort.Strings(keys)
              %}
-             <div class="d-flex w-100 flex-column group-items alert-danger">
+             <div class="w-100 flex-column vm-group alert-danger">
                  <span id="group-{%s g.ID %}" class="d-flex justify-content-between">
                      <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %}</a>
                      <span
                          class="flex-grow-1 d-flex justify-content-end"
                          role="button"
                          data-bs-toggle="collapse"
-                         data-bs-target="#sub-{%s g.ID %}"
+                         data-bs-target="#item-{%s g.ID %}"
                      >
                          <span class="badge bg-danger" title="Number of active alerts">{%d len(ga.Alerts) %}</span>
                      </span>
@@ -258,10 +266,10 @@
                          class="fs-6 text-start w-100 fw-lighter"
                          role="button" 
                          data-bs-toggle="collapse"
-                         data-bs-target="#sub-{%s g.ID %}"
+                         data-bs-target="#item-{%s g.ID %}"
                      >{%s g.File %}</span>
                  </span>
-                 <div class="collapse sub-items" id="sub-{%s g.ID %}">
+                 <div class="collapse" id="item-{%s g.ID %}">
                      {% for _, ruleID := range keys %}
                          {%code
                              defaultAR := alertsByRule[ruleID][0]
@@ -272,7 +280,7 @@
                              sort.Strings(labelKeys)
                          %}
                          <br>
-                         <div class="sub-item">
+                         <div class="vm-item">
                              <b>alert:</b> {%s defaultAR.Name %} ({%d len(alertsByRule[ruleID]) %})
                              | <span><a target="_blank" href="{%s defaultAR.SourceLink %}">Source</a></span>
                              <br>
@@ -337,20 +345,20 @@
                 typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
                 count := len(ns)
             %}
-            <div class="d-flex w-100 flex-column group-items">
+            <div class="w-100 flex-column vm-group">
                 <span class="d-flex justify-content-between" id="group-{%s typeK %}">
                     <a href="#group-{%s typeK %}">{%s typeK %} ({%d count %})</a>
                     <span
                         class="flex-grow-1"
                         role="button"
                         data-bs-toggle="collapse"
-                        data-bs-target="#sub-{%s typeK %}"
+                        data-bs-target="#item-{%s typeK %}"
                     ></span>
                 </span>
-                <div id="sub-{%s typeK %}" class="collapse show sub-items">
+                <div id="item-{%s typeK %}" class="collapse show">
                     <table class="table table-striped table-hover table-sm">
                         <thead>
-                            <tr class="sub-item">
+                            <tr class="vm-item">
                                 <th scope="col">Labels</th>
                                 <th scope="col">Address</th>
                             </tr>
@@ -435,7 +443,7 @@
         <div class="col">
            {% for _, k := range annotationKeys %}
                 <b>{%s k %}:</b><br>
-                <p>{%s alert.Annotations[k] %}</p>
+                <p class="annotations">{%s alert.Annotations[k] %}</p>
           {% endfor %}
         </div>
       </div>
@@ -549,7 +557,7 @@
         <div class="col">
           {% for _, k := range annotationKeys %}
                 <b>{%s k %}:</b><br>
-                <p>{%s rule.Annotations[k] %}</p>
+                <p class="annotations">{%s rule.Annotations[k] %}</p>
           {% endfor %}
         </div>
       </div>

app/vmalert/web_test.go

@@ -23,6 +23,9 @@ func TestHandler(t *testing.T) {
 		Timestamps: []int64{0},
 	})
 	m := &manager{groups: map[uint64]*rule.Group{}}
+	_, cleanup := notifier.InitFakeNotifier()
+	defer cleanup()
+
 	var ar *rule.AlertingRule
 	var rr *rule.RecordingRule
 	var groupIDs []uint64
@@ -45,7 +48,7 @@ func TestHandler(t *testing.T) {
 		}, fq, 1*time.Minute, nil)
 		ar = g.Rules[0].(*rule.AlertingRule)
 		rr = g.Rules[1].(*rule.RecordingRule)
-		g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
+		g.ExecOnce(context.Background(), nil, time.Time{})
 		id := g.CreateID()
 		m.groups[id] = g
 		groupIDs = append(groupIDs, id)

app/vmauth/Makefile

@@ -27,6 +27,9 @@ vmauth-linux-ppc64le-prod:
 vmauth-linux-386-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-linux-386
 
+vmauth-linux-s390x-prod:
+	APP_NAME=vmauth $(MAKE) app-via-docker-linux-s390x
+
 vmauth-darwin-amd64-prod:
 	APP_NAME=vmauth $(MAKE) app-via-docker-darwin-amd64
 

app/vmauth/auth_config.go

@@ -482,27 +482,34 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		if bu.isBroken() {
 			continue
 		}
-		if bu.concurrentRequests.Load() == 0 {
-			// Fast path - return the backend with zero concurrently executed requests.
-			// Do not use CompareAndSwap() instead of Load(), since it is much slower on systems with many CPU cores.
-			bu.concurrentRequests.Add(1)
+
+		// The Load() in front of CompareAndSwap() avoids CAS overhead for items with values bigger than 0.
+		if bu.concurrentRequests.Load() == 0 && bu.concurrentRequests.CompareAndSwap(0, 1) {
+			atomicCounter.CompareAndSwap(n+1, idx+1)
+			// There is no need in the call bu.get(), because we already incremented bu.concrrentRequests above.
 			return bu
 		}
 	}
 
 	// Slow path - return the backend with the minimum number of concurrently executed requests.
-	buMin := bus[n%uint32(len(bus))]
-	minRequests := buMin.concurrentRequests.Load()
-	for _, bu := range bus {
+	buMinIdx := n % uint32(len(bus))
+	minRequests := bus[buMinIdx].concurrentRequests.Load()
+	for i := uint32(0); i < uint32(len(bus)); i++ {
+		idx := (n + i) % uint32(len(bus))
+		bu := bus[idx]
 		if bu.isBroken() {
 			continue
 		}
-		if n := bu.concurrentRequests.Load(); n < minRequests || buMin.isBroken() {
-			buMin = bu
-			minRequests = n
+
+		reqs := bu.concurrentRequests.Load()
+		if reqs < minRequests || bus[buMinIdx].isBroken() {
+			buMinIdx = idx
+			minRequests = reqs
 		}
 	}
+	buMin := bus[buMinIdx]
 	buMin.get()
+	atomicCounter.CompareAndSwap(n+1, buMinIdx+1)
 	return buMin
 }
 

app/vmauth/auth_config_test.go

@@ -752,10 +752,12 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	})
 	up.loadBalancingPolicy = "least_loaded"
 
+	pbus := up.bus.Load()
+	bus := *pbus
+
 	fn := func(ns ...int) {
 		t.Helper()
-		pbus := up.bus.Load()
-		bus := *pbus
+
 		for i, b := range bus {
 			got := int(b.concurrentRequests.Load())
 			exp := ns[i]
@@ -767,45 +769,52 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 
 	up.getBackendURL()
 	fn(1, 0, 0)
+
 	up.getBackendURL()
 	fn(1, 1, 0)
+
 	up.getBackendURL()
 	fn(1, 1, 1)
 
-	up.getBackendURL()
-	up.getBackendURL()
-	fn(2, 2, 1)
-
-	bus := up.bus.Load()
-	pbus := *bus
-	pbus[0].concurrentRequests.Add(2)
-	pbus[2].concurrentRequests.Add(5)
-	fn(4, 2, 6)
+	bus[1].put()
+	bus[2].put()
+	fn(1, 0, 0)
 
 	up.getBackendURL()
-	fn(4, 3, 6)
+	fn(1, 1, 0)
 
+	bus[1].put()
 	up.getBackendURL()
-	fn(4, 4, 6)
-
-	up.getBackendURL()
-	fn(4, 5, 6)
-
-	up.getBackendURL()
-	fn(5, 5, 6)
-
-	up.getBackendURL()
-	fn(6, 5, 6)
-
-	up.getBackendURL()
-	fn(6, 6, 6)
-
-	up.getBackendURL()
-	fn(6, 6, 7)
+	fn(1, 0, 1)
 
 	up.getBackendURL()
 	up.getBackendURL()
-	fn(7, 7, 7)
+	fn(1, 1, 2)
+
+	bus[0].concurrentRequests.Add(2)
+	bus[2].concurrentRequests.Add(2)
+	fn(3, 1, 4)
+
+	up.getBackendURL()
+	fn(3, 2, 4)
+
+	up.getBackendURL()
+	fn(3, 3, 4)
+
+	up.getBackendURL()
+	fn(4, 3, 4)
+
+	up.getBackendURL()
+	fn(4, 4, 4)
+
+	bus[0].put()
+	bus[2].put()
+
+	up.getBackendURL()
+	fn(3, 4, 4)
+
+	up.getBackendURL()
+	fn(4, 4, 4)
 }
 
 func TestBrokenBackend(t *testing.T) {

app/vmauth/main.go

@@ -310,14 +310,21 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	res, err := ui.rt.RoundTrip(req)
+
+	if ctxErr := r.Context().Err(); ctxErr != nil {
+		// Override the error returned by the RoundTrip with the context error if it isn't non-nil
+		// This makes sure the proper logging for canceled and timed out requests - log the real cause of the error
+		// instead of the random error, which could be returned from RoundTrip because of canceled or timed out request.
+		err = ctxErr
+	}
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 			// Do not retry canceled or timed out requests
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
-			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
 			if errors.Is(err, context.DeadlineExceeded) {
 				// Timed out request must be counted as errors, since this usually means that the backend is slow.
+				logger.Warnf("remoteAddr: %s; requestURI: %s; timeout while proxying the response from %s: %s", remoteAddr, requestURI, targetURL, err)
 				ui.backendErrors.Inc()
 			}
 			return false, false

app/vmbackup/Makefile

@@ -31,6 +31,9 @@ vmbackup-linux-ppc64le-prod:
 vmbackup-linux-386-prod:
 	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-386
 
+vmbackup-linux-s390x-prod:
+	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-s390x
+
 vmbackup-darwin-amd64-prod:
 	APP_NAME=vmbackup EXTRA_GO_BUILD_TAGS=$(VMBACKUP_GO_BUILD_TAGS) $(MAKE) app-via-docker-darwin-amd64
 

app/vmctl/Makefile

@@ -27,6 +27,9 @@ vmctl-linux-ppc64le-prod:
 vmctl-linux-386-prod:
 	APP_NAME=vmctl $(MAKE) app-via-docker-linux-386
 
+vmctl-linux-s390x-prod:
+	APP_NAME=vmctl $(MAKE) app-via-docker-linux-s390x
+
 vmctl-darwin-amd64-prod:
 	APP_NAME=vmctl $(MAKE) app-via-docker-darwin-amd64
 

app/vmctl/flags.go

@@ -689,15 +689,15 @@ var (
 			Usage:  "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
 			Layout: time.RFC3339,
 		},
-		&cli.StringFlag{
-			Name:  remoteReadFilterLabel,
-			Usage: "Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
-			Value: "__name__",
+		&cli.StringSliceFlag{
+			Name:        remoteReadFilterLabel,
+			Usage:       "Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
+			DefaultText: "__name__",
 		},
-		&cli.StringFlag{
-			Name:  remoteReadFilterLabelValue,
-			Usage: fmt.Sprintf("Prometheus regular expression to filter label from %q flag.", remoteReadFilterLabelValue),
-			Value: ".*",
+		&cli.StringSliceFlag{
+			Name:        remoteReadFilterLabelValue,
+			Usage:       fmt.Sprintf("Prometheus regular expression to filter label from %q flag.", remoteReadFilterLabelValue),
+			DefaultText: ".*",
 		},
 		&cli.BoolFlag{
 			Name:  remoteRead,

app/vmctl/main.go

@@ -192,6 +192,14 @@ func main() {
 						return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
 					}
 
+					// Backwards compatible default values if none provided by user
+					rrLabelNames := c.StringSlice(remoteReadFilterLabel)
+					rrLabelValues := c.StringSlice(remoteReadFilterLabelValue)
+					if len(rrLabelNames) == 0 && len(rrLabelValues) == 0 {
+						rrLabelNames = []string{"__name__"}
+						rrLabelValues = []string{".*"}
+					}
+
 					rr, err := remoteread.NewClient(remoteread.Config{
 						Addr:              addr,
 						Transport:         tr,
@@ -200,8 +208,8 @@ func main() {
 						Timeout:           c.Duration(remoteReadHTTPTimeout),
 						UseStream:         c.Bool(remoteReadUseStream),
 						Headers:           c.String(remoteReadHeaders),
-						LabelName:         c.String(remoteReadFilterLabel),
-						LabelValue:        c.String(remoteReadFilterLabelValue),
+						LabelNames:        rrLabelNames,
+						LabelValues:       rrLabelValues,
 						DisablePathAppend: c.Bool(remoteReadDisablePathAppend),
 					})
 					if err != nil {

app/vmctl/remoteread/remoteread.go

@@ -11,14 +11,15 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 const (
@@ -63,9 +64,9 @@ type Config struct {
 	UseStream bool
 	// Headers optional HTTP headers to send with each request to the corresponding remote storage
 	Headers string
-	// LabelName, LabelValue stands for label=~value pair used for read requests.
+	// LabelNames, LabelValues stands for label=~value pair used for read requests.
 	// Is optional.
-	LabelName, LabelValue string
+	LabelNames, LabelValues []string
 }
 
 // Filter defines a list of filters applied to requested data
@@ -94,12 +95,22 @@ func NewClient(cfg Config) (*Client, error) {
 		return nil, err
 	}
 
-	var m *prompb.LabelMatcher
-	if cfg.LabelName != "" && cfg.LabelValue != "" {
-		m = &prompb.LabelMatcher{
-			Type:  prompb.LabelMatcher_RE,
-			Name:  cfg.LabelName,
-			Value: cfg.LabelValue,
+	var matchers []*prompb.LabelMatcher
+	if len(cfg.LabelNames) > 0 || len(cfg.LabelValues) > 0 {
+		if len(cfg.LabelNames) != len(cfg.LabelValues) {
+			return nil, fmt.Errorf("the number of label names and label values must be the same")
+		}
+
+		for i := range cfg.LabelNames {
+			if cfg.LabelNames[i] == "" {
+				return nil, fmt.Errorf("label name cannot be empty")
+			}
+			matcher := &prompb.LabelMatcher{
+				Type:  prompb.LabelMatcher_RE,
+				Name:  cfg.LabelNames[i],
+				Value: cfg.LabelValues[i],
+			}
+			matchers = append(matchers, matcher)
 		}
 	}
 
@@ -116,7 +127,7 @@ func NewClient(cfg Config) (*Client, error) {
 		password:          cfg.Password,
 		useStream:         cfg.UseStream,
 		headers:           headers,
-		matchers:          []*prompb.LabelMatcher{m},
+		matchers:          matchers,
 	}
 
 	return c, nil

app/vminsert/common/insert_ctx.go

@@ -221,7 +221,7 @@ func (ctx *InsertCtx) FlushBufs() error {
 	}
 }
 
-func (ctx *InsertCtx) dropAggregatedRows(matchIdxs []byte) {
+func (ctx *InsertCtx) dropAggregatedRows(matchIdxs []uint32) {
 	dst := ctx.mrs[:0]
 	src := ctx.mrs
 	if !*streamAggrDropInput {
@@ -239,4 +239,4 @@ func (ctx *InsertCtx) dropAggregatedRows(matchIdxs []byte) {
 	ctx.mrs = dst
 }
 
-var matchIdxsPool bytesutil.ByteBufferPool
+var matchIdxsPool slicesutil.BufferPool[uint32]

app/vminsert/common/streamaggr.go

@@ -13,6 +13,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
@@ -22,11 +23,11 @@ var (
 	streamAggrConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/victoriametrics/stream-aggregation/ . "+
 		"See also -streamAggr.keepInput, -streamAggr.dropInput and -streamAggr.dedupInterval")
-	streamAggrKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep all the input samples after the aggregation with -streamAggr.config. "+
-		"By default, only aggregated samples are dropped, while the remaining samples are stored in the database. "+
+	streamAggrKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep input samples that match any rule in -streamAggr.config. "+
+		"By default, matched raw samples are aggregated and dropped, while unmatched samples are written to the remote storage. "+
 		"See also -streamAggr.dropInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
-	streamAggrDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop all the input samples after the aggregation with -streamAggr.config. "+
-		"By default, only aggregated samples are dropped, while the remaining samples are stored in the database. "+
+	streamAggrDropInput = flag.Bool("streamAggr.dropInput", false, "Whether to drop input samples that not matching any rule in -streamAggr.config. "+
+		"By default, only matched raw samples are dropped, while unmatched samples are written to the remote storage."+
 		"See also -streamAggr.keepInput and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/")
 	streamAggrDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before optional aggregation with -streamAggr.config . "+
 		"See also -streamAggr.dropInputLabels and -dedup.minScrapeInterval and https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#deduplication")
@@ -189,7 +190,7 @@ func (ctx *streamAggrCtx) Reset() {
 	ctx.buf = ctx.buf[:0]
 }
 
-func (ctx *streamAggrCtx) push(mrs []storage.MetricRow, matchIdxs []byte) []byte {
+func (ctx *streamAggrCtx) push(mrs []storage.MetricRow, matchIdxs []uint32) []uint32 {
 	mn := &ctx.mn
 	tss := ctx.tss
 	labels := ctx.labels
@@ -248,7 +249,7 @@ func (ctx *streamAggrCtx) push(mrs []storage.MetricRow, matchIdxs []byte) []byte
 	if sas.IsEnabled() {
 		matchIdxs = sas.Push(tss, matchIdxs)
 	} else if deduplicator != nil {
-		matchIdxs = bytesutil.ResizeNoCopyMayOverallocate(matchIdxs, len(tss))
+		matchIdxs = slicesutil.SetLength(matchIdxs, len(tss))
 		for i := range matchIdxs {
 			matchIdxs[i] = 1
 		}

app/vminsert/relabel/relabel.go

@@ -86,7 +86,7 @@ func loadRelabelConfig() (*promrelabel.ParsedConfigs, error) {
 	if len(*relabelConfig) == 0 {
 		return nil, nil
 	}
-	pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig)
+	pcs, _, err := promrelabel.LoadRelabelConfigs(*relabelConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error when reading -relabelConfig=%q: %w", *relabelConfig, err)
 	}

app/vmrestore/Makefile

@@ -31,6 +31,9 @@ vmrestore-linux-ppc64le-prod:
 vmrestore-linux-386-prod:
 	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-386
 
+vmrestore-linux-s390x-prod:
+	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-linux-s390x
+
 vmrestore-darwin-amd64-prod:
 	APP_NAME=vmrestore EXTRA_GO_BUILD_TAGS=$(VMRESTORE_GO_BUILD_TAGS) $(MAKE) app-via-docker-darwin-amd64
 

app/vmselect/prometheus/prometheus.go

@@ -56,7 +56,7 @@ var (
 	maxTSDBStatusSeries     = flag.Int("search.maxTSDBStatusSeries", 10e6, "The maximum number of time series, which can be processed during the call to /api/v1/status/tsdb. This option allows limiting memory usage")
 	maxSeriesLimit          = flag.Int("search.maxSeries", 30e3, "The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage")
 	maxDeleteSeries         = flag.Int("search.maxDeleteSeries", 1e6, "The maximum number of time series, which can be deleted using /api/v1/admin/tsdb/delete_series. This option allows limiting memory usage")
-	maxTSDBStatusTopNSeries = flag.Int("search.maxTSDBStatusTopNSeries", 1000, "The maximum value of `topN` argument that can be passed to /api/v1/status/tsdb API. This option allows limiting memory usage. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats")
+	maxTSDBStatusTopNSeries = flag.Int("search.maxTSDBStatusTopNSeries", 1000, "The maximum value of 'topN' argument that can be passed to /api/v1/status/tsdb API. This option allows limiting memory usage. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#tsdb-stats")
 	maxLabelsAPISeries      = flag.Int("search.maxLabelsAPISeries", 1e6, "The maximum number of time series, which could be scanned when searching for the matching time series "+
 		"at /api/v1/labels and /api/v1/label/.../values. This option allows limiting memory usage and CPU usage. See also -search.maxLabelsAPIDuration, "+
 		"-search.maxTagKeys, -search.maxTagValues and -search.ignoreExtraFiltersAtLabelsAPI")

app/vmselect/promql/eval.go

@@ -49,7 +49,7 @@ var (
 	minWindowForInstantRollupOptimization = flag.Duration("search.minWindowForInstantRollupOptimization", time.Hour*3, "Enable cache-based optimization for repeated queries "+
 		"to /api/v1/query (aka instant queries), which contain rollup functions with lookbehind window exceeding the given value")
 	maxBinaryOpPushdownLabelValues = flag.Int("search.maxBinaryOpPushdownLabelValues", 100, "The maximum number of values for a label in the first expression that can be extracted as a common label filter and pushed down to the second expression in a binary operation. "+
-		"A larger value makes the pushed-down filter more complex but fewer time series will be returned. This flag is useful when selective label contains numerous values, for example `instance`, and storage resources are abundant.")
+		"A larger value makes the pushed-down filter more complex but fewer time series will be returned. This flag is useful when selective label (e.g., 'instance') contains numerous values, and storage resources are abundant.")
 )
 
 // The minimum number of points per timeseries for enabling time rounding.

app/vmselect/promql/rollup_result_cache.go

@@ -183,24 +183,12 @@ func InitRollupResultCache(cachePath string) {
 
 // StopRollupResultCache closes the rollupResult cache.
 func StopRollupResultCache() {
-	if len(rollupResultCachePath) == 0 {
-		rollupResultCacheV.c.Stop()
-		rollupResultCacheV.c = nil
-		return
+	if rollupResultCachePath != "" {
+		rollupResultCacheV.c.MustSave(rollupResultCachePath)
+		mustSaveRollupResultCacheKeyPrefix(rollupResultCachePath)
 	}
-	logger.Infof("saving rollupResult cache to %q...", rollupResultCachePath)
-	startTime := time.Now()
-	if err := rollupResultCacheV.c.Save(rollupResultCachePath); err != nil {
-		logger.Errorf("cannot save rollupResult cache at %q: %s", rollupResultCachePath, err)
-		return
-	}
-	mustSaveRollupResultCacheKeyPrefix(rollupResultCachePath)
-	var fcs fastcache.Stats
-	rollupResultCacheV.c.UpdateStats(&fcs)
 	rollupResultCacheV.c.Stop()
 	rollupResultCacheV.c = nil
-	logger.Infof("saved rollupResult cache to %q in %.3f seconds; entriesCount: %d, sizeBytes: %d",
-		rollupResultCachePath, time.Since(startTime).Seconds(), fcs.EntriesCount, fcs.BytesSize)
 }
 
 type rollupResultCache struct {

app/vmselect/vmui/index.html

@@ -37,10 +37,10 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-D13qGB62.js"></script>
+  <script type="module" crossorigin src="./assets/index-zpalCSif.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-DY9kCvzk.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-I8MVeF75.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-CBxdwuZH.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmstorage/main.go

@@ -673,15 +673,6 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexps"}`, storage.RegexpCacheMisses())
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMisses())
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/tsid", reason="cache_size"}`, m.TSIDCacheSizeEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/tsid", reason="miss_percentage"}`, m.TSIDCacheMissEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/tsid", reason="expiration"}`, m.TSIDCacheExpireEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricName", reason="cache_size"}`, m.MetricNameCacheSizeEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricName", reason="miss_percentage"}`, m.MetricNameCacheMissEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricName", reason="expiration"}`, m.MetricNameCacheExpireEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricIDs", reason="cache_size"}`, m.MetricIDCacheSizeEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricIDs", reason="miss_percentage"}`, m.MetricIDCacheMissEvictionBytes)
-	metrics.WriteCounterUint64(w, `vm_cache_eviction_bytes_total{type="storage/metricIDs", reason="expiration"}`, m.MetricIDCacheExpireEvictionBytes)
 
 	metrics.WriteCounterUint64(w, `vm_deleted_metrics_total{type="indexdb"}`, m.DeletedMetricsCount)
 

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.3 AS build-web-stage
+FROM golang:1.25.4 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/package.json

@@ -12,6 +12,7 @@
     "build": "vite build",
     "build:anomaly": "vite build --mode vmanomaly",
     "lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
+    "lint:local": "eslint --ext .ts,.tsx -f stylish src",
     "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
     "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
     "preview": "vite preview",
@@ -29,7 +30,7 @@
     "react-input-mask": "^2.0.4",
     "react-router-dom": "^7.6.3",
     "uplot": "^1.6.32",
-    "vite": "^7.1.5",
+    "vite": "^7.1.11",
     "web-vitals": "^5.0.3"
   },
   "devDependencies": {

app/vmui/packages/vmui/src/components/Chart/Heatmap/LegendHeatmap/LegendHeatmap.tsx

@@ -21,8 +21,9 @@ const LegendHeatmap: FC<LegendHeatmapProps> = ({
   const [maxFormat, setMaxFormat] = useState("");
 
   const value = useMemo(() => {
-    return parseFloat(String(legendValue?.value || 0).replace("%", ""));
-  }, [legendValue]);
+    const n = Number(String(legendValue?.value ?? "").replace("%","").replace(",", "."));
+    return Number.isFinite(n) ? n : 0;
+  }, [legendValue?.value]);
 
   useEffect(() => {
     setPercent(value ? (value - min) / (max - min) * 100 : 0);

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendGroup.tsx

@@ -1,4 +1,5 @@
-import { FC, MouseEvent, useMemo } from "react";
+import { FC, useMemo } from "react";
+import { TargetedMouseEvent } from "preact";
 import { LegendItemType } from "../../../../types";
 import { useLegendView } from "./hooks/useLegendView";
 import LegendLines from "./LegendViews/LegendLines";
@@ -7,6 +8,7 @@ import { useHideDuplicateFields } from "./hooks/useHideDuplicateFields";
 import Accordion from "../../../Main/Accordion/Accordion";
 import { useLegendGroup } from "./hooks/useLegendGroup";
 import useCopyToClipboard from "../../../../hooks/useCopyToClipboard";
+import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
 
 export type LegendProps = {
   labels: LegendItemType[];
@@ -29,7 +31,7 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
     return labels.sort((x, y) => (y.median || 0) - (x.median || 0));
   }, [labels]);
 
-  const createHandlerCopy = (value: string) => async (e: MouseEvent<HTMLDivElement>) => {
+  const createHandlerCopy = (value: string) => async (e: TargetedMouseEvent<HTMLDivElement>) => {
     e.stopPropagation();
     await copyToClipboard(value, `${value} has been copied`);
   };
@@ -42,7 +44,7 @@ const LegendGroup: FC<LegendGroupProps> = ({ labels, group, isAnomalyView, onCha
       key={group}
     >
       <Accordion
-        defaultExpanded={true}
+        defaultExpanded={sortedLabels.length < DEFAULT_MAX_SERIES.chart}
         title={(
           <div className="vm-legend-group-header">
             <div className="vm-legend-group-header-title">

app/vmui/packages/vmui/src/components/Chart/Line/LineChart/LineChart.tsx

@@ -42,6 +42,7 @@ export interface LineChartProps {
   height?: number;
   isAnomalyView?: boolean;
   spanGaps?: boolean;
+  showAllPoints?: boolean;
 }
 
 const LineChart: FC<LineChartProps> = ({
@@ -55,7 +56,8 @@ const LineChart: FC<LineChartProps> = ({
   layoutSize,
   height,
   isAnomalyView,
-  spanGaps = false
+  spanGaps = false,
+  showAllPoints = false,
 }) => {
   const { isDarkTheme } = useAppState();
 
@@ -108,10 +110,10 @@ const LineChart: FC<LineChartProps> = ({
   useEffect(() => {
     if (!uPlotInst) return;
     delSeries(uPlotInst);
-    addSeries(uPlotInst, series, spanGaps);
+    addSeries(uPlotInst, series, spanGaps, showAllPoints);
     setBand(uPlotInst, series);
     uPlotInst.redraw();
-  }, [series, spanGaps]);
+  }, [series, spanGaps, showAllPoints]);
 
   useEffect(() => {
     if (!uPlotInst) return;

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/GraphSettings.tsx

@@ -14,6 +14,7 @@ import LegendConfigs from "../../Chart/Line/Legend/LegendConfigs/LegendConfigs";
 import Modal from "../../Main/Modal/Modal";
 import { useGraphDispatch, useGraphState } from "../../../state/graph/GraphStateContext";
 import { useEffect } from "react";
+import PointsConfigurator from "./PointsConfigurator/PointsConfigurator";
 
 const title = "Graph & Legend Settings";
 
@@ -26,10 +27,14 @@ interface GraphSettingsProps {
     value: boolean,
     onChange: (value: boolean) => void,
   },
+  showAllPoints: {
+    value: boolean,
+    onChange: (value: boolean) => void,
+  },
   isHistogram?: boolean,
 }
 
-const GraphSettings: FC<GraphSettingsProps> = ({ data, yaxis, setYaxisLimits, toggleEnableLimits, spanGaps }) => {
+const GraphSettings: FC<GraphSettingsProps> = ({ data, yaxis, setYaxisLimits, toggleEnableLimits, spanGaps, showAllPoints }) => {
   const { openSettings } = useGraphState();
   const graphDispatch = useGraphDispatch();
 
@@ -84,6 +89,10 @@ const GraphSettings: FC<GraphSettingsProps> = ({ data, yaxis, setYaxisLimits, to
                   spanGaps={spanGaps.value}
                   onChange={spanGaps.onChange}
                 />
+                <PointsConfigurator
+                  showAllPoints={showAllPoints.value}
+                  onChangeShow={showAllPoints.onChange}
+                />
                 {displayHistogramMode && <GraphTypeSwitcher onChange={handleClose}/>}
               </div>
             </div>

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/LinesConfigurator/LinesConfigurator.tsx

@@ -20,7 +20,7 @@ const LinesConfigurator: FC<Props> = ({ spanGaps, onChange }) => {
         fullWidth={isMobile}
       />
       <span className="vm-legend-configs-item__info">
-        Connects data points by skipping null values instead of creating gaps.
+        Connects data points by skipping null values instead of gaps.
       </span>
     </div>
   );

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/PointsConfigurator/PointsConfigurator.tsx

@@ -0,0 +1,31 @@
+import { FC } from "preact/compat";
+import Switch from "../../../Main/Switch/Switch";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+
+interface Props {
+  showAllPoints: boolean;
+  onChangeShow: (value: boolean) => void;
+}
+
+const PointsConfigurator: FC<Props> = ({ showAllPoints, onChangeShow }) => {
+  const { isMobile } = useDeviceDetect();
+
+  return (
+    <>
+      <div className="vm-graph-settings-row">
+        <span className="vm-graph-settings-row__label">Show all data points</span>
+        <Switch
+          value={showAllPoints}
+          onChange={onChangeShow}
+          label={showAllPoints ? "Enabled" : "Disabled"}
+          fullWidth={isMobile}
+        />
+        <span className="vm-legend-configs-item__info">
+          Display every data point, even when no line can be drawn.
+        </span>
+      </div>
+    </>
+  );
+};
+
+export default PointsConfigurator;

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/style.scss

@@ -46,5 +46,6 @@
     align-items: center;
     justify-content: stretch;
     grid-template-columns: 1fr 100px;
+    gap: 0 $padding-large;
   }
 }

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseAlert/index.tsx

@@ -4,10 +4,10 @@ import { Alert as APIAlert } from "../../../types";
 import { createSearchParams } from "react-router-dom";
 import Button from "../../Main/Button/Button";
 import Badges, { BadgeColor } from "../Badges";
+import { formatEventTime } from "../helpers";
 import {
   SearchIcon,
 } from "../../Main/Icons";
-import dayjs from "dayjs";
 import CodeExample from "../../Main/CodeExample/CodeExample";
 
 interface BaseAlertProps {
@@ -66,7 +66,7 @@ const BaseAlert = ({ item }: BaseAlertProps) => {
           </tr>
           <tr>
             <td>Active at</td>
-            <td>{dayjs(item.activeAt).format("DD MMM YYYY HH:mm:ss")}</td>
+            <td>{formatEventTime(item.activeAt)}</td>
           </tr>
           {!!Object.keys(alertLabels).length && (
             <tr>
@@ -82,7 +82,7 @@ const BaseAlert = ({ item }: BaseAlertProps) => {
       </table>
       {!!Object.keys(item.annotations || {}).length && (
         <>
-          <span className="title">Annotations</span>
+          <span className="vm-alerts-title">Annotations</span>
           <table>
             <colgroup>
               <col className="vm-col-md"/>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseGroup/index.tsx

@@ -1,8 +1,7 @@
 import { useMemo } from "preact/compat";
 import "./style.scss";
 import { Group as APIGroup } from "../../../types";
-import dayjs from "dayjs";
-import { formatDuration } from "../helpers";
+import { formatDuration, formatEventTime } from "../helpers";
 import Badges, { BadgeColor } from "../Badges";
 
 interface BaseGroupProps {
@@ -48,12 +47,10 @@ const BaseGroup = ({ group }: BaseGroupProps) => {
               <td>{formatDuration(group.interval)}</td>
             </tr>
           )}
-          {!!group.lastEvaluation && (
-            <tr>
-              <td className="vm-col-md">Last evaluation</td>
-              <td>{dayjs(group.lastEvaluation).format("DD MMM YYYY HH:mm:ss")}</td>
-            </tr>
-          )}
+          <tr>
+            <td className="vm-col-md">Last evaluation</td>
+            <td>{formatEventTime(group.lastEvaluation)}</td>
+          </tr>
           {!!group.eval_offset && (
             <tr>
               <td className="vm-col-md">Eval offset</td>

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/index.tsx

@@ -6,8 +6,7 @@ import { SearchIcon, DetailsIcon } from "../../Main/Icons";
 import Button from "../../Main/Button/Button";
 import Alert from "../../Main/Alert/Alert";
 import Badges, { BadgeColor } from "../Badges";
-import dayjs from "dayjs";
-import { formatDuration } from "../helpers";
+import { formatDuration, formatEventTime } from "../helpers";
 import CodeExample from "../../Main/CodeExample/CodeExample";
 
 interface BaseRuleProps {
@@ -80,12 +79,10 @@ const BaseRule = ({ item }: BaseRuleProps) => {
               <td>{formatDuration(item.duration)}</td>
             </tr>
           )}
-          {!!item.lastEvaluation && (
-            <tr>
-              <td>Last evaluation</td>
-              <td>{dayjs(item.lastEvaluation).format("DD MMM YYYY HH:mm:ss")}</td>
-            </tr>
-          )}
+          <tr>
+            <td>Last evaluation</td>
+            <td>{formatEventTime(item.lastEvaluation)}</td>
+          </tr>
           {!!item.lastError && item.health !== "ok" && (
             <tr>
               <td>Last error</td>
@@ -108,7 +105,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
       </table>
       {!!Object.keys(item?.annotations || {}).length && (
         <>
-          <span className="title">Annotations</span>
+          <span className="vm-alerts-title">Annotations</span>
           <table>
             <colgroup>
               <col className="vm-col-md"/>
@@ -127,7 +124,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
       )}
       {!!item?.updates?.length && (
         <>
-          <span className="title">{`Last updates ${item.updates.length}/${item.max_updates_entries}`}</span>
+          <span className="vm-alerts-title">{`Last updates ${item.updates.length}/${item.max_updates_entries}`}</span>
           <table>
             <thead>
               <tr>
@@ -143,11 +140,11 @@ const BaseRule = ({ item }: BaseRuleProps) => {
                 <tr
                   key={update.at}
                 >
-                  <td>{dayjs(update.time).format("DD MMM YYYY HH:mm:ss")}</td>
+                  <td>{formatEventTime(update.time)}</td>
                   <td>{update.samples}</td>
                   <td>{update.series_fetched}</td>
                   <td>{formatDuration(update.duration / 1e9)}</td>
-                  <td>{dayjs(update.at).format("DD MMM YYYY HH:mm:ss")}</td>
+                  <td>{formatEventTime(update.at)}</td>
                 </tr>
               ))}
             </tbody>
@@ -156,7 +153,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
       )}
       {!!item?.alerts?.length && (
         <>
-          <span className="title">Alerts</span>
+          <span className="vm-alerts-title">Alerts</span>
           <table>
             <colgroup>
               <col className="vm-col-sm"/>
@@ -170,7 +167,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
                 <th>Active since</th>
                 <th>State</th>
                 <th>Value</th>
-                <th className="title">Labels</th>
+                <th className="vm-alerts-title">Labels</th>
                 <th></th>
               </tr>
             </thead>
@@ -180,9 +177,7 @@ const BaseRule = ({ item }: BaseRuleProps) => {
                   id={`alert-${alert.id}`}
                   key={alert.id}
                 >
-                  <td>
-                    {dayjs(alert.activeAt).format("DD MMM YYYY HH:mm:ss")}
-                  </td>
+                  <td>{formatEventTime(alert.activeAt)}</td>
                   <td>
                     <Badges
                       items={{ [alert.state]: { color: alert.state as BadgeColor } }}

app/vmui/packages/vmui/src/components/ExploreAlerts/BaseRule/style.scss

@@ -12,7 +12,7 @@
   display: flex;
   flex-direction: column;
 
-  .title {
+  .vm-alerts-title {
     font-weight: bold;
     text-align: center;
   }
@@ -48,11 +48,13 @@
       line-height: 30px;
       padding: 4px $padding-small;
       vertical-align: middle;
-      white-space: nowrap;
       text-align: left;
       overflow: hidden;
       text-overflow: ellipsis;
     }
+    th {
+      white-space: nowrap;
+    }
     td.align-center {
       text-align: center
     }

app/vmui/packages/vmui/src/components/ExploreAlerts/helpers.ts

@@ -13,3 +13,8 @@ export const formatDuration = (raw: number) => {
   }
   return duration.format(fmt.join(" "));
 };
+
+export const formatEventTime = (raw: string) => {
+  const t = dayjs(raw);
+  return t.year() <= 1 ? "Never" : t.format("DD MMM YYYY HH:mm:ss");
+}

app/vmui/packages/vmui/src/components/Main/Alert/style.scss

@@ -1,9 +1,7 @@
 @use "src/styles/variables" as *;
 
 .vm-alert {
-  z-index: 20;
-  position: sticky;
-  top: $padding-global;
+  position: relative;
   display: grid;
   grid-template-columns: 20px 1fr;
   align-items: center;

app/vmui/packages/vmui/src/components/Main/TextField/style.scss

@@ -1,6 +1,7 @@
 @use "src/styles/variables" as *;
 
 .vm-text-field {
+  position: relative;
   display: grid;
   margin: 6px 0;
   width: 100%;
@@ -156,10 +157,10 @@
   }
 
   &__icon-start {
-    left: $padding-small;
+    left: $padding-global;
   }
 
   &__icon-end {
-    right: $padding-small;
+    right: $padding-global;
   }
 }

app/vmui/packages/vmui/src/components/Table/helpers.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect } from "vitest";
 import { descendingComparator } from "./helpers";
-import { getNanoTimestamp } from "../../utils/time"; // используем реальную реализацию
+import { getNanoTimestamp } from "../../utils/time";
 
 describe("descendingComparator", () => {
   it("returns 0 for equal numbers", () => {

app/vmui/packages/vmui/src/components/Views/GraphView/GraphView.tsx

@@ -9,13 +9,12 @@ import {
   getLegendItem,
   getSeriesItemContext,
   normalizeData,
-  getLimitsYAxis,
   getMinMaxBuffer,
   getTimeSeries,
 } from "../../../utils/uplot";
 import { TimeParams, SeriesItem, LegendItemType } from "../../../types";
 import { AxisRange, YaxisState } from "../../../state/graph/reducer";
-import { getAvgFromArray, getMaxFromArray, getMinFromArray } from "../../../utils/math";
+import { getMathStats } from "../../../utils/math";
 import classNames from "classnames";
 import { useTimeState } from "../../../state/time/TimeStateContext";
 import HeatmapChart from "../../Chart/Heatmap/HeatmapChart/HeatmapChart";
@@ -27,6 +26,9 @@ import { ChartTooltipProps } from "../../Chart/ChartTooltip/ChartTooltip";
 import LegendAnomaly from "../../Chart/Line/LegendAnomaly/LegendAnomaly";
 import { groupByMultipleKeys } from "../../../utils/array";
 import { useGraphDispatch } from "../../../state/graph/GraphStateContext";
+import { sameTs } from "../../../utils/time";
+import { useLocation } from "react-router-dom";
+import router from "../../../router";
 
 export interface GraphViewProps {
   data?: MetricResult[];
@@ -45,6 +47,7 @@ export interface GraphViewProps {
   isAnomalyView?: boolean;
   isPredefinedPanel?: boolean;
   spanGaps?: boolean;
+  showAllPoints?: boolean;
 }
 
 const GraphView: FC<GraphViewProps> = ({
@@ -63,10 +66,16 @@ const GraphView: FC<GraphViewProps> = ({
   isHistogram,
   isAnomalyView,
   isPredefinedPanel,
-  spanGaps
+  spanGaps,
+  showAllPoints
 }) => {
+  const location = useLocation();
+  const isRawQuery = useMemo(() => location.pathname === router.rawQuery, [location.pathname]);
+
   const graphDispatch = useGraphDispatch();
 
+  const [containerRef, containerSize] = useElementSize();
+
   const { isMobile } = useDeviceDetect();
   const { timezone } = useTimeState();
   const currentStep = useMemo(() => customStep || period.step || "1s", [period.step, customStep]);
@@ -80,12 +89,16 @@ const GraphView: FC<GraphViewProps> = ({
   const [legendValue, setLegendValue] = useState<ChartTooltipProps | null>(null);
 
   const getSeriesItem = useMemo(() => {
-    return getSeriesItemContext(data, hideSeries, alias, isAnomalyView);
-  }, [data, hideSeries, alias, isAnomalyView]);
+    return getSeriesItemContext(data, hideSeries, alias, showAllPoints, isAnomalyView);
+  }, [data, hideSeries, alias, showAllPoints, isAnomalyView]);
 
-  const setLimitsYaxis = (values: { [key: string]: number[] }) => {
-    const limits = getLimitsYAxis(values, !isHistogram);
-    setYaxisLimits(limits);
+  const setLimitsYaxis = (minVal: number, maxVal: number) => {
+    let min = Number.isFinite(minVal) ? minVal : 0;
+    let max = Number.isFinite(maxVal) ? maxVal : 1;
+
+    if (min > max) [min, max] = [max, min];
+
+    setYaxisLimits({ "1": isHistogram ? [min, max] : getMinMaxBuffer(min, max) });
   };
 
   const onChangeLegend = (legend: LegendItemType, metaKey: boolean) => {
@@ -129,78 +142,103 @@ const GraphView: FC<GraphViewProps> = ({
   };
 
   useEffect(() => {
-    const tempTimes: number[] = [];
-    const tempValues: { [key: string]: number[] } = {};
-    const tempLegend: LegendItemType[] = [];
-    const tempSeries: uPlotSeries[] = [{}];
+    const dLen = data.length;
 
-    data?.forEach((d, i) => {
+    const tsAnchor = data?.[0]?.values?.[0]?.[0]
+    const tsSet = new Set<number>([])
+    const tempLegend = new Array<LegendItemType>(dLen);
+    const tempSeries = new Array<uPlotSeries>(dLen + 1);
+    tempSeries[0] = {};
+
+    let minVal = Infinity;
+    let maxVal = -Infinity;
+
+    for (let i = 0; i < dLen; i++) {
+      const d = data[i];
       const seriesItem = getSeriesItem(d, i);
+      tempSeries[i + 1] = seriesItem;
+      tempLegend[i] = getLegendItem(seriesItem, d.group);
 
-      tempSeries.push(seriesItem);
-      tempLegend.push(getLegendItem(seriesItem, d.group));
-      const tmpValues = tempValues[d.group] || [];
-      for (const v of d.values) {
-        tempTimes.push(v[0]);
-        tmpValues.push(promValueToNumber(v[1]));
-      }
-      tempValues[d.group] = tmpValues;
-    });
-
-    const timeSeries = getTimeSeries(tempTimes, currentStep, period);
-    const timeDataSeries = data.map(d => {
-      const results = [];
-      const values = d.values;
-      const length = values.length;
-      let j = 0;
-      for (const t of timeSeries) {
-        while (j < length && values[j][0] < t) j++;
-        let v = null;
-        if (j < length && values[j][0] == t) {
-          v = promValueToNumber(values[j][1]);
-          if (!Number.isFinite(v)) {
-            // Treat special values as nulls in order to satisfy uPlot.
-            // Otherwise it may draw unexpected graphs.
-            v = null;
-          }
+      const vals = d.values;
+      for (let j = 0, vLen = vals.length; j < vLen; j++) {
+        const v = vals[j];
+        if (isRawQuery) tsSet.add(v[0])
+        const num = promValueToNumber(v[1]);
+        if (Number.isFinite(num)) {
+          if (num < minVal) minVal = num;
+          if (num > maxVal) maxVal = num;
         }
-        results.push(v);
+      }
+    }
+
+    const dpr = window.devicePixelRatio || 1
+    const widthPx = containerSize.width || window.innerWidth || 4096;
+    const pixels = Math.max(1, Math.floor(widthPx * Math.max(1, dpr)));
+
+    const timeSeries = isRawQuery
+      ? Array.from(tsSet).sort((a,b) => a - b)
+      : getTimeSeries(currentStep, period, pixels, tsAnchor);
+
+    const timeDataSeries: (number | null)[][] = data.map(d => {
+      const tsLen = timeSeries.length;
+      const results = new Array<number | null>(tsLen);
+      const values = d.values;
+      const vLen = values.length;
+
+      let j = 0;
+      for (let k = 0; k < tsLen; k++) {
+        const t = timeSeries[k];
+        while (j < vLen && values[j][0] < t) j++;
+        let v: number | null = null;
+        if (j < vLen && sameTs(values[j][0], t)) {
+          const num = promValueToNumber(values[j][1]);
+          // Treat special values as nulls in order to satisfy uPlot.
+          // Otherwise it may draw unexpected graphs.
+          v = Number.isFinite(num) ? num : null;
+        }
+        results[k] = v;
       }
 
-      // stabilize float numbers
-      const resultAsNumber = results.filter(s => s !== null) as number[];
-      const avg = Math.abs(getAvgFromArray(resultAsNumber));
-      const range = getMinMaxBuffer(getMinFromArray(resultAsNumber), getMaxFromArray(resultAsNumber));
+      // // stabilize float numbers
+      const { min, max, avg: avgRaw } = getMathStats(results, { min: true, max: true, avg: true });
+      const avg = Math.abs(Number(avgRaw));
+      const range = getMinMaxBuffer(min, max);
       const rangeStep = Math.abs(range[1] - range[0]);
+      const needStabilize = (avg > rangeStep * 1e10) && !isAnomalyView;
 
-      return (avg > rangeStep * 1e10) && !isAnomalyView ? results.map(() => avg) : results;
+      return needStabilize ? results.fill(avg) : results;
     });
+
     timeDataSeries.unshift(timeSeries);
-    setLimitsYaxis(tempValues);
+
     const result = isHistogram ? prepareHistogramData(timeDataSeries) : timeDataSeries;
+    const legend = prepareAnomalyLegend(tempLegend);
+
+    setLimitsYaxis(minVal, maxVal);
     setDataChart(result as uPlotData);
     setSeries(tempSeries);
-    const legend = prepareAnomalyLegend(tempLegend);
     setLegend(legend);
-    if (isAnomalyView) {
-      setHideSeries(legend.map(s => s.label || "").slice(1));
-    }
-  }, [data, timezone, isHistogram, currentStep]);
+    isAnomalyView && setHideSeries(legend.map(s => s.label || "").slice(1));
+  }, [data, timezone, isHistogram, currentStep, isRawQuery]);
 
   useEffect(() => {
-    const tempLegend: LegendItemType[] = [];
-    const tempSeries: uPlotSeries[] = [{}];
-    data?.forEach((d, i) => {
+    const dLen = data.length;
+
+    const tempLegend = new Array<LegendItemType>(dLen);
+    const tempSeries = new Array<uPlotSeries>(dLen + 1);
+    tempSeries[0] = {};
+
+    for (let i = 0; i < dLen; i++) {
+      const d = data[i];
       const seriesItem = getSeriesItem(d, i);
-      tempSeries.push(seriesItem);
-      tempLegend.push(getLegendItem(seriesItem, d.group));
-    });
+      tempSeries[i + 1] = seriesItem;
+      tempLegend[i] = getLegendItem(seriesItem, d.group);
+    }
+
     setSeries(tempSeries);
     setLegend(prepareAnomalyLegend(tempLegend));
   }, [hideSeries]);
 
-  const [containerRef, containerSize] = useElementSize();
-
   const hasTimeData = dataChart[0]?.length > 0;
 
   useEffect(() => {
@@ -243,6 +281,7 @@ const GraphView: FC<GraphViewProps> = ({
           height={height}
           isAnomalyView={isAnomalyView}
           spanGaps={spanGaps}
+          showAllPoints={showAllPoints}
         />
       )}
       {isHistogram && (

app/vmui/packages/vmui/src/hooks/useFetchQuery.ts

@@ -41,7 +41,6 @@ interface FetchQueryReturn {
 
 interface FetchDataParams {
   fetchUrl: string[],
-  fetchQueue: AbortController[],
   displayType: DisplayType,
   query: string[],
   stateSeriesLimits: SeriesLimits,
@@ -81,21 +80,25 @@ export const useFetchQuery = ({
 
   const fetchData = async ({
     fetchUrl,
-    fetchQueue,
     displayType,
     query,
     stateSeriesLimits,
     showAllSeries,
     hideQuery,
   }: FetchDataParams) => {
+
     const controller = new AbortController();
-    setFetchQueue([...fetchQueue, controller]);
+    setFetchQueue(prev => [...prev, controller]);
+
     try {
       const isDisplayChart = displayType === DisplayType.chart;
       const defaultLimit = showAllSeries ? Infinity : (+stateSeriesLimits[displayType] || Infinity);
       let seriesLimit = defaultLimit;
       const tempData: MetricBase[] = [];
       const tempTraces: Trace[] = [];
+      const tempStats: QueryStats[] = [];
+      const tempErrors: string[] = [];
+
       let counter = 1;
       let totalLength = 0;
       let isHistogramResult = false;
@@ -104,8 +107,8 @@ export const useFetchQuery = ({
 
         const isHideQuery = hideQuery?.includes(counter - 1);
         if (isHideQuery) {
-          setQueryErrors(prev => [...prev, ""]);
-          setQueryStats(prev => [...prev, {}]);
+          tempErrors.push("");
+          tempStats.push({});
           counter++;
           continue;
         }
@@ -119,12 +122,12 @@ export const useFetchQuery = ({
         const resp = await response.json();
 
         if (response.ok) {
-          setQueryStats(prev => [...prev, {
+          tempStats.push({
             ...resp?.stats,
             isPartial: resp?.isPartial,
             resultLength: resp.data.result.length,
-          }]);
-          setQueryErrors(prev => [...prev, ""]);
+          });
+          tempErrors.push("");
 
           if (resp.trace) {
             const trace = new Trace(resp.trace, query[counter - 1]);
@@ -134,7 +137,7 @@ export const useFetchQuery = ({
           const preventChangeType = !!getQueryStringValue("display_mode", null);
           isHistogramResult = !APP_TYPE_ANOMALY && isDisplayChart && !preventChangeType && isHistogramData(resp.data.result);
           seriesLimit = isHistogramResult ? Infinity : defaultLimit;
-          const freeTempSize = seriesLimit - tempData.length;
+          const freeTempSize = Math.max(0, seriesLimit - tempData.length);
           resp.data.result.slice(0, freeTempSize).forEach((d: MetricBase) => {
             d.group = counter;
             tempData.push(d);
@@ -146,14 +149,20 @@ export const useFetchQuery = ({
           const errorType = resp.errorType || ErrorTypes.unknownType;
           const errorMessage = resp?.error || resp?.message || "see console for more details";
           const error = [errorType, errorMessage].join(",\r\n");
-          setQueryErrors(prev => [...prev, `${error}`]);
+          tempErrors.push(error);
           console.error(`Fetch query error: ${errorType}`, resp);
         }
         counter++;
       }
+      setQueryErrors(tempErrors);
+      setQueryStats(tempStats);
+
+      const shownSeries = tempData.length;
+      setWarning(shownSeries < totalLength
+        ? `Showing ${shownSeries} series out of ${totalLength} series due to performance reasons. Please narrow down the query, so it returns fewer series`
+        : ""
+      );
 
-      const limitText = `Showing ${tempData.length} series out of ${totalLength} series due to performance reasons. Please narrow down the query, so it returns fewer series`;
-      setWarning(totalLength > seriesLimit ? limitText : "");
       isDisplayChart ? setGraphData(tempData as MetricResult[]) : setLiveData(tempData as InstantMetricResult[]);
       setTraces(tempTraces);
       setIsHistogram(prev => totalLength ? isHistogramResult : prev);
@@ -177,30 +186,20 @@ export const useFetchQuery = ({
   const throttledFetchData = useCallback(debounce(fetchData, 300), []);
 
   const fetchUrl = useMemo(() => {
-    setError("");
-    setQueryErrors([]);
-    setQueryStats([]);
-    const expr = predefinedQuery ?? query;
+    const expr = (predefinedQuery ?? query).filter(Boolean);
     const displayChart = (display || displayType) === DisplayType.chart;
-    if (!period) return;
-    if (!serverUrl) {
-      setError(ErrorTypes.emptyServer);
-    } else if (expr.every(q => !q.trim())) {
-      setQueryErrors(expr.map(() => ErrorTypes.validQuery));
-    } else if (isValidHttpUrl(serverUrl)) {
-      const updatedPeriod = { ...period };
-      updatedPeriod.step = customStep;
-      return expr.map(q => displayChart
-        ? getQueryRangeUrl(serverUrl, q, updatedPeriod, nocache, isTracingEnabled)
-        : getQueryUrl(serverUrl, q, updatedPeriod, nocache, isTracingEnabled));
-    } else {
-      setError(ErrorTypes.validServer);
-    }
-  },
-  [serverUrl, period, displayType, customStep, hideQuery]);
+
+    if (!period || !serverUrl || !isValidHttpUrl(serverUrl) || !expr.length) return;
+
+    const updatedPeriod = { ...period, step: customStep };
+    return expr.map(q => displayChart
+      ? getQueryRangeUrl(serverUrl, q, updatedPeriod, nocache, isTracingEnabled)
+      : getQueryUrl(serverUrl, q, updatedPeriod, nocache, isTracingEnabled));
+  }, [serverUrl, period, displayType, customStep, nocache, isTracingEnabled, display, predefinedQuery, query]);
 
   const abortFetch = useCallback(() => {
-    fetchQueue.map(f => f.abort());
+    fetchQueue.forEach(f => f.abort());
+
     setFetchQueue([]);
     setGraphData([]);
     setLiveData([]);
@@ -215,7 +214,6 @@ export const useFetchQuery = ({
     const expr = predefinedQuery ?? query;
     throttledFetchData({
       fetchUrl,
-      fetchQueue,
       displayType: display || displayType,
       query: expr,
       stateSeriesLimits,
@@ -229,13 +227,26 @@ export const useFetchQuery = ({
     const fetchPast = fetchQueue.slice(0, -1);
     if (!fetchPast.length) return;
     fetchPast.map(f => f.abort());
-    setFetchQueue(fetchQueue.filter(f => !f.signal.aborted));
+
+    setFetchQueue(prev => prev.filter(f => !f.signal.aborted));
   }, [fetchQueue]);
 
   useEffect(() => {
     if (defaultStep === customStep) setGraphData([]);
   }, [isHistogram]);
 
+  useEffect(() => {
+    setError("");
+    setQueryErrors([]);
+    setQueryStats([]);
+
+    const expr = predefinedQuery ?? query;
+    if (!period) return;
+    if (!serverUrl) { setError(ErrorTypes.emptyServer); return; }
+    if (!isValidHttpUrl(serverUrl)) { setError(ErrorTypes.validServer); return; }
+    if (expr.every(q => !q.trim())) { setQueryErrors(expr.map(() => ErrorTypes.validQuery)); }
+  }, [serverUrl, period, displayType, customStep, nocache, isTracingEnabled, display, predefinedQuery, query]);
+
   return {
     fetchUrl,
     isLoading,

app/vmui/packages/vmui/src/pages/CustomPanel/CustomPanelTabs/GraphTab.tsx

@@ -1,4 +1,4 @@
-import { FC } from "react";
+import { FC, createPortal, useCallback, RefObject } from "preact/compat";
 import GraphView from "../../../components/Views/GraphView/GraphView";
 import GraphTips from "../../../components/Chart/GraphTips/GraphTips";
 import GraphSettings from "../../../components/Configurators/GraphSettings/GraphSettings";
@@ -8,40 +8,43 @@ import { useGraphDispatch, useGraphState } from "../../../state/graph/GraphState
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import { useQueryState } from "../../../state/query/QueryStateContext";
 import { MetricResult } from "../../../api/types";
-import { createPortal } from "preact/compat";
 
 type Props = {
   isHistogram: boolean;
   graphData: MetricResult[];
-  controlsRef: React.RefObject<HTMLDivElement>;
+  controlsRef: RefObject<HTMLDivElement>;
   isAnomalyView?: boolean;
 }
 
 const GraphTab: FC<Props> = ({ isHistogram, graphData, controlsRef, isAnomalyView }) => {
   const { isMobile } = useDeviceDetect();
 
-  const { customStep, yaxis, spanGaps } = useGraphState();
+  const { customStep, yaxis, spanGaps, showAllPoints } = useGraphState();
   const { period } = useTimeState();
   const { query } = useQueryState();
 
   const timeDispatch = useTimeDispatch();
   const graphDispatch = useGraphDispatch();
 
-  const setYaxisLimits = (limits: AxisRange) => {
+  const setYaxisLimits = useCallback((limits: AxisRange) => {
     graphDispatch({ type: "SET_YAXIS_LIMITS", payload: limits });
-  };
+  }, [graphDispatch]);
 
-  const toggleEnableLimits = () => {
+  const toggleEnableLimits = useCallback(() => {
     graphDispatch({ type: "TOGGLE_ENABLE_YAXIS_LIMITS" });
-  };
+  }, [graphDispatch]);
 
-  const setSpanGaps = (value: boolean) => {
+  const setSpanGaps = useCallback((value: boolean) => {
     graphDispatch({ type: "SET_SPAN_GAPS", payload: value });
-  };
+  }, [graphDispatch]);
 
-  const setPeriod = ({ from, to }: {from: Date, to: Date}) => {
+  const setPeriod = useCallback(({ from, to }: { from: Date; to: Date }) => {
     timeDispatch({ type: "SET_PERIOD", payload: { from, to } });
-  };
+  }, [timeDispatch]);
+
+  const setShowPoints = useCallback((value: boolean) => {
+    graphDispatch({ type: "SET_SHOW_POINTS", payload: value });
+  }, [graphDispatch]);
 
   const controls = (
     <div className="vm-custom-panel-body-header__graph-controls">
@@ -53,6 +56,7 @@ const GraphTab: FC<Props> = ({ isHistogram, graphData, controlsRef, isAnomalyVie
         setYaxisLimits={setYaxisLimits}
         toggleEnableLimits={toggleEnableLimits}
         spanGaps={{ value: spanGaps, onChange: setSpanGaps }}
+        showAllPoints={{ value: showAllPoints, onChange: setShowPoints }}
       />
     </div>
   );
@@ -72,6 +76,7 @@ const GraphTab: FC<Props> = ({ isHistogram, graphData, controlsRef, isAnomalyVie
         isHistogram={isHistogram}
         isAnomalyView={isAnomalyView}
         spanGaps={spanGaps}
+        showAllPoints={showAllPoints}
       />
     </>
   );

app/vmui/packages/vmui/src/pages/CustomPanel/QueryConfigurator/QueryConfigurator.tsx

@@ -31,7 +31,7 @@ import QueryEditorAutocomplete from "../../../components/Configurators/QueryEdit
 import { getUpdatedHistory } from "../../../components/QueryHistory/utils";
 
 export interface QueryConfiguratorProps {
-  queryErrors: string[];
+  queryErrors?: string[];
   setQueryErrors: Dispatch<SetStateAction<string[]>>;
   setHideError: Dispatch<SetStateAction<boolean>>;
   stats: QueryStats[];
@@ -217,7 +217,7 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({
             value={stateQuery[i]}
             autocomplete={!hideButtons?.autocomplete && (autocomplete || autocompleteQuick)}
             autocompleteEl={QueryEditorAutocomplete}
-            error={queryErrors[i]}
+            error={queryErrors && queryErrors[i]}
             stats={stats[i]}
             onArrowUp={createHandlerArrow(-1, i)}
             onArrowDown={createHandlerArrow(1, i)}

app/vmui/packages/vmui/src/pages/CustomPanel/index.tsx

@@ -83,17 +83,17 @@ const CustomPanel: FC = () => {
   const showInstantQueryTip = !liveData?.length && (displayType !== DisplayType.chart);
   const showError = !hideError && error;
 
-  const handleHideQuery = (queries: number[]) => {
+  const handleHideQuery = useCallback((queries: number[]) => {
     setHideQuery(queries);
-  };
+  }, []);
 
-  const handleRunQuery = () => {
+  const handleRunQuery = useCallback(() => {
     setHideError(false);
-  };
+  }, []);
 
   useEffect(() => {
     graphDispatch({ type: "SET_IS_HISTOGRAM", payload: isHistogram });
-  }, [graphData]);
+  }, [isHistogram]);
 
   return (
     <div
@@ -103,7 +103,7 @@ const CustomPanel: FC = () => {
       })}
     >
       <QueryConfigurator
-        queryErrors={!hideError ? queryErrors : []}
+        queryErrors={!hideError ? queryErrors : undefined}
         setQueryErrors={setQueryErrors}
         setHideError={setHideError}
         stats={queryStats}

app/vmui/packages/vmui/src/pages/PredefinedPanels/PredefinedPanel/PredefinedPanel.tsx

@@ -35,6 +35,7 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({
   const containerRef = useRef<HTMLDivElement>(null);
   const [visible, setVisible] = useState(false);
   const [spanGaps, setSpanGaps] = useState(false);
+  const [showAllPoints, setShowPoints] = useState(false);
   const [yaxis, setYaxis] = useState<YaxisState>({
     limits: {
       enable: false,
@@ -124,6 +125,7 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({
         setYaxisLimits={setYaxisLimits}
         toggleEnableLimits={toggleEnableLimits}
         spanGaps={{ value: spanGaps, onChange: setSpanGaps }}
+        showAllPoints={{ value: showAllPoints, onChange: setShowPoints }}
       />
     </div>
     <div className="vm-predefined-panel-body">
@@ -145,6 +147,7 @@ const PredefinedPanel: FC<PredefinedPanelsProps> = ({
         fullWidth={false}
         height={isMobile ? window.innerHeight * 0.5 : 500}
         spanGaps={spanGaps}
+        showAllPoints={showAllPoints}
       />
       }
     </div>

app/vmui/packages/vmui/src/pages/QueryAnalyzer/QueryAnalyzerView/QueryAnalyzerView.tsx

@@ -53,7 +53,7 @@ const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
   }, [data]);
   const [displayType, setDisplayType] = useState(tabs[0].value);
 
-  const { yaxis, spanGaps } = useGraphState();
+  const { yaxis, spanGaps, showAllPoints } = useGraphState();
   const graphDispatch = useGraphDispatch();
 
   const setYaxisLimits = (limits: AxisRange) => {
@@ -68,6 +68,10 @@ const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
     graphDispatch({ type: "SET_SPAN_GAPS", payload: value });
   };
 
+  const setShowPoints = (value: boolean) => {
+    graphDispatch({ type: "SET_SHOW_POINTS", payload: value });
+  };
+
   const handleChangeDisplayType = (newValue: string) => {
     setDisplayType(newValue as DisplayType);
   };
@@ -154,6 +158,7 @@ const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
                 setYaxisLimits={setYaxisLimits}
                 toggleEnableLimits={toggleEnableLimits}
                 spanGaps={{ value: spanGaps, onChange: setSpanGaps }}
+                showAllPoints={{ value: showAllPoints, onChange: setShowPoints }}
               />
             )}
             {displayType === "table" && (
@@ -179,6 +184,7 @@ const QueryAnalyzerView: FC<Props> = ({ data, period }) => {
             height={isMobile ? window.innerHeight * 0.5 : 500}
             isHistogram={isHistogram}
             spanGaps={spanGaps}
+            showAllPoints={showAllPoints}
           />
         )}
         {liveData && (displayType === "code") && (

app/vmui/packages/vmui/src/pages/TopQueries/index.tsx

@@ -3,7 +3,7 @@ import { useFetchTopQueries } from "./hooks/useFetchTopQueries";
 import Spinner from "../../components/Main/Spinner/Spinner";
 import TopQueryPanel from "./TopQueryPanel/TopQueryPanel";
 import { formatPrettyNumber } from "../../utils/uplot";
-import { isSupportedDuration } from "../../utils/time";
+import { parseSupportedDuration } from "../../utils/time";
 import dayjs from "dayjs";
 import { TopQueryStats } from "../../types";
 import Button from "../../components/Main/Button/Button";
@@ -29,7 +29,7 @@ const TopQueries: FC = () => {
   const maxLifetimeValid = useMemo(() => {
     const durItems = maxLifetime.trim().split(" ");
     const durObject = durItems.reduce((prev, curr) => {
-      const dur = isSupportedDuration(curr);
+      const dur = parseSupportedDuration(curr);
       return dur ? { ...prev, ...dur } : { ...prev };
     }, {});
     const delta = dayjs.duration(durObject).asMilliseconds();

app/vmui/packages/vmui/src/state/graph/reducer.ts

@@ -1,4 +1,5 @@
 import { getQueryStringValue } from "../../utils/query-string";
+import { getFromStorage, saveToStorage } from "../../utils/storage";
 
 export interface AxisRange {
   [key: string]: [number, number]
@@ -18,6 +19,7 @@ export interface GraphState {
   isEmptyHistogram: boolean
   /** when true, null data values will not cause line breaks */
   spanGaps: boolean
+  showAllPoints: boolean
   openSettings: boolean
 }
 
@@ -28,6 +30,7 @@ export type GraphAction =
   | { type: "SET_IS_HISTOGRAM", payload: boolean }
   | { type: "SET_IS_EMPTY_HISTOGRAM", payload: boolean }
   | { type: "SET_SPAN_GAPS", payload: boolean }
+  | { type: "SET_SHOW_POINTS", payload: boolean }
   | { type: "SET_OPEN_SETTINGS", payload: boolean }
 
 export const initialGraphState: GraphState = {
@@ -38,6 +41,7 @@ export const initialGraphState: GraphState = {
   isHistogram: false,
   isEmptyHistogram: false,
   spanGaps: false,
+  showAllPoints: Boolean(getFromStorage("POINTS_SHOW_ALL")),
   openSettings: false
 };
 
@@ -85,6 +89,12 @@ export function reducer(state: GraphState, action: GraphAction): GraphState {
         ...state,
         spanGaps: action.payload
       };
+    case "SET_SHOW_POINTS":
+      saveToStorage("POINTS_SHOW_ALL", action.payload);
+      return {
+        ...state,
+        showAllPoints: action.payload
+      };
     case "SET_OPEN_SETTINGS":
       return {
         ...state,

app/vmui/packages/vmui/src/types/index.ts

@@ -192,7 +192,7 @@ export interface Group {
   rules: Rule[];
   interval: number;
   limit: number;
-  lastEvaluation: number;
+  lastEvaluation: string;
   evaluationTime: number;
   type: string;
   id: string;
@@ -216,7 +216,7 @@ export interface Rule {
   annotations: Record<string, string>;
   alerts: Alert[];
   health: string;
-  lastEvaluation: number;
+  lastEvaluation: string;
   lastError: string;
   evaluationTime: number;
   type: string;
@@ -247,7 +247,7 @@ export interface Alert {
   expression: string;
   labels: Record<string, string>;
   annotations: Record<string, string>;
-  activeAt: number;
+  activeAt: string;
   id: string;
   source: string;
   restored: boolean;

app/vmui/packages/vmui/src/types/uplot.ts

@@ -14,7 +14,6 @@ export interface SeriesItemStatsFormatted {
     min: string,
     max: string,
     median: string,
-    last: string
 }
 
 export interface SeriesItem extends Series {

app/vmui/packages/vmui/src/utils/math.test.ts

@@ -0,0 +1,144 @@
+import { describe, it, expect } from "vitest";
+import { getMathStats } from "./math";
+
+const finite = (x: number) => Number.isFinite(x);
+
+const expectedMin = (arr: number[]): number | null => {
+  const vals = arr.filter(finite);
+  return vals.length ? Math.min(...vals) : null;
+};
+
+const expectedMax = (arr: number[]): number | null => {
+  const vals = arr.filter(finite);
+  return vals.length ? Math.max(...vals) : null;
+};
+
+const expectedAvg = (arr: number[]): number | null => {
+  const vals = arr.filter(finite);
+  if (!vals.length) return null;
+  const sum = vals.reduce((s, x) => s + x, 0);
+  return sum / vals.length;
+};
+
+const expectedMedian = (arr: number[]): number | null => {
+  const vals = arr.filter(finite).slice().sort((a, b) => a - b);
+  const m = vals.length;
+  if (!m) return null;
+  const k = m >> 1;
+  return m & 1 ? vals[k] : (vals[k - 1] + vals[k]) / 2;
+};
+
+describe("getMathStats — basics", () => {
+  it("returns all nulls when no options are requested", () => {
+    const a = [1, 2, 3];
+    const r = getMathStats(a, {});
+    expect(r).toEqual({ min: null, max: null, median: null, avg: null });
+  });
+
+  it("does not mutate the input array", () => {
+    const a = [7, 3, 10, -2, 5];
+    const before = a.slice();
+    getMathStats(a, { min: true, max: true, median: true, avg: true });
+    expect(a).toEqual(before);
+  });
+});
+
+describe("getMathStats — individual metrics", () => {
+  const arrays = [
+    [7, 3, 10, -2, 5],
+    [0, -0, 0, 0.5, 0.25, -1.25],
+    [100],
+    [NaN, Infinity, -Infinity, 42],
+    [NaN, Infinity, -Infinity],
+    [],
+  ];
+
+  it("min only", () => {
+    for (const a of arrays) {
+      const r = getMathStats(a, { min: true });
+      expect(r.min).toBe(expectedMin(a));
+      expect(r.max).toBeNull();
+      expect(r.avg).toBeNull();
+      expect(r.median).toBeNull();
+    }
+  });
+
+  it("max only", () => {
+    for (const a of arrays) {
+      const r = getMathStats(a, { max: true });
+      expect(r.max).toBe(expectedMax(a));
+      expect(r.min).toBeNull();
+      expect(r.avg).toBeNull();
+      expect(r.median).toBeNull();
+    }
+  });
+
+  it("average only", () => {
+    for (const a of arrays) {
+      const r = getMathStats(a, { avg: true });
+      const exp = expectedAvg(a);
+      if (exp == null) {
+        expect(r.avg).toBeNull();
+      } else {
+        expect(r.avg!).toBeCloseTo(exp, 12);
+      }
+      expect(r.min).toBeNull();
+      expect(r.max).toBeNull();
+      expect(r.median).toBeNull();
+    }
+  });
+
+  it("median only (odd/even, with non-finite filtered)", () => {
+    const cases = [
+      [7, 3, 10, -2, 5],              // odd
+      [7, 3, 10, -2, 5, 8],           // even
+      [NaN, Infinity, 3, 3, 3, 1],    // duplicates + non-finite
+      [100],                          // single
+      [],                             // empty
+      [NaN, Infinity, -Infinity],     // only non-finite
+    ];
+    for (const a of cases) {
+      const r = getMathStats(a, { median: true });
+      const exp = expectedMedian(a);
+      if (exp == null) {
+        expect(r.median).toBeNull();
+      } else {
+        expect(r.median!).toBeCloseTo(exp, 12);
+      }
+      expect(r.min).toBeNull();
+      expect(r.max).toBeNull();
+      expect(r.avg).toBeNull();
+    }
+  });
+});
+
+describe("getMathStats — combined metrics", () => {
+  it("all metrics together", () => {
+    const a = [7, 3, 10, -2, 5, NaN, Infinity];
+    const r = getMathStats(a, { min: true, max: true, avg: true, median: true });
+
+    // expected values computed independently
+    const expMin = expectedMin(a);
+    const expMax = expectedMax(a);
+    const expAvg = expectedAvg(a);
+    const expMedian = expectedMedian(a);
+
+    expect(r.min).toBe(expMin);
+    expect(r.max).toBe(expMax);
+    if (expAvg == null) expect(r.avg).toBeNull();
+    else expect(r.avg!).toBeCloseTo(expAvg, 12);
+    if (expMedian == null) expect(r.median).toBeNull();
+    else expect(r.median!).toBeCloseTo(expMedian, 12);
+  });
+
+  it("does not return median when not requested", () => {
+    const a = [9, 1, 5, 3, 7, 2, 11, 4];
+    const r = getMathStats(a, { min: true, max: true, avg: true /* median: false */ });
+    expect(r.min).toBe(expectedMin(a));
+    expect(r.max).toBe(expectedMax(a));
+    const expAvg = expectedAvg(a)!;
+    expect(r.avg!).toBeCloseTo(expAvg, 12);
+    // IMPORTANT: median should be null if not requested
+    expect(r.median).toBeNull();
+  });
+});

app/vmui/packages/vmui/src/utils/math.ts

@@ -1,71 +1,86 @@
-export const getMaxFromArray = (a: number[]) => {
-  let len = a.length;
-  let max = -Infinity;
-  while (len--) {
-    const v = a[len];
-    if (Number.isFinite(v) && v > max) {
-      max = v;
-    }
-  }
-  return Number.isFinite(max) ? max : null;
+import quickselect from "./quickselect";
+
+export const roundToThousandths = (num: number): number => Math.round(num*1000)/1000;
+
+type MathStatsOptions = {
+  min?: boolean;
+  max?: boolean;
+  median?: boolean;
+  avg?: boolean;
 };
 
-export const getMinFromArray = (a: number[]) => {
-  let len = a.length;
-  let min = Infinity;
-  while (len--) {
-    const v = a[len];
-    if (Number.isFinite(v) && v < min) {
-      min = v;
-    }
-  }
-  return Number.isFinite(min) ? min : null;
+type MathStatsResult = {
+  min: number | null;
+  max: number | null;
+  median: number | null;
+  avg: number | null;
 };
 
-export const getAvgFromArray = (a: number[]) => {
-  let mean = a[0];
-  let n = 1;
-  for (let i = 1; i < a.length; i++) {
-    const v = a[i];
-    if (Number.isFinite(v)) {
-      mean = mean * (n-1)/n + v / n;
-      n++;
-    }
+/**
+ * Returns median of finite numbers in `vals`.
+ * MUTATES `vals` in place (uses quickselect).
+ */
+const medianFromFiniteInPlace = (vals: number[]): number | null => {
+  const m = vals.length;
+  if (m === 0) return null;
+
+  const k = m >> 1;
+  quickselect(vals, k); // place upper median at vals[k]
+  const upper = vals[k];
+
+  if (m & 1) return upper; // odd length
+
+  // even length: take max of the left half [0..k-1]
+  let lower = -Infinity;
+  for (let i = 0; i < k; i++) {
+    const v = vals[i];
+    if (v > lower) lower = v;
   }
-  return mean;
+  return (lower + upper) / 2;
 };
 
-export const getMedianFromArray = (a: number[]) => {
-  let len = a.length;
-  const aCopy = [];
-  while (len--) {
-    const v = a[len];
-    if (Number.isFinite(v)) {
-      aCopy.push(v);
-    }
-  }
-  aCopy.sort();
-  return aCopy[aCopy.length>>1];
-};
+export const getMathStats = (
+  a: (number | null)[],
+  ops: MathStatsOptions
+): MathStatsResult => {
+  const needMin = !!ops.min;
+  const needMax = !!ops.max;
+  const needAvg = !!ops.avg;
+  const needMedian = !!ops.median;
 
-export const getLastFromArray = (a: number[]) => {
-  let len = a.length;
-  while (len--) {
-    const v = a[len];
-    if (Number.isFinite(v)) {
-      return v;
-    }
+  if (!needMin && !needMax && !needAvg && !needMedian) {
+    return { min: null, max: null, median: null, avg: null };
   }
-};
 
-export const formatNumberShort = (value: number) => {
-  if (value >= 1_000_000_000) {
-    return (value / 1_000_000_000).toFixed(1).replace(/\.0$/, "") + "B";
-  } else if (value >= 1_000_000) {
-    return (value / 1_000_000).toFixed(1).replace(/\.0$/, "") + "M";
-  } else if (value >= 1_000) {
-    return (value / 1_000).toFixed(1).replace(/\.0$/, "") + "K";
-  } else {
-    return value.toString();
+  // min & max
+  let minVal = Infinity;
+  let maxVal = -Infinity;
+
+  // average
+  let avgVal = 0;
+  let avgCount = 0;
+
+  // collect finite values for median
+  const vals: number[] = [];
+
+  for (const v of a) {
+    if (v == null || !Number.isFinite(v)) continue;
+
+    if (needMin && v < minVal) minVal = v;
+    if (needMax && v > maxVal) maxVal = v;
+
+    if (needAvg) {
+      avgCount++;
+      avgVal += (v - avgVal) / avgCount;
+    }
+
+    if (needMedian) vals.push(v);
   }
+
+  return {
+    min: Number.isFinite(minVal) ? minVal : null,
+    max: Number.isFinite(maxVal) ? maxVal : null,
+    avg: avgCount ? avgVal : null,
+    median: (vals && needMedian) ? medianFromFiniteInPlace(vals) : null,
+  };
 };

app/vmui/packages/vmui/src/utils/quickselect.test.ts

@@ -0,0 +1,207 @@
+import { describe, it, expect } from "vitest";
+import quickselect from "./quickselect";
+
+// Helper: verifies partition property around k using given comparator
+function checkPartition<T>(
+  arr: T[],
+  k: number,
+  compare: (a: T, b: T) => number
+) {
+  const pivot = arr[k];
+  for (let i = 0; i < arr.length; i++) {
+    if (i < k) {
+      expect(compare(arr[i], pivot)).toBeLessThanOrEqual(0);
+    } else if (i > k) {
+      expect(compare(arr[i], pivot)).toBeGreaterThanOrEqual(0);
+    }
+  }
+}
+
+const numCmp = (a: number, b: number) => (a < b ? -1 : a > b ? 1 : 0);
+
+describe("quickselect (numbers)", () => {
+  it("finds k-th smallest on small array", () => {
+    const arr = [7, 2, 9, 1, 5, 3];
+    const k = 3;
+    const orig = arr.slice();
+    quickselect(arr, k);
+    const sorted = orig.slice().sort(numCmp);
+    expect(arr[k]).toBe(sorted[k]);
+    checkPartition(arr, k, numCmp);
+  });
+
+  it("works for k = 0 (minimum)", () => {
+    const arr = [10, -1, 4, 4, 2];
+    const orig = arr.slice();
+    quickselect(arr, 0);
+    expect(arr[0]).toBe(Math.min(...orig));
+    checkPartition(arr, 0, numCmp);
+  });
+
+  it("works for k = n-1 (maximum)", () => {
+    const arr = [10, -1, 4, 4, 2];
+    const k = arr.length - 1;
+    const orig = arr.slice();
+    quickselect(arr, k);
+    expect(arr[k]).toBe(Math.max(...orig));
+    checkPartition(arr, k, numCmp);
+  });
+
+  it("handles duplicates correctly", () => {
+    const arr = [5, 1, 3, 3, 3, 2, 5, 4];
+    const k = 4;
+    const orig = arr.slice();
+    quickselect(arr, k);
+    const sorted = orig.slice().sort(numCmp);
+    expect(arr[k]).toBe(sorted[k]);
+    checkPartition(arr, k, numCmp);
+  });
+
+  it("handles negative numbers and mixed values", () => {
+    const arr = [0, -100, 50, -3, 7, 7, 2, -1];
+    const k = 2;
+    const orig = arr.slice();
+    quickselect(arr, k);
+    const sorted = orig.slice().sort(numCmp);
+    expect(arr[k]).toBe(sorted[k]);
+    checkPartition(arr, k, numCmp);
+  });
+
+  it("matches fully sorted array at many random ks", () => {
+    for (let t = 0; t < 25; t++) {
+      const n = 50;
+      const arr = Array.from({ length: n }, () => Math.floor(Math.random() * 1000) - 500);
+      const k = Math.floor(Math.random() * n);
+      const orig = arr.slice();
+      quickselect(arr, k);
+      const sorted = orig.slice().sort(numCmp);
+      expect(arr[k]).toBe(sorted[k]);
+      checkPartition(arr, k, numCmp);
+    }
+  });
+
+  it("handles already sorted and reverse-sorted", () => {
+    const asc = [1, 2, 3, 4, 5, 6, 7, 8, 9];
+    const k1 = 4;
+    const origAsc = asc.slice();
+    quickselect(asc, k1);
+    expect(asc[k1]).toBe(origAsc[k1]);
+    checkPartition(asc, k1, numCmp);
+
+    const desc = [9, 8, 7, 6, 5, 4, 3, 2, 1];
+    const k2 = 3;
+    const origDesc = desc.slice();
+    quickselect(desc, k2);
+    const sortedDesc = origDesc.slice().sort(numCmp);
+    expect(desc[k2]).toBe(sortedDesc[k2]);
+    checkPartition(desc, k2, numCmp);
+  });
+
+  it("handles all-equal values", () => {
+    const arr = Array(100).fill(42);
+    const k = 50;
+    quickselect(arr, k);
+    expect(arr[k]).toBe(42);
+    checkPartition(arr, k, numCmp);
+  });
+});
+
+describe("quickselect (custom comparator)", () => {
+  it("works with strings (lexicographic)", () => {
+    const arr = ["pear", "apple", "banana", "apricot", "kiwi"];
+    const k = 2;
+    const cmp = (a: string, b: string) => a.localeCompare(b);
+    const orig = arr.slice();
+    quickselect(arr, k, 0, arr.length - 1, cmp);
+    const sorted = orig.slice().sort(cmp);
+    expect(arr[k]).toBe(sorted[k]);
+    checkPartition(arr, k, cmp);
+  });
+
+  it("works with objects via custom comparator (by score asc)", () => {
+    type Item = { id: string; score: number };
+    const items: Item[] = [
+      { id: "a", score: 10 },
+      { id: "b", score: -3 },
+      { id: "c", score: 7 },
+      { id: "d", score: 7 },
+      { id: "e", score: 0 },
+    ];
+    const k = 3;
+    const cmp = (x: Item, y: Item) => (x.score < y.score ? -1 : x.score > y.score ? 1 : 0);
+
+    const orig = items.slice();
+    quickselect(items, k, 0, items.length - 1, cmp);
+
+    const sorted = orig.slice().sort(cmp);
+    expect(items[k].score).toBe(sorted[k].score);
+    checkPartition(items, k, cmp);
+  });
+});
+
+describe("quickselect (bounds/segments)", () => {
+  it("respects left/right boundaries (partial region)", () => {
+    const arr = [9, 8, 7, 6, 5, 4, 3, 2, 1];
+    // We'll only work inside [2..6], so k=4 is within that range.
+    const left = 2;
+    const right = 6;
+    const k = 4;
+
+    const before = arr.slice();
+    quickselect(arr, k, left, right);
+
+    // Elements outside [left..right] remain untouched
+    expect(arr.slice(0, left)).toEqual(before.slice(0, left));
+    expect(arr.slice(right + 1)).toEqual(before.slice(right + 1));
+
+    // Inside the segment, property holds
+    const seg = arr.slice(left, right + 1);
+    const segCmp = numCmp;
+    checkPartition(seg, k - left, segCmp);
+
+    // And k-th inside the segment matches the k-th of the sorted segment
+    const segSorted = before.slice(left, right + 1).sort(segCmp);
+    expect(arr[k]).toBe(segSorted[k - left]);
+  });
+
+  it("single-element segment is a no-op", () => {
+    const arr = [5, 4, 3, 2, 1];
+    const left = 2, right = 2, k = 2;
+    const before = arr.slice();
+    quickselect(arr, k, left, right);
+    expect(arr).toEqual(before);
+  });
+
+  it("k at segment boundaries (left/right)", () => {
+    const arr1 = [7, 4, 6, 1, 9, 2, 5, 0];
+    const left1 = 2, right1 = 6, k1 = left1;
+    const segSorted1 = arr1.slice(left1, right1 + 1).slice().sort(numCmp);
+    quickselect(arr1, k1, left1, right1);
+    expect(arr1[k1]).toBe(segSorted1[0]);
+    checkPartition(arr1.slice(left1, right1 + 1), k1 - left1, numCmp);
+
+    const arr2 = [7, 4, 6, 1, 9, 2, 5, 0];
+    const left2 = 1, right2 = 5, k2 = right2;
+    const segSorted2 = arr2.slice(left2, right2 + 1).slice().sort(numCmp);
+    quickselect(arr2, k2, left2, right2);
+    expect(arr2[k2]).toBe(segSorted2[segSorted2.length - 1]);
+    checkPartition(arr2.slice(left2, right2 + 1), k2 - left2, numCmp);
+  });
+});
+
+describe("quickselect (Floyd–Rivest path)", () => {
+  it("covers the large-array acceleration branch", () => {
+    const n = 2000; // right - left = 1999 > 600 -> triggers acceleration
+    const base = 2654435761; // Knuth multiplicative hash (32-bit after >>> 0)
+    const arr = Array.from({ length: n }, (_, i) => ((i * base) >>> 0))
+      .map(x => (x % 2000) - 1000); // keep numbers in a small range to include duplicates
+    const k = Math.floor(n * 0.63);
+
+    const orig = arr.slice();
+    quickselect(arr, k);
+    const sorted = orig.slice().sort(numCmp);
+
+    expect(arr[k]).toBe(sorted[k]);
+    checkPartition(arr, k, numCmp);
+  });
+});