docs/CHANGELOG.md: cut v1.79.3

Signed-off-by: hagen1778 <roman@victoriametrics.com>

Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/vmagent/remotewrite/client.go

@@ -56,10 +56,12 @@ var (
 
 	awsUseSigv4 = flagutil.NewArrayBool("remoteWrite.aws.useSigv4", "Enables SigV4 request signing for the corresponding -remoteWrite.url. "+
 		"It is expected that other -remoteWrite.aws.* command-line flags are set if sigv4 request signing is enabled")
-	awsRegion    = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsRoleARN   = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsAccessKey = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
-	awsService   = flagutil.NewArray("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
+	awsEC2Endpoint = flagutil.NewArray("remoteWrite.aws.ec2Endpoint", "Optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsSTSEndpoint = flagutil.NewArray("remoteWrite.aws.stsEndpoint", "Optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRegion      = flagutil.NewArray("remoteWrite.aws.region", "Optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsRoleARN     = flagutil.NewArray("remoteWrite.aws.roleARN", "Optional AWS roleARN to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsAccessKey   = flagutil.NewArray("remoteWrite.aws.accessKey", "Optional AWS AccessKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
+	awsService     = flagutil.NewArray("remoteWrite.aws.service", "Optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set. "+
 		"Defaults to \"aps\"")
 	awsSecretKey = flagutil.NewArray("remoteWrite.aws.secretKey", "Optional AWS SecretKey to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set")
 )
@@ -231,12 +233,14 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
 	if !awsUseSigv4.GetOptionalArg(argIdx) {
 		return nil, nil
 	}
+	ec2Endpoint := awsEC2Endpoint.GetOptionalArg(argIdx)
+	stsEndpoint := awsSTSEndpoint.GetOptionalArg(argIdx)
 	region := awsRegion.GetOptionalArg(argIdx)
 	roleARN := awsRoleARN.GetOptionalArg(argIdx)
 	accessKey := awsAccessKey.GetOptionalArg(argIdx)
 	secretKey := awsSecretKey.GetOptionalArg(argIdx)
 	service := awsService.GetOptionalArg(argIdx)
-	cfg, err := awsapi.NewConfig(region, roleARN, accessKey, secretKey, service)
+	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
 	if err != nil {
 		return nil, err
 	}

app/vmalert/README.md

@@ -642,6 +642,8 @@ The shortlist of configuration flags is the following:
      Whether to align "time" parameter with evaluation interval.Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257 (default true)
   -datasource.roundDigits int
      Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
+  -datasource.showURL
+     Whether to show -datasource.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -datasource.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
   -datasource.tlsCertFile string
@@ -653,7 +655,7 @@ The shortlist of configuration flags is the following:
   -datasource.tlsServerName string
      Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
   -datasource.url string
-     VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend
+     Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. E.g. http://127.0.0.1:8428 . See also '-datasource.disablePathAppend', '-datasource.showURL'.
   -defaultTenant.graphite string
      Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
   -defaultTenant.prometheus string
@@ -813,6 +815,8 @@ The shortlist of configuration flags is the following:
      Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.
   -remoteRead.oauth2.tokenUrl string
      Optional OAuth2 tokenURL to use for -remoteRead.url. 
+  -remoteRead.showURL
+     Whether to show -remoteRead.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteRead.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
   -remoteRead.tlsCertFile string
@@ -824,7 +828,7 @@ The shortlist of configuration flags is the following:
   -remoteRead.tlsServerName string
      Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
   -remoteRead.url vmalert
-     Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
+     Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
   -remoteWrite.basicAuth.password string
      Optional basic auth password for -remoteWrite.url
   -remoteWrite.basicAuth.passwordFile string
@@ -855,6 +859,8 @@ The shortlist of configuration flags is the following:
      Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.
   -remoteWrite.oauth2.tokenUrl string
      Optional OAuth2 tokenURL to use for -notifier.url.
+  -remoteWrite.showURL
+     Whether to show -remoteWrite.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteWrite.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
   -remoteWrite.tlsCertFile string
@@ -866,7 +872,7 @@ The shortlist of configuration flags is the following:
   -remoteWrite.tlsServerName string
      Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
   -remoteWrite.url string
-     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int

app/vmalert/datasource/init.go

@@ -8,12 +8,15 @@ import (
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 var (
-	addr = flag.String("datasource.url", "", "VictoriaMetrics or vmselect url. Required parameter. "+
-		"E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend")
-	appendTypePrefix = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. "+
+		"E.g. http://127.0.0.1:8428 . See also '-datasource.disablePathAppend', '-datasource.showURL'.")
+	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
+	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to show -datasource.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
 
 	basicAuthUsername     = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url")
 	basicAuthPassword     = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url")
@@ -47,6 +50,13 @@ var (
 		`In VM "round_digits" limits the number of digits after the decimal point in response values.`)
 )
 
+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showDatasourceURL {
+		flagutil.RegisterSecretFlag("datasource.url")
+	}
+}
+
 // Param represents an HTTP GET param
 type Param struct {
 	Key, Value string

app/vmalert/main.go

@@ -79,6 +79,9 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	remoteread.InitSecretFlags()
+	remotewrite.InitSecretFlags()
+	datasource.InitSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 	err := templates.Load(*ruleTemplatesPath, true)

app/vmalert/remoteread/init.go

@@ -7,12 +7,17 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 var (
-	addr = flag.String("remoteRead.url", "", "Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts "+
-		"state. This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend")
+	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
+		"Remote read is used to restore alerts state."+
+		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
+		"E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
+
+	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
 
 	basicAuthUsername     = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteRead.url")
 	basicAuthPassword     = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteRead.url")
@@ -36,6 +41,13 @@ var (
 	oauth2Scopes           = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )
 
+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showRemoteReadURL {
+		flagutil.RegisterSecretFlag("remoteRead.url")
+	}
+}
+
 // Init creates a Querier from provided flag values.
 // Returns nil if addr flag wasn't set.
 func Init() (datasource.QuerierBuilder, error) {

app/vmalert/remotewrite/init.go

@@ -7,12 +7,15 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
-		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend")
+		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
+	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
+		"It is hidden by default, since it can contain sensitive info such as auth key")
 
 	basicAuthUsername     = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remoteWrite.url")
 	basicAuthPassword     = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remoteWrite.url")
@@ -41,6 +44,13 @@ var (
 	oauth2Scopes           = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )
 
+// InitSecretFlags must be called after flag.Parse and before any logging
+func InitSecretFlags() {
+	if !*showRemoteWriteURL {
+		flagutil.RegisterSecretFlag("remoteWrite.url")
+	}
+}
+
 // Init creates Client object from given flags.
 // Returns nil if addr flag wasn't set.
 func Init(ctx context.Context) (*Client, error) {

app/vmalert/web.go

@@ -160,7 +160,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
 			redirectURL = alert.APILink()
 		}
-		http.Redirect(w, r, "/"+redirectURL, http.StatusPermanentRedirect)
+		httpserver.RedirectPermanent(w, "/"+redirectURL)
 		return true
 	}
 }

app/vmselect/main.go

@@ -168,7 +168,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		_ = r.ParseForm()
 		path = strings.TrimPrefix(path, "/")
 		newURL := path + "/?" + r.Form.Encode()
-		http.Redirect(w, r, newURL, http.StatusMovedPermanently)
+		httpserver.RedirectPermanent(w, newURL)
 		return true
 	}
 	if strings.HasPrefix(path, "/vmui/") {
@@ -217,7 +217,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		// vmalert access via incomplete url without `/` in the end. Redirecto to complete url.
 		// Use relative redirect, since, since the hostname and path prefix may be incorrect if VictoriaMetrics
 		// is hidden behind vmauth or similar proxy.
-		http.Redirect(w, r, "vmalert/", http.StatusMovedPermanently)
+		httpserver.RedirectPermanent(w, "vmalert/")
 		return true
 	}
 	if strings.HasPrefix(path, "/vmalert/") {

app/vmselect/promql/binary_op.go

@@ -36,9 +36,9 @@ var binaryOpFuncs = map[string]binaryOpFunc{
 	"unless": binaryOpUnless,
 
 	// New ops
-	"if":      newBinaryOpArithFunc(binaryop.If),
-	"ifnot":   newBinaryOpArithFunc(binaryop.Ifnot),
-	"default": newBinaryOpArithFunc(binaryop.Default),
+	"if":      binaryOpIf,
+	"ifnot":   binaryOpIfnot,
+	"default": binaryOpDefault,
 }
 
 func getBinaryOpFunc(op string) binaryOpFunc {
@@ -86,17 +86,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 		right := bfa.right
 		op := bfa.be.Op
 		switch true {
-		case op == "ifnot":
-			left = removeEmptySeries(left)
-			// Do not remove empty series on the right side,
-			// so the left-side series could be matched against them.
-		case op == "default":
-			// Do not remove empty series on the left and the right side,
-			// since this may lead to missing result:
-			// - if empty time series are removed on the left side,
-			// then they won't be substituted by time series from the right side.
-			// - if empty time series are removed on the right side,
-			// then this may result in missing time series from the left side.
 		case metricsql.IsBinaryOpCmp(op):
 			// Do not remove empty series for comparison operations,
 			// since this may lead to missing result.
@@ -136,7 +125,7 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 
 func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
 	if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
-		if isScalar(left) && be.Op != "default" && be.Op != "if" && be.Op != "ifnot" {
+		if isScalar(left) {
 			// Fast path: `scalar op vector`
 			rvsLeft := make([]*timeseries, len(right))
 			tsLeft := left[0]
@@ -324,14 +313,23 @@ func resetMetricGroupIfRequired(be *metricsql.BinaryOpExpr, ts *timeseries) {
 		// Do not reset MetricGroup for non-boolean `compare` binary ops like Prometheus does.
 		return
 	}
-	switch be.Op {
-	case "default", "if", "ifnot":
-		// Do not reset MetricGroup for these ops.
-		return
-	}
 	ts.MetricName.ResetMetricGroup()
 }
 
+func binaryOpIf(bfa *binaryOpFuncArg) ([]*timeseries, error) {
+	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
+	var rvs []*timeseries
+	for k, tssLeft := range mLeft {
+		tssRight := seriesByKey(mRight, k)
+		if tssRight == nil {
+			continue
+		}
+		tssLeft = addRightNaNsToLeft(tssLeft, tssRight)
+		rvs = append(rvs, tssLeft...)
+	}
+	return rvs, nil
+}
+
 func binaryOpAnd(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries
@@ -340,24 +338,47 @@ func binaryOpAnd(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 		if tssLeft == nil {
 			continue
 		}
-		// Add gaps to tssLeft if there are gaps at tssRight.
-		for _, tsLeft := range tssLeft {
-			valuesLeft := tsLeft.Values
-			for i := range valuesLeft {
-				hasValue := false
-				for _, tsRight := range tssRight {
-					if !math.IsNaN(tsRight.Values[i]) {
-						hasValue = true
-						break
-					}
-				}
-				if !hasValue {
-					valuesLeft[i] = nan
+		tssLeft = addRightNaNsToLeft(tssLeft, tssRight)
+		rvs = append(rvs, tssLeft...)
+	}
+	return rvs, nil
+}
+
+func addRightNaNsToLeft(tssLeft, tssRight []*timeseries) []*timeseries {
+	for _, tsLeft := range tssLeft {
+		valuesLeft := tsLeft.Values
+		for i := range valuesLeft {
+			hasValue := false
+			for _, tsRight := range tssRight {
+				if !math.IsNaN(tsRight.Values[i]) {
+					hasValue = true
+					break
 				}
 			}
+			if !hasValue {
+				valuesLeft[i] = nan
+			}
 		}
-		tssLeft = removeEmptySeries(tssLeft)
+	}
+	return removeEmptySeries(tssLeft)
+}
+
+func binaryOpDefault(bfa *binaryOpFuncArg) ([]*timeseries, error) {
+	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
+	var rvs []*timeseries
+	if len(mLeft) == 0 {
+		for _, tss := range mRight {
+			rvs = append(rvs, tss...)
+		}
+		return rvs, nil
+	}
+	for k, tssLeft := range mLeft {
 		rvs = append(rvs, tssLeft...)
+		tssRight := seriesByKey(mRight, k)
+		if tssRight == nil {
+			continue
+		}
+		fillLeftNaNsWithRightValues(tssLeft, tssRight)
 	}
 	return rvs, nil
 }
@@ -374,24 +395,43 @@ func binaryOpOr(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 			rvs = append(rvs, tssRight...)
 			continue
 		}
-		// Fill gaps in tssLeft with values from tssRight as Prometheus does.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/552
-		for _, tsLeft := range tssLeft {
-			valuesLeft := tsLeft.Values
-			for i, v := range valuesLeft {
-				if !math.IsNaN(v) {
-					continue
-				}
-				for _, tsRight := range tssRight {
-					vRight := tsRight.Values[i]
-					if !math.IsNaN(vRight) {
-						valuesLeft[i] = vRight
-						break
-					}
+		fillLeftNaNsWithRightValues(tssLeft, tssRight)
+	}
+	return rvs, nil
+}
+
+func fillLeftNaNsWithRightValues(tssLeft, tssRight []*timeseries) {
+	// Fill gaps in tssLeft with values from tssRight as Prometheus does.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/552
+	for _, tsLeft := range tssLeft {
+		valuesLeft := tsLeft.Values
+		for i, v := range valuesLeft {
+			if !math.IsNaN(v) {
+				continue
+			}
+			for _, tsRight := range tssRight {
+				vRight := tsRight.Values[i]
+				if !math.IsNaN(vRight) {
+					valuesLeft[i] = vRight
+					break
 				}
 			}
 		}
 	}
+}
+
+func binaryOpIfnot(bfa *binaryOpFuncArg) ([]*timeseries, error) {
+	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
+	var rvs []*timeseries
+	for k, tssLeft := range mLeft {
+		tssRight := seriesByKey(mRight, k)
+		if tssRight == nil {
+			rvs = append(rvs, tssLeft...)
+			continue
+		}
+		tssLeft = addLeftNaNsIfNoRightNaNs(tssLeft, tssRight)
+		rvs = append(rvs, tssLeft...)
+	}
 	return rvs, nil
 }
 
@@ -404,24 +444,44 @@ func binaryOpUnless(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 			rvs = append(rvs, tssLeft...)
 			continue
 		}
-		// Add gaps to tssLeft if the are no gaps at tssRight.
-		for _, tsLeft := range tssLeft {
-			valuesLeft := tsLeft.Values
-			for i := range valuesLeft {
-				for _, tsRight := range tssRight {
-					if !math.IsNaN(tsRight.Values[i]) {
-						valuesLeft[i] = nan
-						break
-					}
-				}
-			}
-		}
-		tssLeft = removeEmptySeries(tssLeft)
+		tssLeft = addLeftNaNsIfNoRightNaNs(tssLeft, tssRight)
 		rvs = append(rvs, tssLeft...)
 	}
 	return rvs, nil
 }
 
+func addLeftNaNsIfNoRightNaNs(tssLeft, tssRight []*timeseries) []*timeseries {
+	for _, tsLeft := range tssLeft {
+		valuesLeft := tsLeft.Values
+		for i := range valuesLeft {
+			for _, tsRight := range tssRight {
+				if !math.IsNaN(tsRight.Values[i]) {
+					valuesLeft[i] = nan
+					break
+				}
+			}
+		}
+	}
+	return removeEmptySeries(tssLeft)
+}
+
+func seriesByKey(m map[string][]*timeseries, key string) []*timeseries {
+	tss := m[key]
+	if tss != nil {
+		return tss
+	}
+	if len(m) != 1 {
+		return nil
+	}
+	for _, tss := range m {
+		if isScalar(tss) {
+			return tss
+		}
+		return nil
+	}
+	return nil
+}
+
 func createTimeseriesMapByTagSet(be *metricsql.BinaryOpExpr, left, right []*timeseries) (map[string][]*timeseries, map[string][]*timeseries) {
 	groupTags := be.GroupModifier.Args
 	groupOp := strings.ToLower(be.GroupModifier.Op)

app/vmselect/promql/exec_test.go

@@ -2803,7 +2803,12 @@ func TestExecSuccess(t *testing.T) {
 	t.Run(`scalar default vector1`, func(t *testing.T) {
 		t.Parallel()
 		q := `time() > 1400 default label_set(123, "foo", "bar")`
-		resultExpected := []netstorage.Result{}
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
 	t.Run(`scalar default vector2`, func(t *testing.T) {
@@ -6092,7 +6097,18 @@ func TestExecSuccess(t *testing.T) {
 	t.Run(`ifnot-no-matching-timeseries`, func(t *testing.T) {
 		t.Parallel()
 		q := `label_set(time(), "foo", "bar") ifnot label_set(time() > 1400, "x", "y")`
-		resultExpected := []netstorage.Result{}
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+		}
+		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
 	t.Run(`quantile(-2)`, func(t *testing.T) {
@@ -7925,6 +7941,7 @@ func TestExecError(t *testing.T) {
 	f(`limit_offet(1, (alias(1,"foo"),alias(2,"bar")), 10)`)
 	f(`round(1, 1 or label_set(2, "xx", "foo"))`)
 	f(`histogram_quantile(1 or label_set(2, "xx", "foo"), 1)`)
+	f(`histogram_quantiles("foo", 1 or label_set(2, "xxx", "foo"), 2)`)
 	f(`label_set(1, 2, 3)`)
 	f(`label_set(1, "foo", (label_set(1, "foo", bar") or label_set(2, "xxx", "yy")))`)
 	f(`label_set(1, "foo", 3)`)

app/vmselect/promql/transform.go

@@ -822,8 +822,12 @@ func transformHistogramQuantiles(tfa *transformFuncArg) ([]*timeseries, error) {
 	tssOrig := args[len(args)-1]
 	// Calculate quantile individually per each phi.
 	var rvs []*timeseries
-	for _, phiArg := range phiArgs {
-		phiStr := fmt.Sprintf("%g", phiArg[0].Values[0])
+	for i, phiArg := range phiArgs {
+		phis, err := getScalar(phiArg, i)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse phi: %w", err)
+		}
+		phiStr := fmt.Sprintf("%g", phis[0])
 		tss := copyTimeseries(tssOrig)
 		tfaTmp := &transformFuncArg{
 			ec: tfa.ec,
@@ -2183,17 +2187,13 @@ func transformTimezoneOffset(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, fmt.Errorf("cannot load timezone %q: %w", tzString, err)
 	}
 
-	var ts timeseries
-	ts.denyReuse = true
-	timestamps := tfa.ec.getSharedTimestamps()
-	values := make([]float64, len(timestamps))
-	for i, v := range timestamps {
-		_, offset := time.Unix(v/1000, 0).In(loc).Zone()
-		values[i] = float64(offset)
+	tss := evalNumber(tfa.ec, nan)
+	ts := tss[0]
+	for i, timestamp := range ts.Timestamps {
+		_, offset := time.Unix(timestamp/1000, 0).In(loc).Zone()
+		ts.Values[i] = float64(offset)
 	}
-	ts.Values = values
-	ts.Timestamps = timestamps
-	return []*timeseries{&ts}, nil
+	return tss, nil
 }
 
 func transformTime(tfa *transformFuncArg) ([]*timeseries, error) {

app/vmui/Dockerfile-web

@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.16.0
+FROM alpine:3.16.2
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/src/components/CustomPanel/Configurator/Time/TimeSelector.tsx

@@ -110,7 +110,9 @@ export const TimeSelector: FC = () => {
       open={open}
       anchorEl={anchorEl}
       placement="bottom-end"
-      modifiers={[{name: "offset", options: {offset: [0, 6]}}]}>
+      modifiers={[{name: "offset", options: {offset: [0, 6]}}]}
+      sx={{zIndex: 3, position: "relative"}}
+    >
       <ClickAwayListener onClickAway={() => setAnchorEl(null)}>
         <Paper elevation={3}>
           <Box sx={classes.container}>

deployment/docker/Makefile

@@ -2,9 +2,9 @@
 
 DOCKER_NAMESPACE := victoriametrics
 
-ROOT_IMAGE ?= alpine:3.16.0
-CERTS_IMAGE := alpine:3.16.0
-GO_BUILDER_IMAGE := golang:1.18.4-alpine
+ROOT_IMAGE ?= alpine:3.16.2
+CERTS_IMAGE := alpine:3.16.2
+GO_BUILDER_IMAGE := golang:1.19.0-alpine
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.3-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
 

docs/CHANGELOG.md

@@ -13,12 +13,61 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * [How to build vmauth](https://docs.victoriametrics.com/vmauth.html#how-to-build-from-sources)
 * [How to build vmctl](https://docs.victoriametrics.com/vmctl.html#how-to-build)
 
-## tip
+## v1.79.x long-time support release (LTS)
+
+## [v1.79.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.3)
+
+Released at 30-08-2022
+
+**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
+
+* SECURITY: [vmalert](https://docs.victoriametrics.com/vmalert.html): do not expose `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flag values in logs and at `http://vmalert:8880/flags` page by default, since they may contain sensitive data such as auth keys. This aligns `vmalert` behaviour with [vmagent](), which doesn't expose `-remoteWrite.url` command-line flag value in logs and at `http://vmagent:8429/flags` page by default. Specify `-remoteWrite.showURL`, `-remoteRead.showURL` and `-datasource.showURL` command-line flags for showing values for the corresponding `-*.url` flags in logs. Thanks to @mble for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2965).
+* SECURITY: upgrade base docker image (alpine) from 3.16.1 to 3.16.2. See [alpine 3.16.2 release notes](https://alpinelinux.org/posts/Alpine-3.13.12-3.14.8-3.15.6-3.16.2-released.html).
+
+* BUGFIX: prevent from excess CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode).
+* BUGFIX: improve performance for requests to [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels) and [/api/v1/label/.../values](https://docs.victoriametrics.com/url-examples.html#apiv1labelvalues) when the filter in the `match[]` query arg matches small number of time series. The performance for this case has been reduced in [v1.78.0](https://docs.victoriametrics.com/CHANGELOG.html#v1780). See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1533) issues.
+* BUGFIX: increase the default limit on the number of concurrent merges for small parts from 8 to 16. This should help resolving potential issues with heavy data ingestion. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2673#issuecomment-1218185978) from @lukepalmer .
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): fix panic when incorrect arg is passed as `phi` into [histogram_quantiles](https://docs.victoriametrics.com/MetricsQL.html#histogram_quantiles) function. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3026).
+
+## [v1.79.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.2)
+
+Released at 08-08-2022
+
+**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
+
+* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): fix potential panic in [multi-level cluster setup](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multi-level-cluster-setup) when top-level `vmselect` is configured with `-replicationFactor` bigger than 1. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2961).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly handle custom `endpoint` value in [ec2_sd_configs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config). It was ignored since [v1.77.0](https://docs.victoriametrics.com/CHANGELOG.html#v1770) because of a bug in the implementation of [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1287).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): add missing `__meta_kubernetes_ingress_class_name` meta-label for `role: ingress` service discovery in Kubernetes. See [this commit from Prometheus](https://github.com/prometheus/prometheus/commit/7e65ad3e432bd2837c17e3e63e85dcbcc30f4a8a).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow stale responses from Consul service discovery (aka [consul_sd_configs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config)) by default in the same way as Prometheus does. This should reduce load on Consul when discovering big number of targets. Stale responses can be disabled by specifying `allow_stale: false` option in `consul_sd_config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2940).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): [dockerswarm_sd_configs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config): properly set `__meta_dockerswarm_container_label_*` labels instead of `__meta_dockerswarm_task_label_*` labels as Prometheus does. See [this issue](https://github.com/prometheus/prometheus/issues/9187).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): set `up` metric to `0` for partial scrapes in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). Previously the `up` metric was set to `1` when at least a single metric has been scraped before the error. This aligns the behaviour of `vmselect` with Prometheus.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): restart all the scrape jobs during [config reload](https://docs.victoriametrics.com/vmagent.html#configuration-update) after `global` section is changed inside `-promscrape.config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2884).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with AWS ECS credentials. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2875). Thanks to @transacid for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2876).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not split regex in [relabeling rules](https://docs.victoriametrics.com/vmagent.html#relabeling) into multiple lines if it contains groups. This fixes [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2928).
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): return series from `q1` if `q2` doesn't return matching time series in the query `q1 ifnot q2`. Previously series from `q1` weren't returned in this case.
+* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly show date picker at `Table` tab. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2874).
+* BUGFIX: properly generate http redirects if `-http.pathPrefix` command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2918).
+
+
+## [v1.79.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.1)
+
+Released at 02-08-2022
+
+**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
+
+* SECURITY: upgrade base docker image (alpine) from 3.16.0 to 3.16.1 . See [alpine 3.16.1 release notes](https://alpinelinux.org/posts/Alpine-3.16.1-released.html).
+
 
 ## [v1.79.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.79.0)
 
 Released at 14-07-2022
 
+**v1.79.x is a line of LTS releases (e.g. long-time support). It contains important up-to-date bugfixes.
+The v1.79.x line will be supported for at least 12 months since [v1.79.0](https://docs.victoriametrics.com/CHANGELOG.html#v1790) release**
+
 **Update note 1:** this release introduces backwards-incompatible changes to `vm_partial_results_total` metric by changing its labels to be consistent with `vm_requests_total` metric. If you use alerting rules or Grafana dashboards, which rely on this metric, then they must be updated. The official dashboards for VictoriaMetrics don't use this metric.
 
 **Update note 2:** [vmalert](https://docs.victoriametrics.com/vmalert.html) adds `/vmalert/` prefix to [web urls](https://docs.victoriametrics.com/vmalert.html#web) according to [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825). This may affect `vmalert` instances with non-empty `-http.pathPrefix` command-line flag. After the update, configuring this flag is no longer needed. Here's [why](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2799#issuecomment-1171392005).

docs/MetricsQL.md

@@ -487,7 +487,7 @@ See also [implicit query conversions](#implicit-query-conversions).
 
 #### histogram_quantiles
 
-`histogram_quantiles("phiLabel", phi1, ..., phiN, buckets)` calculates the given `phi*`-quantiles over the given [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350). `phi*` must be in the range `[0...1]`. Each calculated quantile is returned in a separate time series with the corresponding `{phiLabel="phi*"}` label. See also [histogram_quantile](#histogram_quantile).
+`histogram_quantiles("phiLabel", phi1, ..., phiN, buckets)` calculates the given `phi*`-quantiles over the given [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350). Argument `phi*` must be in the range `[0...1]`. For example, `histogram_quantiles('le', 0.3, 0.5, sum(rate(http_request_duration_seconds_bucket[5m]) by (le))`. Each calculated quantile is returned in a separate time series with the corresponding `{phiLabel="phi*"}` label. See also [histogram_quantile](#histogram_quantile).
 
 #### histogram_share
 

docs/vmalert.md

@@ -646,6 +646,8 @@ The shortlist of configuration flags is the following:
      Whether to align "time" parameter with evaluation interval.Alignment supposed to produce deterministic results despite of number of vmalert replicas or time they were started. See more details here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1257 (default true)
   -datasource.roundDigits int
      Adds "round_digits" GET param to datasource requests. In VM "round_digits" limits the number of digits after the decimal point in response values.
+  -datasource.showURL
+     Whether to show -datasource.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -datasource.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -datasource.url. By default, system CA is used
   -datasource.tlsCertFile string
@@ -657,7 +659,7 @@ The shortlist of configuration flags is the following:
   -datasource.tlsServerName string
      Optional TLS server name to use for connections to -datasource.url. By default, the server name from -datasource.url is used
   -datasource.url string
-     VictoriaMetrics or vmselect url. Required parameter. E.g. http://127.0.0.1:8428 . See also -remoteRead.disablePathAppend
+     Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect URL. Required parameter. E.g. http://127.0.0.1:8428 . See also '-datasource.disablePathAppend', '-datasource.showURL'.
   -defaultTenant.graphite string
      Default tenant for Graphite alerting groups. See https://docs.victoriametrics.com/vmalert.html#multitenancy
   -defaultTenant.prometheus string
@@ -817,6 +819,8 @@ The shortlist of configuration flags is the following:
      Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.
   -remoteRead.oauth2.tokenUrl string
      Optional OAuth2 tokenURL to use for -remoteRead.url. 
+  -remoteRead.showURL
+     Whether to show -remoteRead.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteRead.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -remoteRead.url. By default system CA is used
   -remoteRead.tlsCertFile string
@@ -828,7 +832,7 @@ The shortlist of configuration flags is the following:
   -remoteRead.tlsServerName string
      Optional TLS server name to use for connections to -remoteRead.url. By default the server name from -remoteRead.url is used
   -remoteRead.url vmalert
-     Optional URL to VictoriaMetrics or vmselect that will be used to restore alerts state. This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also -remoteRead.disablePathAppend
+     Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect.Remote read is used to restore alerts state.This configuration makes sense only if vmalert was configured with `remoteWrite.url` before and has been successfully persisted its state. E.g. http://127.0.0.1:8428. See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.
   -remoteWrite.basicAuth.password string
      Optional basic auth password for -remoteWrite.url
   -remoteWrite.basicAuth.passwordFile string
@@ -859,6 +863,8 @@ The shortlist of configuration flags is the following:
      Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.
   -remoteWrite.oauth2.tokenUrl string
      Optional OAuth2 tokenURL to use for -notifier.url.
+  -remoteWrite.showURL
+     Whether to show -remoteWrite.url in the exported metrics. It is hidden by default, since it can contain sensitive info such as auth key
   -remoteWrite.tlsCAFile string
      Optional path to TLS CA file to use for verifying connections to -remoteWrite.url. By default system CA is used
   -remoteWrite.tlsCertFile string
@@ -870,7 +876,7 @@ The shortlist of configuration flags is the following:
   -remoteWrite.tlsServerName string
      Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used
   -remoteWrite.url string
-     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend
+     Optional URL to VictoriaMetrics or vminsert where to persist alerts state and recording rules results in form of timeseries. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int

lib/awsapi/config.go

@@ -43,8 +43,8 @@ type credentials struct {
 	Expiration      time.Time
 }
 
-// NewConfig returns new AWS Config.
-func NewConfig(region, roleARN, accessKey, secretKey, service string) (*Config, error) {
+// NewConfig returns new AWS Config from the given args.
+func NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service string) (*Config, error) {
 	cfg := &Config{
 		client:           http.DefaultClient,
 		region:           region,
@@ -65,8 +65,8 @@ func NewConfig(region, roleARN, accessKey, secretKey, service string) (*Config,
 		}
 		cfg.region = r
 	}
-	cfg.ec2Endpoint = buildAPIEndpoint(cfg.ec2Endpoint, cfg.region, "ec2")
-	cfg.stsEndpoint = buildAPIEndpoint(cfg.stsEndpoint, cfg.region, "sts")
+	cfg.ec2Endpoint = buildAPIEndpoint(ec2Endpoint, cfg.region, "ec2")
+	cfg.stsEndpoint = buildAPIEndpoint(stsEndpoint, cfg.region, "sts")
 	if cfg.roleARN == "" {
 		cfg.roleARN = os.Getenv("AWS_ROLE_ARN")
 	}
@@ -204,7 +204,11 @@ func (cfg *Config) getAPICredentials() (*credentials, error) {
 	}
 	if ecsMetaURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); len(ecsMetaURI) > 0 {
 		path := "http://169.254.170.2" + ecsMetaURI
-		return getECSRoleCredentialsByPath(cfg.client, path)
+		ac, err := getECSRoleCredentialsByPath(cfg.client, path)
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain ECS role credentials: %w", err)
+		}
+		acNew = ac
 	}
 
 	// we need instance credentials if dont have access keys

lib/httpserver/httpserver.go

@@ -235,13 +235,27 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		connTimeoutClosedConns.Inc()
 		w.Header().Set("Connection", "close")
 	}
-	path, err := getCanonicalPath(r.URL.Path)
-	if err != nil {
-		Errorf(w, r, "cannot get canonical path: %s", err)
-		unsupportedRequestErrors.Inc()
-		return
+	path := r.URL.Path
+	prefix := GetPathPrefix()
+	if prefix != "" {
+		// Trim -http.pathPrefix from path
+		prefixNoTrailingSlash := strings.TrimSuffix(prefix, "/")
+		if path == prefixNoTrailingSlash {
+			// Redirect to url with / at the end.
+			// This is needed for proper handling of relative urls in web browsers.
+			// Intentionally ignore query args, since it is expected that the requested url
+			// is composed by a human, so it doesn't contain query args.
+			RedirectPermanent(w, prefix)
+			return
+		}
+		if !strings.HasPrefix(path, prefix) {
+			Errorf(w, r, "missing -http.pathPrefix=%q in the requested path %q", *pathPrefix, path)
+			unsupportedRequestErrors.Inc()
+			return
+		}
+		path = path[len(prefix)-1:]
+		r.URL.Path = path
 	}
-	r.URL.Path = path
 	switch r.URL.Path {
 	case "/health":
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
@@ -327,24 +341,6 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 	}
 }
 
-func getCanonicalPath(path string) (string, error) {
-	if len(*pathPrefix) == 0 || path == "/" {
-		return path, nil
-	}
-	if *pathPrefix == path {
-		return "/", nil
-	}
-	prefix := *pathPrefix
-	if !strings.HasSuffix(prefix, "/") {
-		prefix = prefix + "/"
-	}
-	if !strings.HasPrefix(path, prefix) {
-		return "", fmt.Errorf("missing `-pathPrefix=%q` in the requested path: %q", *pathPrefix, path)
-	}
-	path = path[len(prefix)-1:]
-	return path, nil
-}
-
 func checkBasicAuth(w http.ResponseWriter, r *http.Request) bool {
 	if len(*httpAuthUsername) == 0 {
 		// HTTP Basic Auth is disabled.
@@ -643,7 +639,17 @@ func IsTLS() bool {
 
 // GetPathPrefix - returns http server path prefix.
 func GetPathPrefix() string {
-	return *pathPrefix
+	prefix := *pathPrefix
+	if prefix == "" {
+		return ""
+	}
+	if !strings.HasPrefix(prefix, "/") {
+		prefix = "/" + prefix
+	}
+	if !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+	return prefix
 }
 
 // WriteAPIHelp writes pathList to w in HTML format.
@@ -671,3 +677,12 @@ func GetRequestURI(r *http.Request) string {
 	}
 	return requestURI + delimiter + queryArgs
 }
+
+// RedirectPermanent redirects to the given url using 301 status code.
+func RedirectPermanent(w http.ResponseWriter, url string) {
+	// Do not use http.Redirect, since it breaks relative redirects
+	// if the http.Request.URL contains unexpected url.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2918
+	w.Header().Set("Location", url)
+	w.WriteHeader(http.StatusMovedPermanently)
+}

lib/promrelabel/config.go

@@ -104,6 +104,11 @@ func stringValue(v interface{}) (string, error) {
 
 // MarshalYAML marshals mlr to YAML.
 func (mlr *MultiLineRegex) MarshalYAML() (interface{}, error) {
+	if strings.ContainsAny(mlr.S, "([") {
+		// The mlr.S contains groups. Fall back to returning the regexp as is without splitting it into parts.
+		// This fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2928 .
+		return mlr.S, nil
+	}
 	a := strings.Split(mlr.S, "|")
 	if len(a) == 1 {
 		return a[0], nil

lib/promrelabel/config_test.go

@@ -7,6 +7,30 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+func TestMultiLineRegexUnmarshalMarshal(t *testing.T) {
+	f := func(data, resultExpected string) {
+		t.Helper()
+		var mlr MultiLineRegex
+		if err := yaml.UnmarshalStrict([]byte(data), &mlr); err != nil {
+			t.Fatalf("cannot unmarshal %q: %s", data, err)
+		}
+		result, err := yaml.Marshal(&mlr)
+		if err != nil {
+			t.Fatalf("cannot marshal %q: %s", data, err)
+		}
+		if string(result) != resultExpected {
+			t.Fatalf("unexpected marshaled data; got\n%q\nwant\n%q", result, resultExpected)
+		}
+	}
+	f(``, `""`+"\n")
+	f(`foo`, "foo\n")
+	f(`a|b||c`, "- a\n- b\n- \"\"\n- c\n")
+	f(`(a|b)`, "(a|b)\n")
+	f(`a|b[c|d]`, "a|b[c|d]\n")
+	f("- a\n- b", "- a\n- b\n")
+	f("- a\n- (b)", "a|(b)\n")
+}
+
 func TestRelabelConfigMarshalUnmarshal(t *testing.T) {
 	f := func(data, resultExpected string) {
 		t.Helper()
@@ -31,7 +55,7 @@ func TestRelabelConfigMarshalUnmarshal(t *testing.T) {
 - regex:
   - 'fo.+'
   - '.*ba[r-z]a'
-`, "- regex:\n  - fo.+\n  - .*ba[r-z]a\n")
+`, "- regex: fo.+|.*ba[r-z]a\n")
 	f(`- regex: foo|bar`, "- regex:\n  - foo\n  - bar\n")
 	f(`- regex: True`, `- regex: "true"`+"\n")
 	f(`- regex: true`, `- regex: "true"`+"\n")

lib/promscrape/config.go

@@ -130,6 +130,10 @@ func (cfg *Config) mustRestart(prevCfg *Config) {
 		prevScrapeCfgByName[scPrev.JobName] = scPrev
 	}
 
+	// Restart all the scrape jobs on Global config change.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2884
+	needGlobalRestart := !areEqualGlobalConfigs(&cfg.Global, &prevCfg.Global)
+
 	// Loop over the the new jobs, start new ones and restart updated ones.
 	var started, stopped, restarted int
 	currentJobNames := make(map[string]struct{}, len(cfg.ScrapeConfigs))
@@ -142,7 +146,7 @@ func (cfg *Config) mustRestart(prevCfg *Config) {
 			started++
 			continue
 		}
-		if areEqualScrapeConfigs(scPrev, sc) {
+		if !needGlobalRestart && areEqualScrapeConfigs(scPrev, sc) {
 			// The scrape config didn't change, so no need to restart it.
 			// Use the reference to the previous job, so it could be stopped properly later.
 			cfg.ScrapeConfigs[i] = scPrev
@@ -165,6 +169,12 @@ func (cfg *Config) mustRestart(prevCfg *Config) {
 	logger.Infof("restarted service discovery routines in %.3f seconds, stopped=%d, started=%d, restarted=%d", time.Since(startTime).Seconds(), stopped, started, restarted)
 }
 
+func areEqualGlobalConfigs(a, b *GlobalConfig) bool {
+	sa := a.marshalJSON()
+	sb := b.marshalJSON()
+	return string(sa) == string(sb)
+}
+
 func areEqualScrapeConfigs(a, b *ScrapeConfig) bool {
 	sa := a.marshalJSON()
 	sb := b.marshalJSON()
@@ -183,6 +193,14 @@ func (sc *ScrapeConfig) marshalJSON() []byte {
 	return data
 }
 
+func (gc *GlobalConfig) marshalJSON() []byte {
+	data, err := json.Marshal(gc)
+	if err != nil {
+		logger.Panicf("BUG: cannot marshal GlobalConfig: %s", err)
+	}
+	return data
+}
+
 func (cfg *Config) mustStop() {
 	startTime := time.Now()
 	logger.Infof("stopping service discovery routines...")

lib/promscrape/discovery/consul/consul.go

@@ -27,7 +27,7 @@ type SDConfig struct {
 	Tags              []string                   `yaml:"tags,omitempty"`
 	NodeMeta          map[string]string          `yaml:"node_meta,omitempty"`
 	TagSeparator      *string                    `yaml:"tag_separator,omitempty"`
-	AllowStale        bool                       `yaml:"allow_stale,omitempty"`
+	AllowStale        *bool                      `yaml:"allow_stale,omitempty"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.consulSDCheckInterval` command-line option.
 }

lib/promscrape/discovery/consul/watch.go

@@ -45,7 +45,7 @@ type serviceWatcher struct {
 // newConsulWatcher creates new watcher and starts background service discovery for Consul.
 func newConsulWatcher(client *discoveryutils.Client, sdc *SDConfig, datacenter, namespace string) *consulWatcher {
 	baseQueryArgs := "?dc=" + url.QueryEscape(datacenter)
-	if sdc.AllowStale {
+	if sdc.AllowStale == nil || *sdc.AllowStale {
 		baseQueryArgs += "&stale"
 	}
 	if namespace != "" {

lib/promscrape/discovery/dockerswarm/tasks.go

@@ -15,7 +15,6 @@ type task struct {
 	ID                  string
 	ServiceID           string
 	NodeID              string
-	Labels              map[string]string
 	DesiredState        string
 	NetworksAttachments []struct {
 		Addresses []string
@@ -32,6 +31,11 @@ type task struct {
 			Ports []portConfig
 		}
 	}
+	Spec struct {
+		ContainerSpec struct {
+			Labels map[string]string
+		}
+	}
 	Slot int
 }
 
@@ -82,8 +86,8 @@ func addTasksLabels(tasks []task, nodesLabels, servicesLabels []map[string]strin
 			"__meta_dockerswarm_task_slot":          strconv.Itoa(task.Slot),
 			"__meta_dockerswarm_task_state":         task.Status.State,
 		}
-		for k, v := range task.Labels {
-			commonLabels["__meta_dockerswarm_task_label_"+discoveryutils.SanitizeLabelName(k)] = v
+		for k, v := range task.Spec.ContainerSpec.Labels {
+			commonLabels["__meta_dockerswarm_container_label_"+discoveryutils.SanitizeLabelName(k)] = v
 		}
 		var svcPorts []portConfig
 		for i, v := range services {

lib/promscrape/discovery/dockerswarm/tasks_test.go

@@ -27,13 +27,13 @@ func Test_parseTasks(t *testing.T) {
     "Version": {
       "Index": 23
     },
-    "Labels": {
-	    "label1": "value1"
-    },
     "Spec": {
       "ContainerSpec": {
         "Image": "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
-        "Init": false
+        "Init": false,
+        "Labels": {
+	    "label1": "value1"
+        }
       },
       "Resources": {
         "Limits": {},
@@ -70,8 +70,18 @@ func Test_parseTasks(t *testing.T) {
 					ID:        "t4rdm7j2y9yctbrksiwvsgpu5",
 					ServiceID: "t91nf284wzle1ya09lqvyjgnq",
 					NodeID:    "qauwmifceyvqs0sipvzu8oslu",
-					Labels: map[string]string{
-						"label1": "value1",
+					Spec: struct {
+						ContainerSpec struct {
+							Labels map[string]string
+						}
+					}{
+						ContainerSpec: struct {
+							Labels map[string]string
+						}{
+							Labels: map[string]string{
+								"label1": "value1",
+							},
+						},
 					},
 					DesiredState: "running",
 					Slot:         1,
@@ -97,7 +107,7 @@ func Test_parseTasks(t *testing.T) {
 				return
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseTasks() got = %v, want %v", got, tt.want)
+				t.Errorf("parseTasks() got\n%v\nwant\n%v", got, tt.want)
 			}
 		})
 	}
@@ -126,7 +136,6 @@ func Test_addTasksLabels(t *testing.T) {
 						ID:           "t4rdm7j2y9yctbrksiwvsgpu5",
 						ServiceID:    "t91nf284wzle1ya09lqvyjgnq",
 						NodeID:       "qauwmifceyvqs0sipvzu8oslu",
-						Labels:       map[string]string{},
 						DesiredState: "running",
 						Slot:         1,
 						Status: struct {
@@ -194,7 +203,6 @@ func Test_addTasksLabels(t *testing.T) {
 						ID:           "t4rdm7j2y9yctbrksiwvsgpu5",
 						ServiceID:    "tgsci5gd31aai3jyudv98pqxf",
 						NodeID:       "qauwmifceyvqs0sipvzu8oslu",
-						Labels:       map[string]string{},
 						DesiredState: "running",
 						Slot:         1,
 						NetworksAttachments: []struct {

lib/promscrape/discovery/ec2/api.go

@@ -33,7 +33,11 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	if sdc.Port != nil {
 		port = *sdc.Port
 	}
-	awsCfg, err := awsapi.NewConfig(sdc.Region, sdc.RoleARN, sdc.AccessKey, sdc.SecretKey.String(), "ec2")
+	stsEndpoint := sdc.STSEndpoint
+	if stsEndpoint == "" {
+		stsEndpoint = sdc.Endpoint
+	}
+	awsCfg, err := awsapi.NewConfig(sdc.Endpoint, stsEndpoint, sdc.Region, sdc.RoleARN, sdc.AccessKey, sdc.SecretKey.String(), "ec2")
 	if err != nil {
 		return nil, err
 	}

lib/promscrape/discovery/ec2/ec2.go

@@ -18,12 +18,13 @@ var SDCheckInterval = flag.Duration("promscrape.ec2SDCheckInterval", time.Minute
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config
 type SDConfig struct {
-	Region    string           `yaml:"region,omitempty"`
-	Endpoint  string           `yaml:"endpoint,omitempty"`
-	AccessKey string           `yaml:"access_key,omitempty"`
-	SecretKey *promauth.Secret `yaml:"secret_key,omitempty"`
+	Region      string           `yaml:"region,omitempty"`
+	Endpoint    string           `yaml:"endpoint,omitempty"`
+	STSEndpoint string           `yaml:"sts_endpoint,omitempty"`
+	AccessKey   string           `yaml:"access_key,omitempty"`
+	SecretKey   *promauth.Secret `yaml:"secret_key,omitempty"`
 	// TODO add support for Profile, not working atm
-	Profile string `yaml:"profile,omitempty"`
+	// Profile string `yaml:"profile,omitempty"`
 	RoleARN string `yaml:"role_arn,omitempty"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.

lib/promscrape/discovery/kubernetes/ingress.go

@@ -52,8 +52,9 @@ type Ingress struct {
 //
 // See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingressspec-v1-networking-k8s-io
 type IngressSpec struct {
-	TLS   []IngressTLS `json:"tls"`
-	Rules []IngressRule
+	TLS              []IngressTLS `json:"tls"`
+	Rules            []IngressRule
+	IngressClassName string
 }
 
 // IngressTLS represents ingress TLS spec in k8s.
@@ -130,12 +131,13 @@ func matchesHostPattern(pattern, host string) bool {
 
 func getLabelsForIngressPath(ig *Ingress, scheme, host, path string) map[string]string {
 	m := map[string]string{
-		"__address__":                      host,
-		"__meta_kubernetes_namespace":      ig.Metadata.Namespace,
-		"__meta_kubernetes_ingress_name":   ig.Metadata.Name,
-		"__meta_kubernetes_ingress_scheme": scheme,
-		"__meta_kubernetes_ingress_host":   host,
-		"__meta_kubernetes_ingress_path":   path,
+		"__address__":                          host,
+		"__meta_kubernetes_namespace":          ig.Metadata.Namespace,
+		"__meta_kubernetes_ingress_name":       ig.Metadata.Name,
+		"__meta_kubernetes_ingress_scheme":     scheme,
+		"__meta_kubernetes_ingress_host":       host,
+		"__meta_kubernetes_ingress_path":       path,
+		"__meta_kubernetes_ingress_class_name": ig.Spec.IngressClassName,
 	}
 	ig.Metadata.registerLabelsAndAnnotations("__meta_kubernetes_ingress", m)
 	return m

lib/promscrape/discovery/kubernetes/ingress_test.go

@@ -78,7 +78,8 @@ func TestParseIngressListSuccess(t *testing.T) {
 	  {
             "host": "foobar"
           }
-	]
+	],
+	"ingressClassName": "foo-class"
       },
       "status": {
         "loadBalancer": {
@@ -107,11 +108,12 @@ func TestParseIngressListSuccess(t *testing.T) {
 			"__address__": "foobar",
 			"__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration":        `{"apiVersion":"networking.k8s.io/v1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n",
 			"__meta_kubernetes_ingress_annotationpresent_kubectl_kubernetes_io_last_applied_configuration": "true",
-			"__meta_kubernetes_ingress_host":   "foobar",
-			"__meta_kubernetes_ingress_name":   "test-ingress",
-			"__meta_kubernetes_ingress_path":   "/",
-			"__meta_kubernetes_ingress_scheme": "http",
-			"__meta_kubernetes_namespace":      "default",
+			"__meta_kubernetes_ingress_host":       "foobar",
+			"__meta_kubernetes_ingress_name":       "test-ingress",
+			"__meta_kubernetes_ingress_path":       "/",
+			"__meta_kubernetes_ingress_scheme":     "http",
+			"__meta_kubernetes_ingress_class_name": "foo-class",
+			"__meta_kubernetes_namespace":          "default",
 		}),
 	}
 	if !areEqualLabelss(sortedLabelss, expectedLabelss) {

lib/promscrape/scrapework.go

@@ -585,9 +585,9 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	scrapeResponseSize.Update(float64(sbr.bodyLen))
 	up := 1
 	if err != nil {
-		if samplesScraped == 0 {
-			up = 0
-		}
+		// Mark the scrape as failed even if it already read and pushed some samples
+		// to remote storage. This makes the logic compatible with Prometheus.
+		up = 0
 		scrapesFailed.Inc()
 	}
 	seriesAdded := 0

lib/storage/index_db.go

@@ -815,9 +815,13 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
 	if err != nil {
 		return err
 	}
-	if filter != nil && filter.Len() == 0 {
-		qt.Printf("found zero label names for filter=%s", tfss)
-		return nil
+	if filter != nil && filter.Len() <= 100e3 {
+		// It is faster to obtain label names by metricIDs from the filter
+		// instead of scanning the inverted index for the matching filters.
+		// This would help https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978
+		metricIDs := filter.AppendTo(nil)
+		qt.Printf("sort %d metricIDs", len(metricIDs))
+		return is.getLabelNamesForMetricIDs(qt, metricIDs, lns, maxLabelNames)
 	}
 	var prevLabelName []byte
 	ts := &is.ts
@@ -877,6 +881,41 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
 	return nil
 }
 
+func (is *indexSearch) getLabelNamesForMetricIDs(qt *querytracer.Tracer, metricIDs []uint64, lns map[string]struct{}, maxLabelNames int) error {
+	lns["__name__"] = struct{}{}
+	var mn MetricName
+	foundLabelNames := 0
+	var buf []byte
+	for _, metricID := range metricIDs {
+		var err error
+		buf, err = is.searchMetricNameWithCache(buf[:0], metricID)
+		if err != nil {
+			if err == io.EOF {
+				// It is likely the metricID->metricName entry didn't propagate to inverted index yet.
+				// Skip this metricID for now.
+				continue
+			}
+			return fmt.Errorf("cannot find metricName by metricID %d: %w", metricID, err)
+		}
+		if err := mn.Unmarshal(buf); err != nil {
+			return fmt.Errorf("cannot unmarshal metricName %q: %w", buf, err)
+		}
+		for _, tag := range mn.Tags {
+			_, ok := lns[string(tag.Key)]
+			if !ok {
+				foundLabelNames++
+				lns[string(tag.Key)] = struct{}{}
+				if len(lns) >= maxLabelNames {
+					qt.Printf("hit the limit on the number of unique label names: %d", maxLabelNames)
+					return nil
+				}
+			}
+		}
+	}
+	qt.Printf("get %d distinct label names from %d metricIDs", foundLabelNames, len(metricIDs))
+	return nil
+}
+
 // SearchLabelValuesWithFiltersOnTimeRange returns label values for the given labelName, tfss and tr.
 func (db *indexDB) SearchLabelValuesWithFiltersOnTimeRange(qt *querytracer.Tracer, labelName string, tfss []*TagFilters, tr TimeRange,
 	maxLabelValues, maxMetrics int, deadline uint64) ([]string, error) {
@@ -972,9 +1011,13 @@ func (is *indexSearch) searchLabelValuesWithFiltersOnDate(qt *querytracer.Tracer
 	if err != nil {
 		return err
 	}
-	if filter != nil && filter.Len() == 0 {
-		qt.Printf("found zero label values for filter=%s", tfss)
-		return nil
+	if filter != nil && filter.Len() < 100e3 {
+		// It is faster to obtain label values by metricIDs from the filter
+		// instead of scanning the inverted index for the matching filters.
+		// This would help https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2978
+		metricIDs := filter.AppendTo(nil)
+		qt.Printf("sort %d metricIDs", len(metricIDs))
+		return is.getLabelValuesForMetricIDs(qt, lvs, labelName, metricIDs, maxLabelValues)
 	}
 	if labelName == "__name__" {
 		// __name__ label is encoded as empty string in indexdb.
@@ -1033,6 +1076,42 @@ func (is *indexSearch) searchLabelValuesWithFiltersOnDate(qt *querytracer.Tracer
 	return nil
 }
 
+func (is *indexSearch) getLabelValuesForMetricIDs(qt *querytracer.Tracer, lvs map[string]struct{}, labelName string, metricIDs []uint64, maxLabelValues int) error {
+	if labelName == "" {
+		labelName = "__name__"
+	}
+	var mn MetricName
+	foundLabelValues := 0
+	var buf []byte
+	for _, metricID := range metricIDs {
+		var err error
+		buf, err = is.searchMetricNameWithCache(buf[:0], metricID)
+		if err != nil {
+			if err == io.EOF {
+				// It is likely the metricID->metricName entry didn't propagate to inverted index yet.
+				// Skip this metricID for now.
+				continue
+			}
+			return fmt.Errorf("cannot find metricName by metricID %d: %w", metricID, err)
+		}
+		if err := mn.Unmarshal(buf); err != nil {
+			return fmt.Errorf("cannot unmarshal metricName %q: %w", buf, err)
+		}
+		tagValue := mn.GetTagValue(labelName)
+		_, ok := lvs[string(tagValue)]
+		if !ok {
+			foundLabelValues++
+			lvs[string(tagValue)] = struct{}{}
+			if len(lvs) >= maxLabelValues {
+				qt.Printf("hit the limit on the number of unique label values for label %q: %d", labelName, maxLabelValues)
+				return nil
+			}
+		}
+	}
+	qt.Printf("get %d distinct values for label %q from %d metricIDs", foundLabelValues, labelName, len(metricIDs))
+	return nil
+}
+
 // SearchTagValueSuffixes returns all the tag value suffixes for the given tagKey and tagValuePrefix on the given tr.
 //
 // This allows implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find or similar APIs.

lib/storage/partition.go

@@ -569,7 +569,7 @@ func (pt *partition) addRowsPart(rows []rawRow) {
 		atomic.AddUint64(&pt.smallAssistedMerges, 1)
 		return
 	}
-	if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) {
+	if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
 		return
 	}
 	logger.Panicf("FATAL: cannot merge small parts: %s", err)
@@ -871,7 +871,7 @@ func hasActiveMerges(pws []*partWrapper) bool {
 
 var (
 	bigMergeWorkersCount   = getDefaultMergeConcurrency(4)
-	smallMergeWorkersCount = getDefaultMergeConcurrency(8)
+	smallMergeWorkersCount = getDefaultMergeConcurrency(16)
 )
 
 func getDefaultMergeConcurrency(max int) int {
@@ -956,7 +956,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
 			// The merger has been stopped.
 			return nil
 		}
-		if !errors.Is(err, errNothingToMerge) {
+		if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) {
 			return err
 		}
 		if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
@@ -1012,11 +1012,13 @@ func (pt *partition) canBackgroundMerge() bool {
 	return atomic.LoadUint32(pt.isReadOnly) == 0
 }
 
+var errReadOnlyMode = fmt.Errorf("storage is in readonly mode")
+
 func (pt *partition) mergeBigParts(isFinal bool) error {
 	if !pt.canBackgroundMerge() {
 		// Do not perform merge in read-only mode, since this may result in disk space shortage.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
-		return nil
+		return errReadOnlyMode
 	}
 	maxOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)
 
@@ -1032,7 +1034,7 @@ func (pt *partition) mergeSmallParts(isFinal bool) error {
 	if !pt.canBackgroundMerge() {
 		// Do not perform merge in read-only mode, since this may result in disk space shortage.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
-		return nil
+		return errReadOnlyMode
 	}
 	// Try merging small parts to a big part at first.
 	maxBigPartOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)