app/vmalert: tolerate 400 evaluation error with -replay.continueWithExecutionErr in replay mode

Follow up https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11348

VM would then return `400` for missing or incorrect parameters, and
`422` for evaluation errors such as `duplicate time series on the left
side`, or when the request hits a [resource
limit](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#resource-usage-limits).
`-replay.continueWithExecutionErr` should work with both status codes.

app/victoria-metrics/main.go

@@ -63,6 +63,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -170,3 +171,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
 `
 	flagutil.Usage(s)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+	vmselect.InitSecretFlags()
+}

app/vmagent/main.go

@@ -84,7 +84,7 @@ var (
 	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
 
-	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", false, "Enables multitenancy via HTTP headers. "+
+	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", true, "Enables multitenancy via HTTP headers. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy")
 )
 
@@ -115,7 +115,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	remotewrite.InitSecretFlags()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 	opentelemetry.Init()
@@ -843,3 +843,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
 `
 	flagutil.Usage(s)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	remotewrite.InitSecretFlags()
+	pushmetrics.InitSecretFlags()
+}

app/vmagent/remotewrite/remotewrite.go

@@ -156,7 +156,8 @@ var maxQueues = cgroup.AvailableCPUs() * 16
 
 const persistentQueueDirname = "persistent-queue"
 
-// InitSecretFlags must be called after flag.Parse and before any logging.
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.

app/vmalert/datasource/init.go

@@ -60,7 +60,8 @@ var (
 		`Only valid for VictoriaMetrics as the datasource.`)
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showDatasourceURL {
 		flagutil.RegisterSecretFlag("datasource.url")

app/vmalert/main.go

@@ -88,10 +88,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	remoteread.InitSecretFlags()
-	remotewrite.InitSecretFlags()
-	datasource.InitSecretFlags()
-	notifier.InitSecretFlags()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -438,3 +435,12 @@ func getLastConfigError() error {
 	defer lastConfigErrMu.RUnlock()
 	return lastConfigErr
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	remoteread.InitSecretFlags()
+	remotewrite.InitSecretFlags()
+	datasource.InitSecretFlags()
+	notifier.InitSecretFlags()
+	pushmetrics.InitSecretFlags()
+}

app/vmalert/notifier/init.go

@@ -189,7 +189,8 @@ func Init(extLabels map[string]string, extURL string) error {
 	return nil
 }
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showNotifierURL {
 		flagutil.RegisterSecretFlag("notifier.url")

app/vmalert/remoteread/init.go

@@ -56,7 +56,8 @@ var (
 	oauth2Scopes   = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteReadURL {
 		flagutil.RegisterSecretFlag("remoteRead.url")

app/vmalert/remotewrite/init.go

@@ -57,7 +57,8 @@ var (
 	oauth2Scopes   = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		flagutil.RegisterSecretFlag("remoteWrite.url")

app/vmalert/replay.go

@@ -30,7 +30,8 @@ var (
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
 		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
-	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
+	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, "+
+		"which can happen due to an expression syntax error or a resource limit being hit.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {

app/vmalert/rule/rule.go

@@ -132,9 +132,10 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
 		var esc *httpserver.ErrorWithStatusCode
 		if errors.As(err, &esc) {
 			statusCode := esc.StatusCode
-			// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
-			// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
-			if statusCode == http.StatusUnprocessableEntity {
+			// if the status code is 400 or 422, the query failed due to reasons such as an expression syntax error or a resource limit being hit,
+			// rather than datasource unavailability.
+			// Continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
+			if statusCode == http.StatusUnprocessableEntity || statusCode == http.StatusBadRequest {
 				if continueWithExecutionErr {
 					logger.Errorf("rule %q: %s", r, err)
 					return 0, nil

app/vmauth/main.go

@@ -96,6 +96,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -911,3 +912,8 @@ func slowdownUnauthorizedResponse(r *http.Request) {
 	}
 	timerpool.Put(t)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmbackup/main.go

@@ -47,9 +47,8 @@ func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
-	flagutil.RegisterSecretFlag("snapshot.createURL")
-	flagutil.RegisterSecretFlag("snapshot.deleteURL")
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -273,3 +272,10 @@ func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
 	}
 	return fs, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	flagutil.RegisterSecretFlag("snapshot.createURL")
+	flagutil.RegisterSecretFlag("snapshot.deleteURL")
+	pushmetrics.InitSecretFlags()
+}

app/vmctl/main.go

@@ -47,6 +47,7 @@ func main() {
 	start := time.Now()
 	beforeFn := func(c *cli.Context) error {
 		flag.Parse()
+		initSecretFlags()
 		logger.Init()
 		isSilent = c.Bool(globalSilent)
 		if c.Bool(globalDisableProgressBar) {
@@ -619,3 +620,8 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		Backoff:            bf,
 	}, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmrestore/main.go

@@ -38,6 +38,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -112,3 +113,8 @@ func newSrcFS(ctx context.Context) (common.RemoteFS, error) {
 	}
 	return fs, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmselect/main.go

@@ -59,6 +59,12 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
 	initVMUIConfig()
 
 	vmalertproxy.Init(*vmalertProxyURL)
+
+}
+
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
+func InitSecretFlags() {
 	flagutil.RegisterSecretFlag("vmalert.proxyURL")
 }
 

apptest/client.go

@@ -135,7 +135,7 @@ func tenantViaURL(addr, prefix, tenant, suffix string) string {
 }
 
 // tenantViaHeaders returns path in cluster's URL format where tenant is omitted in URL
-// Only supported if -enableMultitenancyViaHeaders is specified
+// Only supported if -enableMultitenancyViaHeaders is enabled
 func tenantViaHeaders(addr, prefix, suffix string) string {
 	return fmt.Sprintf("http://%s/%s/%s", addr, prefix, suffix)
 }

apptest/tests/multitenancy_via_headers_test.go

@@ -25,12 +25,10 @@ func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
 	})
 	vminsert := tc.MustStartVminsert("vminsert", []string{
 		"-storageNode=" + vmstorage.VminsertAddr(),
-		"-enableMultitenancyViaHeaders",
 	})
 	vmselect := tc.MustStartVmselect("vmselect", []string{
 		"-storageNode=" + vmstorage.VmselectAddr(),
 		"-search.tenantCacheExpireDuration=0",
-		"-enableMultitenancyViaHeaders",
 	})
 
 	multitenant := make(http.Header)

apptest/tests/vmagent_remotewrite_test.go

@@ -594,7 +594,6 @@ func TestSingleVMAgentMultitenancy(t *testing.T) {
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-multitenancy",
 		"-enableMultitenantHandlers",
-		"-enableMultitenancyViaHeaders",
 	})
 
 	vmagent.APIV1ImportPrometheus(t, []string{

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -95,16 +95,15 @@ See also multitenancy [via headers](#multitenancy-via-headers) and [via labels](
 
 ### Multitenancy via headers
 
-By default, VictoriaMetrics allows specifying `accountID` and `projectID` only in the request URL.
+With `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag enabled (enabled by default {{% available_from "#" %}})
+tenant ID can be specified via HTTP headers `AccountID` and `ProjectID`. This flag needs to be enabled on vminserts and vmselects.
 
-Set `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag to support 
-specifying `accountID` and `projectID` via HTTP headers `AccountID` and `ProjectID` respectively.
-This flag needs to be specified separately for vminserts and vmselects.
-
-When `--enableMultitenancyViaHeaders` is enabled, [URL format](#url-format) can be simplified to the following:
+With `--enableMultitenancyViaHeaders` enabled [URL format](#url-format) can be simplified to the following:
 - `http://<vminsert>:8480/insert/<suffix>` for writes
 - `http://<vmselect>:8481/select/prometheus/<suffix>` for reads
 
+> Set --enableMultitenancyViaHeaders=false to disable simplified URL format.
+
 For example, the following query will only select metric `up` from `accountID=2` and `projectID=3`:
 ```
 curl 'https://<vmselect>:8481/select/prometheus/api/v1/query' \

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+**Update Note 1:** `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: default value of `-enableMultitenancyViaHeaders` command-line flag has changed from `false` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default. With this change, mentioned components will start supporting URLs with omitted tenant ID in the path: `https://<vmselect>:8481/select/prometheus/api/v1/query` will become a valid URL. To disable multitenancy via headers and simplified URLs set `--enableMultitenancyViaHeaders=false` on vmagent, vminsert and vmselect. 
+
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): extend `-replay.continueWithExecutionErr` to also handle the `400 Bad Request` response code, since it is used for Prometheus querying API requests when request parameters are missing or incorrect. See [#11352](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11352).
+* FEATURE: `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: set default value of `-enableMultitenancyViaHeaders` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default, aligning VictoriaMetrics multitenancy behavior with [multitenancy in VictoriaLogs](https://docs.victoriametrics.com/victorialogs/#multitenancy). See related ticket [#11365](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11365).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add an option to customize the favicon color. This makes it easier to distinguish between different installations opened in multiple browser tabs. See [#11329](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11329).
 
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly parse small fractional Unix timestamps in timestamp args such as `start` and `end` in `/api/v1/query_range` and `--vm-native-filter-time-start` and `--vm-native-filter-time-end` in `vmctl`. Previously, fractional Unix timestamps with the integer part below `9223372` were interpreted with the wrong unit, for example `12.0` was parsed as `12000` seconds instead of `12` seconds. See [#11324](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11324).

docs/victoriametrics/vmagent.md

@@ -642,7 +642,7 @@ specified via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` c
 
 vmagent can write data to multiple distinct tenants if:
 * its `-remoteWrite.url` points to the [VictoriaMetrics cluster multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
-* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are both set
+* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` (enabled by default {{% available_from "#" %}}) command-line flags are both set
 * clients ingest data into vmagent with the tenants specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}
 
 ```mermaid

docs/victoriametrics/vmalert_common_flags.md

@@ -405,7 +405,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -remoteWrite.url string
      Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.continueWithExecutionErr
-     Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
+     Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int

lib/pushmetrics/pushmetrics.go

@@ -24,7 +24,9 @@ var (
 	disableCompression = flag.Bool("pushmetrics.disableCompression", false, "Whether to disable request body compression when pushing metrics to every -pushmetrics.url")
 )
 
-func init() {
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
+func InitSecretFlags() {
 	// The -pushmetrics.url flag can contain basic auth creds, so it mustn't be visible when exposing the flags.
 	flagutil.RegisterSecretFlag("pushmetrics.url")
 }