vmagent: maintenance mode support

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11191

The default `-search.maxConcurrentRequests` for cluster `vmselect` was capped at 16 via `min(2*cgroup.AvailableCPUs(), 16)`. This meant larger vmselect nodes kept the same default concurrency once they had more than 8 CPUs.

This changes the default to `2*cgroup.AvailableCPUs()`, matching the style already used by `vmstorage` and
`clusternative.maxConcurrentRequests`.

Capping it at 16 makes sense for vmsingle, since it handles both ingestion and querying, with ingestion being the higher priority. Select has no such limitation and should scale to all available resources.

---------

Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>

app/victoria-metrics/main.go

@@ -63,6 +63,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -170,3 +171,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
 `
 	flagutil.Usage(s)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+	vmselect.InitSecretFlags()
+}

app/vmagent/main.go

@@ -2,11 +2,13 @@ package main
 
 import (
 	"embed"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -84,7 +86,7 @@ var (
 	maxLabelNameLen        = flag.Int("maxLabelNameLen", 0, "The maximum length of label names in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 0, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
 
-	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", false, "Enables multitenancy via HTTP headers. "+
+	enableMultitenancyViaHeaders = flag.Bool("enableMultitenancyViaHeaders", true, "Enables multitenancy via HTTP headers. "+
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy")
 )
 
@@ -115,7 +117,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	remotewrite.InitSecretFlags()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 	opentelemetry.Init()
@@ -549,6 +551,33 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		procutil.SelfSIGHUP()
 		w.WriteHeader(http.StatusOK)
 		return true
+	case "/remotewrite/maintenance":
+		if !remotewrite.CheckMaintenanceAuthKey(w, r) {
+			return true
+		}
+		remoteWriteMaintenanceRequests.Inc()
+		if v := r.FormValue("enable"); v != "" {
+			enable, err := strconv.ParseBool(v)
+			if err != nil {
+				httpserver.Errorf(w, r, "cannot parse `enable` query arg: %s", err)
+				return true
+			}
+			rwURL := r.FormValue("url")
+			if rwURL == "" {
+				httpserver.Errorf(w, r, "missing `url` query arg; it must match the `url` label of vmagent_remotewrite_* metrics for the target -remoteWrite.url, "+
+					"or be set to `*` to target all the configured -remoteWrite.url destinations")
+				return true
+			}
+			if matched := remotewrite.SetMaintenanceMode(rwURL, enable); matched == 0 {
+				httpserver.Errorf(w, r, "no -remoteWrite.url destinations match `url=%s`", rwURL)
+				return true
+			}
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"maintenance": remotewrite.GetMaintenanceMode(),
+		})
+		return true
 	case "/ready":
 		if rdy := promscrape.PendingScrapeConfigs.Load(); rdy > 0 {
 			errMsg := fmt.Sprintf("waiting for scrapes to init, left: %d", rdy)
@@ -833,6 +862,8 @@ var (
 	remoteWriteStatusURLRelabelConfigRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/status/remotewrite-url-relabel-config"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/-/reload"}`)
+
+	remoteWriteMaintenanceRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/remotewrite/maintenance"}`)
 )
 
 func usage() {
@@ -843,3 +874,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
 `
 	flagutil.Usage(s)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	remotewrite.InitSecretFlags()
+	pushmetrics.InitSecretFlags()
+}

app/vmagent/prometheusimport/request_handler_test.go

@@ -52,6 +52,7 @@ func setUp() {
 
 func tearDown() {
 	protoparserutil.StopUnmarshalWorkers()
+	remotewrite.Stop()
 	srv.Close()
 	logger.ResetOutputForTest()
 	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()

app/vmagent/remotewrite/client.go

@@ -97,6 +97,8 @@ type client struct {
 	useVMProto          atomic.Bool
 	canDowngradeVMProto atomic.Bool
 
+	maintenanceMode atomic.Bool
+
 	fq *persistentqueue.FastQueue
 	hc *http.Client
 
@@ -322,6 +324,12 @@ func (c *client) runWorker(readBlock func(dst []byte) ([]byte, bool)) {
 	var block []byte
 	ch := make(chan bool, 1)
 	for {
+		if c.maintenanceMode.Load() {
+			if !c.waitForMaintenanceModeOff() {
+				return
+			}
+		}
+
 		block, ok = readBlock(block[:0])
 		if !ok {
 			return
@@ -339,6 +347,14 @@ func (c *client) runWorker(readBlock func(dst []byte) ([]byte, bool)) {
 			}
 			// Return unsent block to the queue.
 			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			select {
+			case <-c.stopCh:
+				// c must be stopped.
+			default:
+				// sendBlock returned false because maintenance mode is enabled, not because
+				// c is stopping. Keep the worker alive so it resumes once maintenance mode is disabled.
+				continue
+			}
 			return
 		case <-c.stopCh:
 			// c must be stopped. Wait up to 5 seconds for the in-flight request to complete.
@@ -363,6 +379,23 @@ func (c *client) runWorker(readBlock func(dst []byte) ([]byte, bool)) {
 	}
 }
 
+// waitForMaintenanceModeOff blocks while maintenance mode is enabled for c.
+//
+// It returns false only if c.stopCh is closed while waiting.
+func (c *client) waitForMaintenanceModeOff() bool {
+	t := time.NewTicker(time.Second)
+	defer t.Stop()
+
+	for c.maintenanceMode.Load() {
+		select {
+		case <-t.C:
+		case <-c.stopCh:
+			return false
+		}
+	}
+	return true
+}
+
 func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
 	req, err := c.newRequest(url, body)
 	if err != nil {
@@ -421,7 +454,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
 //
-// The function returns false only if c.stopCh is closed.
+// The function returns false if c.stopCh is closed or if maintenance mode is enabled for c.
 // Otherwise, it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
@@ -429,6 +462,10 @@ func (c *client) sendBlockHTTP(block []byte) bool {
 	retriesCount := 0
 
 again:
+	if c.maintenanceMode.Load() {
+		return false
+	}
+
 	startTime := time.Now()
 	resp, err := c.doRequest(c.remoteWriteURL, block)
 	c.requestDuration.UpdateDuration(startTime)
@@ -511,12 +548,7 @@ again:
 	// Handle response
 	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
-	if err != nil {
-		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
-	} else {
-		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
-			"re-sending the block in %s", len(block), c.sanitizedURL, retriesCount, statusCode, body, bt.CurrentDelay())
-	}
+	logUnexpectedStatusCode(block, c.sanitizedURL, statusCode, retriesCount, bt.CurrentDelay(), retryAfterHeader > 0, body, err)
 	if !bt.Wait(c.stopCh) {
 		return false
 	}
@@ -552,6 +584,26 @@ func (c *client) drainInMemoryQueue(stopCtx context.Context, block []byte) {
 
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
+var remoteWriteUnexpectedStatusLogger = logger.WithThrottler("remoteWriteUnexpectedStatus", 5*time.Second)
+
+func logUnexpectedStatusCode(block []byte, sanitizedURL string, statusCode, retriesCount int, retryDelay time.Duration, isExpectedBackoff bool, body []byte, bodyErr error) {
+	if bodyErr != nil {
+		remoteWriteUnexpectedStatusLogger.Errorf("cannot read response body from %q during retry #%d: %s", sanitizedURL, retriesCount, bodyErr)
+		return
+	}
+
+	msg := fmt.Sprintf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
+		"re-sending the block in %s", len(block), sanitizedURL, retriesCount, statusCode, body, retryDelay)
+
+	if isExpectedBackoff {
+		// The remote storage explicitly signaled backoff duration via the Retry-After header,
+		// so this isn't an anomaly worth an ERROR log.
+		remoteWriteUnexpectedStatusLogger.Warnf("%s", msg)
+		return
+	}
+
+	remoteWriteUnexpectedStatusLogger.Errorf("%s", msg)
+}
 
 // repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
 //

app/vmagent/remotewrite/remotewrite.go

@@ -113,6 +113,7 @@ var (
 		"Multiple label names should be separated by `^^`, e.g. \"job^^instance,ip\". "+
 		"Can be combined with -remoteWrite.mdx.enable to hide sensitive label values in VictoriaMetrics self-monitoring metrics. "+
 		"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#obfuscating-label-values")
+	maintenanceAuthKey = flagutil.NewPassword("remoteWrite.maintenanceAuthKey", "Auth key for /remotewrite/maintenance http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 )
 
 var (
@@ -141,6 +142,42 @@ func MultitenancyEnabled() bool {
 	return *enableMultitenantHandlers
 }
 
+// SetMaintenanceMode enables or disables maintenance mode for the remote write destination(s)
+// identified by sanitizedURL, which must match the `url` label of the corresponding
+// vmagent_remotewrite_* metrics. Pass "*" to target all the configured -remoteWrite.url destinations.
+//
+// While a destination is in maintenance mode, vmagent doesn't attempt to send data to it at all -
+// it stops draining that destination's queue; buffering remains subject to the configured queue limits.
+//
+// It returns the number of destinations matched by sanitizedURL.
+func SetMaintenanceMode(sanitizedURL string, enabled bool) int {
+	matched := 0
+	for _, rwctx := range rwctxsGlobal {
+		if sanitizedURL != "*" && rwctx.c.sanitizedURL != sanitizedURL {
+			continue
+		}
+		rwctx.c.maintenanceMode.Store(enabled)
+		matched++
+	}
+	return matched
+}
+
+// GetMaintenanceMode returns the maintenance mode status for each configured -remoteWrite.url,
+// keyed by its sanitized URL.
+func GetMaintenanceMode() map[string]bool {
+	m := make(map[string]bool, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
+		m[rwctx.c.sanitizedURL] = rwctx.c.maintenanceMode.Load()
+	}
+	return m
+}
+
+// CheckMaintenanceAuthKey verifies the authKey query arg for the /remotewrite/maintenance http endpoint
+// against -remoteWrite.maintenanceAuthKey. See httpserver.CheckAuthFlag for the semantics of the return value.
+func CheckMaintenanceAuthKey(w http.ResponseWriter, r *http.Request) bool {
+	return httpserver.CheckAuthFlag(w, r, maintenanceAuthKey)
+}
+
 // Contains the current relabelConfigs.
 var allRelabelConfigs atomic.Pointer[relabelConfigs]
 
@@ -156,7 +193,8 @@ var maxQueues = cgroup.AvailableCPUs() * 16
 
 const persistentQueueDirname = "persistent-queue"
 
-// InitSecretFlags must be called after flag.Parse and before any logging.
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
@@ -245,6 +283,8 @@ func Init() {
 	dropDanglingQueues()
 
 	// Start config reloader.
+	configReloaderStopCh = make(chan struct{})
+	configReloaderWG = sync.WaitGroup{}
 	configReloaderWG.Go(func() {
 		for {
 			select {
@@ -331,7 +371,7 @@ func initRemoteWriteCtxs(urls []string) {
 }
 
 var (
-	configReloaderStopCh = make(chan struct{})
+	configReloaderStopCh chan struct{}
 	configReloaderWG     sync.WaitGroup
 )
 

app/vmalert/datasource/init.go

@@ -60,7 +60,8 @@ var (
 		`Only valid for VictoriaMetrics as the datasource.`)
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showDatasourceURL {
 		flagutil.RegisterSecretFlag("datasource.url")

app/vmalert/main.go

@@ -88,10 +88,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
-	remoteread.InitSecretFlags()
-	remotewrite.InitSecretFlags()
-	datasource.InitSecretFlags()
-	notifier.InitSecretFlags()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -438,3 +435,12 @@ func getLastConfigError() error {
 	defer lastConfigErrMu.RUnlock()
 	return lastConfigErr
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	remoteread.InitSecretFlags()
+	remotewrite.InitSecretFlags()
+	datasource.InitSecretFlags()
+	notifier.InitSecretFlags()
+	pushmetrics.InitSecretFlags()
+}

app/vmalert/notifier/init.go

@@ -189,7 +189,8 @@ func Init(extLabels map[string]string, extURL string) error {
 	return nil
 }
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showNotifierURL {
 		flagutil.RegisterSecretFlag("notifier.url")

app/vmalert/remoteread/init.go

@@ -56,7 +56,8 @@ var (
 	oauth2Scopes   = flag.String("remoteRead.oauth2.scopes", "", "Optional OAuth2 scopes to use for -remoteRead.url. Scopes must be delimited by ';'.")
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteReadURL {
 		flagutil.RegisterSecretFlag("remoteRead.url")

app/vmalert/remotewrite/init.go

@@ -57,7 +57,8 @@ var (
 	oauth2Scopes   = flag.String("remoteWrite.oauth2.scopes", "", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'.")
 )
 
-// InitSecretFlags must be called after flag.Parse and before any logging
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
 func InitSecretFlags() {
 	if !*showRemoteWriteURL {
 		flagutil.RegisterSecretFlag("remoteWrite.url")

app/vmalert/replay.go

@@ -30,7 +30,8 @@ var (
 		"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
 	ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
 		"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
-	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
+	continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, "+
+		"which can happen due to an expression syntax error or a resource limit being hit.")
 )
 
 func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {

app/vmalert/rule/group.go

@@ -290,6 +290,8 @@ func (g *Group) updateWith(newGroup *Group) error {
 	g.Headers = newGroup.Headers
 	g.NotifierHeaders = newGroup.NotifierHeaders
 	g.Labels = newGroup.Labels
+	g.EvalDelay = newGroup.EvalDelay
+	g.evalAlignment = newGroup.evalAlignment
 	g.Limit = newGroup.Limit
 	g.checksum = newGroup.checksum
 	g.Rules = newRules
@@ -337,7 +339,7 @@ func (g *Group) Init() {
 		i := g.Interval.Seconds()
 		return i
 	})
-	g.metrics.iterationLimit = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_rule_group_results_limit{%s}`, labels), func() float64 {
+	g.metrics.iterationLimit = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_group_rule_results_limit{%s}`, labels), func() float64 {
 		g.mu.RLock()
 		limit := g.Limit
 		g.mu.RUnlock()

app/vmalert/rule/group_test.go

@@ -78,6 +78,12 @@ func TestUpdateWith(t *testing.T) {
 		if g.Debug != expect.Debug {
 			t.Fatalf("expected to have debug %v; got %v", expect.Debug, g.Debug)
 		}
+		if !durationPtrEqual(g.EvalDelay, expect.EvalDelay) {
+			t.Fatalf("expected to have eval_delay %v; got %v", expect.EvalDelay, g.EvalDelay)
+		}
+		if !boolPtrEqual(g.evalAlignment, expect.evalAlignment) {
+			t.Fatalf("expected to have eval_alignment %v; got %v", expect.evalAlignment, g.evalAlignment)
+		}
 	}
 
 	// new rule
@@ -237,6 +243,37 @@ func TestUpdateWith(t *testing.T) {
 			{Alert: "foo1", Debug: &debug},
 		},
 	})
+
+	// update group evaluation settings
+	evalDelay := promutil.NewDuration(time.Minute)
+	evalAlignment := false
+	f(config.Group{
+		Rules: []config.Rule{{
+			Record: "foo",
+			Expr:   "max(up)",
+		}},
+	}, config.Group{
+		EvalDelay:     evalDelay,
+		EvalAlignment: &evalAlignment,
+		Rules: []config.Rule{{
+			Record: "foo",
+			Expr:   "min(up)",
+		}},
+	})
+}
+
+func durationPtrEqual(a, b *time.Duration) bool {
+	if a == nil || b == nil {
+		return a == b
+	}
+	return *a == *b
+}
+
+func boolPtrEqual(a, b *bool) bool {
+	if a == nil || b == nil {
+		return a == b
+	}
+	return *a == *b
 }
 
 func TestUpdateDuringRandSleep(t *testing.T) {

app/vmalert/rule/rule.go

@@ -132,9 +132,10 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
 		var esc *httpserver.ErrorWithStatusCode
 		if errors.As(err, &esc) {
 			statusCode := esc.StatusCode
-			// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
-			// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
-			if statusCode == http.StatusUnprocessableEntity {
+			// if the status code is 400 or 422, the query failed due to reasons such as an expression syntax error or a resource limit being hit,
+			// rather than datasource unavailability.
+			// Continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
+			if statusCode == http.StatusUnprocessableEntity || statusCode == http.StatusBadRequest {
 				if continueWithExecutionErr {
 					logger.Errorf("rule %q: %s", r, err)
 					return 0, nil

app/vmauth/main.go

@@ -96,6 +96,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -911,3 +912,8 @@ func slowdownUnauthorizedResponse(r *http.Request) {
 	}
 	timerpool.Put(t)
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmbackup/main.go

@@ -47,9 +47,8 @@ func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
-	flagutil.RegisterSecretFlag("snapshot.createURL")
-	flagutil.RegisterSecretFlag("snapshot.deleteURL")
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -273,3 +272,10 @@ func newRemoteOriginFS(ctx context.Context) (common.RemoteFS, error) {
 	}
 	return fs, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	flagutil.RegisterSecretFlag("snapshot.createURL")
+	flagutil.RegisterSecretFlag("snapshot.deleteURL")
+	pushmetrics.InitSecretFlags()
+}

app/vmctl/main.go

@@ -47,6 +47,7 @@ func main() {
 	start := time.Now()
 	beforeFn := func(c *cli.Context) error {
 		flag.Parse()
+		initSecretFlags()
 		logger.Init()
 		isSilent = c.Bool(globalSilent)
 		if c.Bool(globalDisableProgressBar) {
@@ -619,3 +620,8 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		Backoff:            bf,
 	}, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmrestore/main.go

@@ -38,6 +38,7 @@ func main() {
 	flag.CommandLine.SetOutput(os.Stdout)
 	flag.Usage = usage
 	envflag.Parse()
+	initSecretFlags()
 	buildinfo.Init()
 	logger.Init()
 
@@ -112,3 +113,8 @@ func newSrcFS(ctx context.Context) (common.RemoteFS, error) {
 	}
 	return fs, nil
 }
+
+// initSecretFlags manages the secret flags for this app and must be called after flag parsing and before logger init.
+func initSecretFlags() {
+	pushmetrics.InitSecretFlags()
+}

app/vmselect/main.go

@@ -59,6 +59,12 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
 	initVMUIConfig()
 
 	vmalertproxy.Init(*vmalertProxyURL)
+
+}
+
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
+func InitSecretFlags() {
 	flagutil.RegisterSecretFlag("vmalert.proxyURL")
 }
 

app/vmselect/prometheus/prometheus.go

@@ -516,7 +516,7 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
 	cp.deadline = searchutil.GetDeadlineForDelete(r, startTime)
 
 	if !cp.IsDefaultTimeRange() {
-		return fmt.Errorf("start=%d and end=%d args aren't supported. Remove these args from the query in order to delete all the matching metrics", cp.start, cp.end)
+		return fmt.Errorf("delete API does not support specific time ranges using start and end args, the series can only be deleted completely")
 	}
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxDeleteSeries)
 	deletedCount, err := netstorage.DeleteSeries(nil, sq, cp.deadline)
@@ -540,11 +540,11 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
 
 	cp, err := getCommonParamsForLabelsAPI(r, startTime, false)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
 
@@ -584,7 +584,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 
 	cp, err := getCommonParams(r, startTime, false)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	cp.deadline = searchutil.GetDeadlineForStatusRequest(r, startTime)
 
@@ -596,7 +596,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		} else {
 			t, err := time.Parse("2006-01-02", dateStr)
 			if err != nil {
-				return fmt.Errorf("cannot parse `date` arg %q: %w", dateStr, err)
+				return httpserver.InvalidParamError(fmt.Errorf("cannot parse `date` arg %q: %w", dateStr, err))
 			}
 			date = uint64(t.Unix()) / secsPerDay
 		}
@@ -607,7 +607,7 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	if len(topNStr) > 0 {
 		n, err := strconv.Atoi(topNStr)
 		if err != nil {
-			return fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err))
 		}
 		if n <= 0 {
 			n = 1
@@ -645,11 +645,11 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 
 	cp, err := getCommonParamsForLabelsAPI(r, startTime, false)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
 	labels, err := netstorage.LabelNames(qt, sq, limit, cp.deadline)
@@ -671,10 +671,9 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 //
 // See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
 func MetadataHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
-
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if limit < 0 {
 		limit = 0
@@ -734,11 +733,11 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
 	cp, err := getCommonParamsForLabelsAPI(r, startTime, true)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	limit, err := httputil.GetInt(r, "limit")
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit)
@@ -772,19 +771,19 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 	mayCache := !httputil.GetBool(r, "nocache")
 	query := r.FormValue("query")
 	if len(query) == 0 {
-		return fmt.Errorf("missing `query` arg")
+		return httpserver.InvalidParamError(fmt.Errorf("missing `query` arg"))
 	}
 	start, err := httputil.GetTime(r, "time", ct)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	lookbackDelta, err := getMaxLookback(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	step, err := httputil.GetDuration(r, "step", lookbackDelta)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if step <= 0 {
 		step = defaultStep
@@ -792,16 +791,16 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 
 	maxLen := searchutil.GetMaxQueryLen()
 	if len(query) > maxLen {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen)
+		return httpserver.InvalidParamError(fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen))
 	}
 	etfs, err := searchutil.GetExtraTagFilters(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if childQuery, windowExpr, offsetExpr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
 		window, err := windowExpr.NonNegativeDuration(step)
 		if err != nil {
-			return fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err))
 		}
 		offset := offsetExpr.Duration(step)
 		start -= offset
@@ -815,7 +814,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 
 		tagFilterss, err := getTagFilterssFromMatches([]string{childQuery})
 		if err != nil {
-			return err
+			return httpserver.InvalidParamError(err)
 		}
 		filterss := searchutil.JoinTagFilterss(tagFilterss, etfs)
 
@@ -831,22 +830,25 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		return nil
 	}
 	if childQuery, windowExpr, stepExpr, offsetExpr := promql.IsRollup(query); childQuery != "" {
+		if len(childQuery) > maxLen {
+			return httpserver.InvalidParamError(fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(childQuery), maxLen))
+		}
 		newStep, err := stepExpr.NonNegativeDuration(step)
 		if err != nil {
-			return fmt.Errorf("cannot parse step in square brackets at %s: %w", query, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse step in square brackets at %s: %w", query, err))
 		}
 		if newStep > 0 {
 			step = newStep
 		}
 		window, err := windowExpr.NonNegativeDuration(step)
 		if err != nil {
-			return fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err))
 		}
 		offset := offsetExpr.Duration(step)
 		start -= offset
 		end := start
 		start = end - window
-		if err := queryRangeHandler(qt, startTime, w, childQuery, start, end, step, r, ct, etfs); err != nil {
+		if err := queryRangeHandler(qt, startTime, w, childQuery, start, end, step, lookbackDelta, r, ct, etfs); err != nil {
 			return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", childQuery, start, end, step, err)
 		}
 		return nil
@@ -854,7 +856,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 
 	queryOffset, err := getLatencyOffsetMilliseconds(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	if !httputil.GetBool(r, "nocache") && ct-start < queryOffset && start-ct < queryOffset {
 		// Adjust start time only if `nocache` arg isn't set.
@@ -928,45 +930,43 @@ func QueryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	ct := startTime.UnixNano() / 1e6
 	query := r.FormValue("query")
 	if len(query) == 0 {
-		return fmt.Errorf("missing `query` arg")
+		return httpserver.InvalidParamError(fmt.Errorf("missing `query` arg"))
+	}
+	maxLen := searchutil.GetMaxQueryLen()
+	if len(query) > maxLen {
+		return httpserver.InvalidParamError(fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen))
 	}
 	start, err := httputil.GetTime(r, "start", ct-defaultStep)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	end, err := httputil.GetTime(r, "end", ct)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	step, err := httputil.GetDuration(r, "step", defaultStep)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
 	etfs, err := searchutil.GetExtraTagFilters(r)
 	if err != nil {
-		return err
+		return httpserver.InvalidParamError(err)
 	}
-	if err := queryRangeHandler(qt, startTime, w, query, start, end, step, r, ct, etfs); err != nil {
+	lookbackDelta, err := getMaxLookback(r)
+	if err != nil {
+		return httpserver.InvalidParamError(err)
+	}
+	if err := queryRangeHandler(qt, startTime, w, query, start, end, step, lookbackDelta, r, ct, etfs); err != nil {
 		return fmt.Errorf("error when executing query=%q on the time range (start=%d, end=%d, step=%d): %w", query, start, end, step, err)
 	}
 	return nil
 }
 
 func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, query string,
-	start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
+	start, end, step, lookbackDelta int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
 	deadline := searchutil.GetDeadlineForQuery(r, startTime)
 	mayCache := !httputil.GetBool(r, "nocache")
 	optimizeRepeatedBinaryOpSubexprs := httputil.GetBool(r, "optimize_repeated_binary_op_subexprs")
-	lookbackDelta, err := getMaxLookback(r)
-	if err != nil {
-		return err
-	}
-
-	// Validate input args.
-	maxLen := searchutil.GetMaxQueryLen()
-	if len(query) > maxLen {
-		return fmt.Errorf("too long query; got %d bytes; mustn't exceed `-search.maxQueryLen=%d` bytes", len(query), maxLen)
-	}
 	if start > end {
 		end = start + defaultStep
 	}
@@ -1005,7 +1005,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 	if step < maxStepForPointsAdjustment.Milliseconds() {
 		queryOffset, err := getLatencyOffsetMilliseconds(r)
 		if err != nil {
-			return err
+			return httpserver.InvalidParamError(err)
 		}
 		if ct-queryOffset < end {
 			result = adjustLastPoints(result, ct-queryOffset, ct+step)
@@ -1156,13 +1156,13 @@ func QueryStatsHandler(w http.ResponseWriter, r *http.Request) error {
 	if len(topNStr) > 0 {
 		n, err := strconv.Atoi(topNStr)
 		if err != nil {
-			return fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err)
+			return httpserver.InvalidParamError(fmt.Errorf("cannot parse `topN` arg %q: %w", topNStr, err))
 		}
 		topN = n
 	}
 	maxLifetimeMsecs, err := httputil.GetDuration(r, "maxLifetime", 10*60*1000)
 	if err != nil {
-		return fmt.Errorf("cannot parse `maxLifetime` arg: %w", err)
+		return httpserver.InvalidParamError(fmt.Errorf("cannot parse `maxLifetime` arg: %w", err))
 	}
 	maxLifetime := time.Duration(maxLifetimeMsecs) * time.Millisecond
 	w.Header().Set("Content-Type", "application/json")

app/vmselect/promql/exec.go

@@ -11,6 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/querystats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
@@ -46,15 +47,15 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
 
 	e, err := parsePromQLWithCache(q)
 	if err != nil {
-		return nil, err
+		return nil, httpserver.InvalidParamError(err)
 	}
 
 	if *disableImplicitConversion || *logImplicitConversion {
 		isInvalid := metricsql.IsLikelyInvalid(e)
 		if isInvalid && *disableImplicitConversion {
 			// we don't add query=%q to err message as it will be added by the caller
-			return nil, fmt.Errorf("query requires implicit conversion and is rejected according to -search.disableImplicitConversion command-line flag. " +
-				"See https://docs.victoriametrics.com/victoriametrics/metricsql/#implicit-query-conversions for details")
+			return nil, httpserver.InvalidParamError(fmt.Errorf("query requires implicit conversion and is rejected according to -search.disableImplicitConversion command-line flag. " +
+				"See https://docs.victoriametrics.com/victoriametrics/metricsql/#implicit-query-conversions for details"))
 		}
 		if isInvalid && *logImplicitConversion {
 			logger.Warnf("query=%q requires implicit conversion, see https://docs.victoriametrics.com/victoriametrics/metricsql/#implicit-query-conversions for details", e.AppendString(nil))

app/vmui/packages/vmui/assets/favicon.svg

@@ -0,0 +1,5 @@
+<svg width="48" height="48" fill="#020202" xmlns="http://www.w3.org/2000/svg">
+    <path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0"/>
+    <path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z"/>
+    <path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z"/>
+</svg>

app/vmui/packages/vmui/index.html

@@ -2,9 +2,9 @@
 <html lang="en">
 <head>
   <meta charset="utf-8"/>
-  <link rel="icon" href="/favicon.svg"/>
-  <link rel="apple-touch-icon" href="/favicon.svg"/>
-  <link rel="mask-icon" href="/favicon.svg" color="#000000">
+  <link id="favicon" rel="icon" href="/assets/favicon.svg" />
+  <link rel="apple-touch-icon" href="/assets/favicon.svg" />
+  <link id="mask-icon" rel="mask-icon" href="/assets/favicon.svg?no-inline" color="#000000">
 
   <meta name="robots" content="noindex">
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>

app/vmui/packages/vmui/public/favicon.svg

@@ -1 +0,0 @@
-<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>

app/vmui/packages/vmui/public/manifest.json

@@ -3,7 +3,7 @@
   "name": "vmui",
   "icons": [
     {
-      "src": "favicon.svg",
+      "src": "./assets/favicon.svg",
       "sizes": "any",
       "type": "image/svg+xml"
     }

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/BrowserTabController/BrowserTabController.tsx

@@ -0,0 +1,61 @@
+import { FC, useMemo } from "preact/compat";
+import "./style.scss";
+import { createFaviconUrl } from "../../../../utils/favicon";
+import classNames from "classnames";
+import { useBrowserTabSync } from "./hooks/useBrowserTabSync";
+import { CloseIcon } from "../../../Main/Icons";
+import { faviconColors } from "../../../../constants/faviconColors";
+
+const BrowserTabController: FC = () => {
+  const { faviconColor, changeFaviconColor } = useBrowserTabSync();
+
+  const faviconUrl = useMemo(() => {
+    return createFaviconUrl(faviconColor);
+  }, [faviconColor]);
+
+  const createHandlerClick = (color?: string) => () => {
+    changeFaviconColor(color);
+  };
+
+  return (
+    <div className="vm-browser-tab-controller">
+      <p className="vm-server-configurator__title">Favicon color</p>
+
+      <div className="vm-browser-tab-controller-palette">
+        <div className="vm-browser-tab-controller-palette-list">
+          <button
+            className="vm-browser-tab-controller-palette-list__item vm-browser-tab-controller-palette-list__item_reset"
+            type="button"
+            onClick={createHandlerClick()}
+            aria-label="Reset favicon color"
+          >
+            <CloseIcon />
+          </button>
+
+          {faviconColors.map(color => (
+            <button
+              className={classNames({
+                "vm-browser-tab-controller-palette-list__item": true,
+                "vm-browser-tab-controller-palette-list__item_selected": faviconColor === color
+              })}
+              key={color}
+              type="button"
+              style={{ color }}
+              onClick={createHandlerClick(color)}
+              aria-label={`Set favicon color to ${color}`}
+              aria-pressed={faviconColor === color}
+            />
+          ))}
+        </div>
+
+        <img
+          className="vm-browser-tab-controller-palette__preview"
+          src={faviconUrl}
+          alt="Favicon preview"
+        />
+      </div>
+    </div>
+  );
+};
+
+export default BrowserTabController;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/BrowserTabController/hooks/useBrowserTabSync.ts

@@ -0,0 +1,41 @@
+import useEventListener from "../../../../../hooks/useEventListener";
+import { useEffect, useState } from "preact/compat";
+import { getFromStorage, removeFromStorage, saveToStorage } from "../../../../../utils/storage";
+import { getFaviconStorageKey, updateFaviconColor } from "../../../../../utils/favicon";
+
+const storageKey = `FAVICON_COLOR:${getFaviconStorageKey()}` as const;
+
+const getColorFromStorage = () => {
+  return getFromStorage(storageKey) as string | undefined;
+};
+
+export const useBrowserTabSync = () => {
+  const [faviconColor, setFaviconColor] = useState(getColorFromStorage);
+
+  const handleUpdateColor = () => {
+    setFaviconColor(getColorFromStorage());
+  };
+
+  const changeFaviconColor = (color?: string) => {
+    if (color) {
+      saveToStorage(storageKey, color);
+    } else {
+      removeFromStorage([storageKey]);
+    }
+  };
+
+  useEffect(() => {
+    handleUpdateColor();
+  }, []);
+
+  useEffect(() => {
+    updateFaviconColor(faviconColor);
+  }, [faviconColor]);
+
+  useEventListener("storage", handleUpdateColor);
+
+  return {
+    faviconColor,
+    changeFaviconColor,
+  };
+};

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/BrowserTabController/style.scss

@@ -0,0 +1,71 @@
+@use "src/styles/variables" as *;
+
+$color-item-size: 28px;
+$outline-width: 2px;
+$outline-offset: 2px;
+$outline-space: $outline-width + $outline-offset;
+
+.vm-browser-tab-controller {
+  .vm-server-configurator__title {
+    padding-bottom: 2px;
+  }
+
+  &-palette {
+    display: flex;
+    align-items: flex-start;
+    justify-content: space-between;
+    gap: calc($padding-large * 2);
+
+    &-list {
+      display: flex;
+      align-items: center;
+      justify-content: flex-start;
+      flex-wrap: wrap;
+      gap: $padding-small;
+
+      &__item {
+        width: $color-item-size;
+        height: $color-item-size;
+        aspect-ratio: 1;
+        border-radius: 50%;
+        background-color: currentColor;
+        cursor: pointer;
+        transition-property: transform;
+        transition-duration: 0.15s;
+        transition-timing-function: linear;
+
+        &:hover {
+          transform: scale(1.1);
+        }
+
+        &:focus-visible {
+          transform: scale(1.1);
+        }
+
+        &_reset {
+          display: flex;
+          align-items: center;
+          justify-content: center;
+          background: transparent;
+          border: $border-divider;
+          color: $color-text-disabled;
+          padding: calc($padding-small / 2);
+        }
+
+        &_selected {
+          width: $color-item-size - 2 * $outline-space;
+          height: $color-item-size - 2 * $outline-space;
+          margin: $outline-space;
+          outline: $outline-width solid currentColor;
+          outline-offset: $outline-offset;
+          pointer-events: none;
+        }
+      }
+    }
+
+    &__preview {
+      width: $color-item-size;
+      height: auto;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -8,10 +8,12 @@ import Tooltip from "../../Main/Tooltip/Tooltip";
 import LimitsConfigurator from "./LimitsConfigurator/LimitsConfigurator";
 import { getAppModeEnable } from "../../../utils/app-mode";
 import classNames from "classnames";
-import Timezones from "./Timezones/Timezones";
+import TimezonesPicker from "./Timezones/TimezonesPicker";
 import ThemeControl from "../ThemeControl/ThemeControl";
 import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import useBoolean from "../../../hooks/useBoolean";
+import BrowserTabController from "./BrowserTabController/BrowserTabController";
+import LegendCollapseController from "./LegendCollapseController/LegendCollapseController";
 
 const title = "Settings";
 
@@ -26,7 +28,6 @@ const GlobalSettings: FC = () => {
 
   const serverSettingRef = useRef<ChildComponentHandle>(null);
   const limitsSettingRef = useRef<ChildComponentHandle>(null);
-  const timezoneSettingRef = useRef<ChildComponentHandle>(null);
 
   const {
     value: open,
@@ -37,7 +38,6 @@ const GlobalSettings: FC = () => {
   const handleApply = () => {
     serverSettingRef.current && serverSettingRef.current.handleApply();
     limitsSettingRef.current && limitsSettingRef.current.handleApply();
-    timezoneSettingRef.current && timezoneSettingRef.current.handleApply();
     handleClose();
   };
 
@@ -49,6 +49,10 @@ const GlobalSettings: FC = () => {
         onClose={handleClose}
       />
     },
+    {
+      show: true,
+      component: <TimezonesPicker/>
+    },
     {
       show: true,
       component: <LimitsConfigurator
@@ -58,12 +62,16 @@ const GlobalSettings: FC = () => {
     },
     {
       show: true,
-      component: <Timezones ref={timezoneSettingRef}/>
+      component: <LegendCollapseController/>
     },
     {
       show: !appModeEnable,
       component: <ThemeControl/>
-    }
+    },
+    {
+      show: true,
+      component: <BrowserTabController/>
+    },
   ].filter(control => control.show);
 
   return <>

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LegendCollapseController/LegendCollapseController.tsx

@@ -0,0 +1,30 @@
+import { FC, useEffect, useState } from "preact/compat";
+import { getFromStorage, saveToStorage } from "../../../../utils/storage";
+import Switch from "../../../Main/Switch/Switch";
+import { LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import "./style.scss";
+
+const LegendCollapseController: FC = () => {
+  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
+  const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
+
+  useEffect(() => {
+    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
+  }, [legendCollapse]);
+
+  return (
+    <div className="vm-legend-collapse-controller">
+      <Switch
+        fullWidth
+        color="neutral"
+        value={legendCollapse}
+        onChange={setLegendCollapse}
+        label={<span className="vm-server-configurator__title">Auto-collapse legend</span>}
+      />
+      <span className="vm-legend-collapse-controller__description">
+        Collapses the legend when series count exceeds {LEGEND_COLLAPSE_SERIES_LIMIT} to reduce UI load.
+      </span>
+    </div>);
+};
+
+export default LegendCollapseController;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LegendCollapseController/style.scss

@@ -0,0 +1,19 @@
+@use "src/styles/variables" as *;
+
+.vm-legend-collapse-controller {
+  background-color: $color-hover-black;
+  border-radius: $border-radius-medium;
+  padding: $padding-large;
+  border: $border-divider;
+
+  .vm-graph-settings-row__label {
+    margin: 0;
+  }
+
+  &__description {
+    padding-top: $padding-global;
+    font-size: $font-size-small;
+    line-height: 1.3;
+    text-wrap: pretty;
+  }
+}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -1,23 +1,21 @@
-import { forwardRef, useCallback, useEffect, useImperativeHandle, useState } from "preact/compat";
+import { forwardRef, useCallback, useImperativeHandle, useState } from "preact/compat";
 import { DisplayType, ErrorTypes } from "../../../../types";
 import TextField from "../../../Main/TextField/TextField";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
 import { InfoIcon, RestartIcon } from "../../../Main/Icons";
 import Button from "../../../Main/Button/Button";
-import { DEFAULT_MAX_SERIES, LEGEND_COLLAPSE_SERIES_LIMIT } from "../../../../constants/graph";
+import { DEFAULT_MAX_SERIES } from "../../../../constants/graph";
 import "./style.scss";
 import classNames from "classnames";
 import useDeviceDetect from "../../../../hooks/useDeviceDetect";
 import { ChildComponentHandle } from "../GlobalSettings";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../../state/customPanel/CustomPanelStateContext";
-import Switch from "../../../Main/Switch/Switch";
-import { getFromStorage, saveToStorage } from "../../../../utils/storage";
 
 interface ServerConfiguratorProps {
-  onClose: () => void
+  onClose: () => void;
 }
 
-const fields: {label: string, type: DisplayType}[] = [
+const fields: { label: string, type: DisplayType }[] = [
   { label: "Graph", type: DisplayType.chart },
   { label: "JSON", type: DisplayType.code },
   { label: "Table", type: DisplayType.table }
@@ -29,8 +27,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
   const { seriesLimits } = useCustomPanelState();
   const customPanelDispatch = useCustomPanelDispatch();
 
-  const storageCollapse = getFromStorage("LEGEND_AUTO_COLLAPSE");
-  const [legendCollapse, setLegendCollapse] = useState(storageCollapse ? storageCollapse === "true" : true);
+
 
   const [limits, setLimits] = useState(seriesLimits);
   const [error, setError] = useState({
@@ -43,7 +40,7 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     setLimits(DEFAULT_MAX_SERIES);
   };
 
-  const createChangeHandler = (type: DisplayType) =>  (val: string) => {
+  const createChangeHandler = (type: DisplayType) => (val: string) => {
     const value = val || "";
     setError(prev => ({ ...prev, [type]: +value < 0 ? ErrorTypes.positiveNumber : "" }));
     setLimits({
@@ -57,10 +54,6 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
     onClose();
   }, [limits]);
 
-  useEffect(() => {
-    saveToStorage("LEGEND_AUTO_COLLAPSE", `${legendCollapse}`);
-  }, [legendCollapse]);
-
   useImperativeHandle(ref, () => ({ handleApply }), [handleApply]);
 
   return (
@@ -106,19 +99,6 @@ const LimitsConfigurator = forwardRef<ChildComponentHandle, ServerConfiguratorPr
           </div>
         ))}
       </div>
-
-      <div className="vm-graph-settings-row">
-        <span className="vm-graph-settings-row__label">Auto-collapse legend</span>
-        <Switch
-          value={legendCollapse}
-          onChange={setLegendCollapse}
-          label={legendCollapse ? "Enabled" : "Disabled"}
-          fullWidth={isMobile}
-        />
-        <span className="vm-legend-configs-item__info">
-          Collapses the legend when series count exceeds {LEGEND_COLLAPSE_SERIES_LIMIT} to reduce UI load.
-        </span>
-      </div>
     </div>
   );
 });

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/Timezones.tsx

@@ -1,183 +0,0 @@
-import { FC, forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "preact/compat";
-import { getBrowserTimezone, getTimezoneList, getUTCByTimezone } from "../../../../utils/time";
-import { ArrowDropDownIcon } from "../../../Main/Icons";
-import classNames from "classnames";
-import Popper from "../../../Main/Popper/Popper";
-import Accordion from "../../../Main/Accordion/Accordion";
-import TextField from "../../../Main/TextField/TextField";
-import { Timezone } from "../../../../types";
-import "./style.scss";
-import useDeviceDetect from "../../../../hooks/useDeviceDetect";
-import useBoolean from "../../../../hooks/useBoolean";
-import WarningTimezone from "./WarningTimezone";
-import { useTimeDispatch, useTimeState } from "../../../../state/time/TimeStateContext";
-
-interface PinnedTimezone extends Timezone {
-  title: string;
-  isInvalid?: boolean;
-}
-
-const browserTimezone = getBrowserTimezone();
-
-const Timezones: FC = forwardRef((props, ref) => {
-  const { isMobile } = useDeviceDetect();
-  const timezones = getTimezoneList();
-
-  const { timezone: stateTimezone, defaultTimezone } = useTimeState();
-  const timeDispatch = useTimeDispatch();
-
-  const [timezone, setTimezone] = useState(stateTimezone);
-  const [search, setSearch] = useState("");
-  const targetRef = useRef<HTMLDivElement>(null);
-
-  const {
-    value: openList,
-    toggle: toggleOpenList,
-    setFalse: handleCloseList,
-  } = useBoolean(false);
-
-  const pinnedTimezones = useMemo(() => [
-    {
-      title: `Default time (${defaultTimezone})`,
-      region: defaultTimezone,
-      utc: defaultTimezone ? getUTCByTimezone(defaultTimezone) : "UTC"
-    },
-    {
-      title: browserTimezone.title,
-      region: browserTimezone.region,
-      utc: getUTCByTimezone(browserTimezone.region),
-      isInvalid: !browserTimezone.isValid
-    },
-    {
-      title: "UTC (Coordinated Universal Time)",
-      region: "UTC",
-      utc: "UTC"
-    },
-  ].filter(t => t.region) as PinnedTimezone[], [defaultTimezone]);
-
-  const searchTimezones = useMemo(() => {
-    if (!search) return timezones;
-    try {
-      return getTimezoneList(search);
-    } catch (e) {
-      return {};
-    }
-  }, [search, timezones]);
-
-  const timezonesGroups = useMemo(() => Object.keys(searchTimezones), [searchTimezones]);
-
-  const activeTimezone = useMemo(() => ({
-    region: timezone,
-    utc: getUTCByTimezone(timezone)
-  }), [timezone]);
-
-  const handleChangeSearch = (val: string) => {
-    setSearch(val);
-  };
-
-  const handleSetTimezone = (val: Timezone) => {
-    setTimezone(val.region);
-    setSearch("");
-    handleCloseList();
-  };
-
-  const createHandlerSetTimezone = (val: Timezone) => () => {
-    handleSetTimezone(val);
-  };
-
-  useEffect(() => {
-    setTimezone(stateTimezone);
-  }, [stateTimezone]);
-
-  useImperativeHandle(ref, () => ({
-    handleApply: () => {
-      timeDispatch({ type: "SET_TIMEZONE", payload: timezone });
-    }
-  }), [timezone]);
-
-  return (
-    <div className="vm-timezones">
-      <div className="vm-server-configurator__title">
-        Time zone
-      </div>
-      <div
-        className="vm-timezones-item vm-timezones-item_selected"
-        onClick={toggleOpenList}
-        ref={targetRef}
-      >
-        <div className="vm-timezones-item__title">{activeTimezone.region}</div>
-        <div className="vm-timezones-item__utc">{activeTimezone.utc}</div>
-        <div
-          className={classNames({
-            "vm-timezones-item__icon": true,
-            "vm-timezones-item__icon_open": openList
-          })}
-        >
-          <ArrowDropDownIcon/>
-        </div>
-      </div>
-      <Popper
-        open={openList}
-        buttonRef={targetRef}
-        placement="bottom-left"
-        onClose={handleCloseList}
-        fullWidth
-        title={isMobile ? "Time zone" : undefined}
-      >
-        <div
-          className={classNames({
-            "vm-timezones-list": true,
-            "vm-timezones-list_mobile": isMobile,
-          })}
-        >
-          <div className="vm-timezones-list-header">
-            <div className="vm-timezones-list-header__search">
-              <TextField
-                autofocus
-                label="Search"
-                value={search}
-                onChange={handleChangeSearch}
-              />
-            </div>
-            {pinnedTimezones.map((t, i) => t && (
-              <div
-                key={`${i}_${t.region}`}
-                className="vm-timezones-item vm-timezones-list-group-options__item"
-                onClick={createHandlerSetTimezone(t)}
-              >
-                <div className="vm-timezones-item__title">{t.title}{t.isInvalid && <WarningTimezone/>}</div>
-                <div className="vm-timezones-item__utc">{t.utc}</div>
-              </div>
-            ))}
-          </div>
-          {timezonesGroups.map(t => (
-            <div
-              className="vm-timezones-list-group"
-              key={t}
-            >
-              <Accordion
-                defaultExpanded={true}
-                title={<div className="vm-timezones-list-group__title">{t}</div>}
-              >
-                <div className="vm-timezones-list-group-options">
-                  {searchTimezones[t] && searchTimezones[t].map(item => (
-                    <div
-                      className="vm-timezones-item vm-timezones-list-group-options__item"
-                      onClick={createHandlerSetTimezone(item)}
-                      key={item.search}
-                    >
-                      <div className="vm-timezones-item__title">{item.region}</div>
-                      <div className="vm-timezones-item__utc">{item.utc}</div>
-                    </div>
-                  ))}
-                </div>
-              </Accordion>
-            </div>
-          ))}
-        </div>
-      </Popper>
-    </div>
-  );
-});
-
-export default Timezones;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/TimezonesList.tsx

@@ -0,0 +1,129 @@
+import { FC, useMemo, useState } from "preact/compat";
+import { getBrowserTimezone, getTimezoneList, getUTCByTimezone } from "../../../../utils/time";
+import classNames from "classnames";
+import Accordion from "../../../Main/Accordion/Accordion";
+import TextField from "../../../Main/TextField/TextField";
+import { Timezone } from "../../../../types";
+import "./style.scss";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import WarningTimezone from "./WarningTimezone";
+import { useTimeState } from "../../../../state/time/TimeStateContext";
+
+interface PinnedTimezone extends Timezone {
+  title: string;
+  isInvalid?: boolean;
+}
+
+type Props = {
+  onChange: (tz: Timezone) => void;
+}
+
+const browserTimezone = getBrowserTimezone();
+
+const TimezonesList: FC<Props> = ({ onChange }) => {
+  const { isMobile } = useDeviceDetect();
+  const { defaultTimezone } = useTimeState();
+
+  const timezones = useMemo(() => getTimezoneList(), []);
+
+  const [search, setSearch] = useState("");
+
+  const pinnedTimezones = useMemo(() => [
+    {
+      title: `Default time (${defaultTimezone})`,
+      region: defaultTimezone,
+      utc: defaultTimezone ? getUTCByTimezone(defaultTimezone) : "UTC"
+    },
+    {
+      title: browserTimezone.title,
+      region: browserTimezone.region,
+      utc: getUTCByTimezone(browserTimezone.region),
+      isInvalid: !browserTimezone.isValid
+    },
+    {
+      title: "UTC (Coordinated Universal Time)",
+      region: "UTC",
+      utc: "UTC"
+    },
+  ].filter(t => t.region) as PinnedTimezone[], [defaultTimezone]);
+
+  const searchTimezones = useMemo(() => {
+    if (!search) return timezones;
+    try {
+      return getTimezoneList(search);
+    } catch (e) {
+      return {};
+    }
+  }, [search, timezones]);
+
+  const timezonesGroups = useMemo(() => Object.keys(searchTimezones), [searchTimezones]);
+
+  const handleChangeSearch = (val: string) => {
+    setSearch(val);
+  };
+
+  const handleSetTimezone = (tz: Timezone) => {
+    onChange(tz);
+    setSearch("");
+  };
+
+  const createHandlerSetTimezone = (val: Timezone) => () => {
+    handleSetTimezone(val);
+  };
+
+  return (
+    <div
+      className={classNames({
+        "vm-list": true,
+        "vm-timezones-list": true,
+        "vm-timezones-list_mobile": isMobile,
+      })}
+    >
+      <div className="vm-timezones-list-header">
+        <div className="vm-timezones-list-header__search">
+          <TextField
+            label="Search"
+            value={search}
+            onChange={handleChangeSearch}
+          />
+        </div>
+      </div>
+      {pinnedTimezones.map((t, i) => t && (
+        <div
+          key={`${i}_${t.region}`}
+          className="vm-list-item vm-timezones-item vm-timezones-list-group-options__item"
+          onClick={createHandlerSetTimezone(t)}
+        >
+          <div className="vm-timezones-item__title">{t.title}{t.isInvalid && <WarningTimezone/>}</div>
+          <div className="vm-timezones-item__utc">{t.utc}</div>
+        </div>
+      ))}
+      {timezonesGroups.map(t => (
+        <div
+          className="vm-timezones-list-group"
+          key={t}
+        >
+          <Accordion
+            defaultExpanded={true}
+            title={<div className="vm-timezones-list-group__title">{t}</div>}
+          >
+            <div className="vm-timezones-list-group-options">
+              {searchTimezones[t] && searchTimezones[t].map(item => (
+                <div
+                  className="vm-list-item vm-timezones-item vm-timezones-list-group-options__item"
+                  onClick={createHandlerSetTimezone(item)}
+                  key={item.search}
+                >
+                  <div className="vm-timezones-item__title">{item.region}</div>
+                  <div className="vm-timezones-item__utc">{item.utc}</div>
+                </div>
+              ))}
+            </div>
+          </Accordion>
+        </div>
+      ))}
+    </div>
+  );
+};
+
+export default TimezonesList;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/TimezonesPicker.tsx

@@ -0,0 +1,71 @@
+import { FC, useMemo, useRef } from "preact/compat";
+import { getUTCByTimezone } from "../../../../utils/time";
+import { ArrowDropDownIcon } from "../../../Main/Icons";
+import classNames from "classnames";
+import { Timezone } from "../../../../types";
+import "./style.scss";
+import useBoolean from "../../../../hooks/useBoolean";
+import { useTimeDispatch, useTimeState } from "../../../../state/time/TimeStateContext";
+import TimezonesList from "./TimezonesList";
+import Popper from "../../../Main/Popper/Popper";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+
+const TimezonesPicker: FC = () => {
+  const { isMobile } = useDeviceDetect();
+  const { timezone: stateTimezone } = useTimeState();
+  const timeDispatch = useTimeDispatch();
+
+  const triggerRef = useRef<HTMLDivElement>(null);
+
+  const {
+    value: isOpenList,
+    toggle: toggleOpenList,
+    setFalse: handleCloseList,
+  } = useBoolean(false);
+
+  const activeTimezone = useMemo(() => ({
+    region: stateTimezone,
+    utc: getUTCByTimezone(stateTimezone)
+  }), [stateTimezone]);
+
+  const handleSetTimezone = (tz: Timezone) => {
+    timeDispatch({ type: "SET_TIMEZONE", payload: tz.region });
+    handleCloseList();
+  };
+
+  return (
+    <div className="vm-timezones">
+      <div className="vm-server-configurator__title">
+        Time zone
+      </div>
+      <div
+        className="vm-timezones-item vm-timezones-item_selected"
+        onClick={toggleOpenList}
+        ref={triggerRef}
+      >
+        <div className="vm-timezones-item__title">{activeTimezone.region}</div>
+        <div className="vm-timezones-item__utc">{activeTimezone.utc}</div>
+        <div
+          className={classNames({
+            "vm-timezones-item__icon": true,
+            "vm-timezones-item__icon_open": isOpenList
+          })}
+        >
+          <ArrowDropDownIcon/>
+        </div>
+      </div>
+      <Popper
+        open={isOpenList}
+        buttonRef={triggerRef}
+        placement="bottom-left"
+        onClose={handleCloseList}
+        fullWidth
+        title={isMobile ? "Time zone" : undefined}
+      >
+        <TimezonesList onChange={handleSetTimezone}/>
+      </Popper>
+    </div>
+  );
+};
+
+export default TimezonesPicker;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/style.scss

@@ -16,6 +16,7 @@
     }
 
     &__title {
+      flex-grow: 1;
       display: flex;
       align-items: center;
       gap: $padding-small;
@@ -34,6 +35,7 @@
       background-color: $color-hover-black;
       padding: calc($padding-small/2);
       border-radius: $border-radius-small;
+      font-size: $font-size-small;
     }
 
     &__icon {
@@ -54,9 +56,11 @@
   }
 
   &-list {
+    padding-top: 0;
     max-height: 300px;
     background-color: $color-background-block;
     border-radius: $border-radius-medium;
+    font-size: $font-size-small;
     overflow: auto;
 
     &_mobile {
@@ -72,10 +76,9 @@
       top: 0;
       background-color: $color-background-block;
       z-index: 2;
-      border-bottom: $border-divider;
 
       &__search {
-        padding: $padding-small;
+        padding: $padding-small $padding-small calc($padding-small / 2);
       }
     }
 
@@ -91,6 +94,7 @@
         font-weight: bold;
         color: $color-text-secondary;
         padding: $padding-small $padding-global;
+        font-size: $font-size-small;
       }
 
       &-options {
@@ -98,7 +102,7 @@
         align-items: flex-start;
 
         &__item {
-          padding: $padding-small $padding-global;
+          padding: calc($padding-small / 2) $padding-global;
           transition: background-color 200ms ease;
 
           &:hover {

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/style.scss

@@ -4,9 +4,9 @@
   display: flex;
   flex-direction: column;
   align-items: center;
-  gap: $padding-large;
+  gap: calc($padding-global * 2);
   width: 600px;
-  padding-bottom: $padding-medium;
+  padding-inline: $padding-large;
 
   &_mobile {
     grid-auto-rows: min-content;
@@ -62,6 +62,7 @@
     justify-content: flex-end;
     gap: $padding-small;
     width: 100%;
+    padding-block: $padding-global;
   }
 
   &_mobile &-footer {

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -22,12 +22,10 @@ const StepConfigurator: FC = () => {
   const { isMobile } = useDeviceDetect();
 
   const { customStep: value, isHistogram } = useGraphState();
-  const { period: { step, end, start } } = useTimeState();
+  const { period: { end, start } } = useTimeState();
   const graphDispatch = useGraphDispatch();
   const { displayType } = useCustomPanelState();
 
-  const prevDuration = usePrevious(end - start);
-
   const defaultStep = useMemo(() => {
     return getStepFromDuration(end - start, isHistogram, displayType);
   }, [end, start, isHistogram, displayType]);
@@ -106,16 +104,14 @@ const StepConfigurator: FC = () => {
   }, [defaultStep]);
 
   useEffect(() => {
-    const dur = end - start;
-    if (dur === prevDuration || !prevDuration || value !== prevDefaultStep) return;
-    if (defaultStep) {
-      handleApply(defaultStep);
-    }
-  }, [prevDuration, defaultStep]);
+    if (!prevDefaultStep) return;
+    if (value !== prevDefaultStep) return;
+    if (value === defaultStep) return;
 
-  useEffect(() => {
-    if (step === value || step === defaultStep) handleApply(defaultStep);
-  }, [isHistogram, displayType]);
+    graphDispatch({ type: "SET_CUSTOM_STEP", payload: defaultStep });
+    setCustomStep(defaultStep);
+    setError("");
+  }, [defaultStep, prevDefaultStep, value, graphDispatch]);
 
   return (
     <div

app/vmui/packages/vmui/src/components/Configurators/ThemeControl/ThemeControl.tsx

@@ -5,8 +5,20 @@ import useDeviceDetect from "../../../hooks/useDeviceDetect";
 import classNames from "classnames";
 import { FC } from "preact/compat";
 import { useAppDispatch, useAppState } from "../../../state/common/StateContext";
+import { DarkIcon, LightIcon, SystemIcon } from "../../Main/Icons";
+
+const themeIcons = {
+  [Theme.system]: <SystemIcon/>,
+  [Theme.light]: <LightIcon/>,
+  [Theme.dark]: <DarkIcon/>,
+};
+
+const options = Object.values(Theme).map(value => ({
+  title: value,
+  value,
+  icon: themeIcons[value],
+}));
 
-const options = Object.values(Theme).map(value => ({ title: value, value }));
 const ThemeControl: FC = () => {
   const { isMobile } = useDeviceDetect();
   const dispatch = useAppDispatch();
@@ -25,13 +37,14 @@ const ThemeControl: FC = () => {
       })}
     >
       <div className="vm-server-configurator__title">
-        Theme preferences
+        Theme
       </div>
       <div
         className="vm-theme-control__toggle"
         key={`${isMobile}`}
       >
         <Toggle
+          size="large"
           options={options}
           value={theme}
           onChange={handleClickItem}

app/vmui/packages/vmui/src/components/Configurators/ThemeControl/style.scss

@@ -4,7 +4,7 @@
 
   &__toggle {
     display: inline-flex;
-    min-width: 300px;
+    width: 100%;
     text-transform: capitalize;
   }
 

app/vmui/packages/vmui/src/components/Main/Icons/index.tsx

@@ -633,3 +633,60 @@ export const DebugIcon = () => (
     />
   </svg>
 );
+
+export const SystemIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M18 4C18.7957 4 19.5595 4.3163 20.1221 4.87891C20.6845 5.44148 21 6.20452 21 7V15.5264L21.0069 15.6426C21.0203 15.7579 21.0542 15.8702 21.1065 15.9746L22.1729 18.0996C22.3271 18.4056 22.4009 18.7466 22.3858 19.0889C22.3705 19.431 22.2675 19.7637 22.0869 20.0547C21.9063 20.3457 21.6534 20.5855 21.3535 20.751C21.0555 20.9154 20.7202 21.0003 20.3799 20.999L3.62013 21C3.28002 21.0012 2.94535 20.9153 2.64748 20.751C2.34748 20.5855 2.09477 20.3458 1.91408 20.0547C1.73343 19.7636 1.63047 19.4311 1.61525 19.0889C1.60006 18.7466 1.67297 18.4056 1.82716 18.0996L2.89455 15.9746L2.94045 15.8682C2.9801 15.7589 3.00007 15.6432 3.00002 15.5264V7C3.00002 6.20442 3.3164 5.4415 3.87892 4.87891C4.44146 4.31636 5.20447 4.00007 6.00002 4H18ZM4.62404 16.9873L3.61427 18.999L3.6133 19H20.3877L20.3867 18.999L19.376 16.9873H4.62404ZM6.00002 6C5.7349 6.00007 5.48045 6.1055 5.29298 6.29297C5.10554 6.48049 5.00002 6.73485 5.00002 7V14.9873H19V7C19 6.73478 18.8946 6.48051 18.707 6.29297C18.5195 6.10552 18.2652 6 18 6H6.00002Z"
+    />
+  </svg>
+);
+
+export const LightIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M12 19C12.5523 19 13 19.4477 13 20V22C13 22.5523 12.5523 23 12 23C11.4477 23 11 22.5523 11 22V20C11 19.4477 11.4477 19 12 19Z"
+    />
+    <path
+      d="M5.63281 16.9531C6.02334 16.5627 6.65638 16.5626 7.04688 16.9531C7.43717 17.3436 7.43725 17.9767 7.04688 18.3672L5.63672 19.7773C5.24625 20.1676 4.61313 20.1676 4.22266 19.7773C3.8322 19.3869 3.83233 18.7538 4.22266 18.3633L5.63281 16.9531Z"
+    />
+    <path
+      d="M16.9531 16.9531C17.3436 16.5626 17.9767 16.5626 18.3672 16.9531L19.7773 18.3633C20.1676 18.7538 20.1678 19.3869 19.7773 19.7773C19.3869 20.1677 18.7538 20.1675 18.3633 19.7773L16.9531 18.3672C16.5626 17.9767 16.5627 17.3437 16.9531 16.9531Z"
+    />
+    <path
+      d="M12 7C14.7614 7 17 9.23858 17 12C17 14.7614 14.7614 17 12 17C9.23858 17 7 14.7614 7 12C7 9.23858 9.23858 7 12 7ZM12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9Z"
+    />
+    <path
+      d="M4 11C4.55228 11 5 11.4477 5 12C5 12.5523 4.55228 13 4 13H2C1.44772 13 1 12.5523 1 12C1 11.4477 1.44772 11 2 11H4Z"
+    />
+    <path
+      d="M22 11C22.5523 11 23 11.4477 23 12C23 12.5523 22.5523 13 22 13H20C19.4477 13 19 12.5523 19 12C19 11.4477 19.4477 11 20 11H22Z"
+    />
+    <path
+      d="M4.22266 4.22266C4.61315 3.83229 5.24623 3.83229 5.63672 4.22266L7.04688 5.63281C7.4372 6.02331 7.43723 6.65639 7.04688 7.04688C6.6564 7.43735 6.02335 7.43724 5.63281 7.04688L4.22266 5.63672C3.83225 5.24618 3.83217 4.61314 4.22266 4.22266Z"
+    />
+    <path
+      d="M18.3633 4.22266C18.7538 3.83237 19.3869 3.83232 19.7773 4.22266C20.1677 4.61312 20.1676 5.2462 19.7773 5.63672L18.3672 7.04688C17.9767 7.4373 17.3436 7.4373 16.9531 7.04688C16.5627 6.65637 16.5627 6.0233 16.9531 5.63281L18.3633 4.22266Z"
+    />
+    <path
+      d="M12 1C12.5523 1 13 1.44772 13 2V4C13 4.55228 12.5523 5 12 5C11.4477 5 11 4.55228 11 4V2C11 1.44772 11.4477 1 12 1Z"
+    />
+  </svg>
+);
+
+export const DarkIcon = () => (
+  <svg
+    viewBox="0 0 24 24"
+    fill="currentColor"
+  >
+    <path
+      d="M11.5809 2.01318C12.1851 2.02907 12.6373 2.40742 12.8475 2.84717C13.0627 3.29758 13.0619 3.86893 12.7615 4.34814L12.7606 4.34717C12.1616 5.30581 11.9061 6.43987 12.034 7.56299C12.162 8.68628 12.6672 9.73328 13.4666 10.5327C14.2661 11.332 15.3131 11.8364 16.4363 11.9644C17.5596 12.0922 18.6934 11.836 19.6522 11.2368L19.8348 11.1382C20.2701 10.9405 20.7563 10.9617 21.1512 11.1499C21.6217 11.3742 22.0194 11.8752 21.9832 12.5405C21.8789 14.4693 21.2181 16.3268 20.0809 17.8882C18.9435 19.4496 17.378 20.6484 15.574 21.3394C13.7701 22.0302 11.8042 22.184 9.91485 21.7817C8.02549 21.3794 6.29356 20.4376 4.92754 19.0718C3.56149 17.7059 2.62012 15.9739 2.21758 14.0845C1.81507 12.195 1.96826 10.2294 2.65899 8.42529C3.34975 6.62115 4.5487 5.05596 6.11016 3.91846C7.6716 2.781 9.52882 2.11947 11.4578 2.01514L11.5809 2.01318ZM10.6209 4.12061C9.42038 4.33038 8.27873 4.81214 7.28692 5.53467C6.03798 6.44459 5.07973 7.69705 4.52715 9.14014C3.97456 10.5834 3.85165 12.156 4.17364 13.6675C4.49565 15.179 5.24872 16.5649 6.34161 17.6577C7.43448 18.7505 8.82026 19.5038 10.3318 19.8257C11.8434 20.1475 13.416 20.0239 14.8592 19.4712C16.3024 18.9184 17.5548 17.9597 18.4647 16.7104C19.1869 15.7188 19.6671 14.5776 19.8768 13.3774C18.7333 13.8927 17.4674 14.0949 16.2098 13.9517C14.637 13.7725 13.1709 13.0661 12.0516 11.9468C10.9324 10.8275 10.2258 9.36126 10.0467 7.78857C9.90352 6.53075 10.1054 5.26417 10.6209 4.12061Z"
+    />
+  </svg>
+);

app/vmui/packages/vmui/src/components/Main/Switch/Switch.tsx

@@ -1,11 +1,10 @@
-import { ReactNode } from "react";
+import { FC, ReactNode } from "preact/compat";
 import classNames from "classnames";
 import "./style.scss";
-import { FC } from "preact/compat";
 
 interface SwitchProps {
   value: boolean
-  color?: "primary" | "secondary" | "error"
+  color?: "primary" | "secondary" | "error" | "neutral"
   disabled?: boolean
   label?: string | ReactNode
   fullWidth?: boolean

app/vmui/packages/vmui/src/components/Main/Switch/style.scss

@@ -29,6 +29,10 @@ $switch-border-radius: $switch-handle-size + ($switch-padding * 2);
     background-color: $color-secondary;
   }
 
+  &_neutral_active &-track {
+    background-color: $color-text;
+  }
+
   &_primary_active &-track {
     background-color: $color-primary;
   }

app/vmui/packages/vmui/src/components/Main/Toggle/Toggle.tsx

@@ -1,22 +1,21 @@
-import { FC, useEffect, useRef, useState } from "preact/compat";
+import { FC, useEffect, useRef, useState, ReactNode } from "preact/compat";
 import classNames from "classnames";
-import { ReactNode } from "react";
 import "./style.scss";
 
 interface ToggleProps {
-  options: {value: string, title?: string, icon?: ReactNode}[]
-  value: string
-  onChange: (val: string) => void
-  label?: string
+  options: { value: string, title?: string, icon?: ReactNode }[];
+  value: string;
+  onChange: (val: string) => void;
+  label?: string;
+  size?: "medium" | "large";
 }
 
-const Toggle: FC<ToggleProps> = ({ options, value, label, onChange }) => {
+const Toggle: FC<ToggleProps> = ({ options, value, label, size = "medium", onChange }) => {
 
   const activeRef = useRef<HTMLDivElement>(null);
   const [position, setPosition] = useState({
     width: "0px",
     left: "0px",
-    borderRadius: "0px"
   });
 
   const createHandlerChange = (value: string) => () => {
@@ -28,35 +27,25 @@ const Toggle: FC<ToggleProps> = ({ options, value, label, onChange }) => {
       setPosition({
         width: "0px",
         left: "0px",
-        borderRadius: "0px"
       });
       return;
     }
     const index = options.findIndex(o => o.value === value);
     const { width: widthRect } = activeRef.current.getBoundingClientRect();
 
-    let width = widthRect;
-    let left = index * width;
-    let borderRadius = "0";
-    if (index === 0) borderRadius = "16px 0 0 16px";
+    const width = widthRect;
+    const left = index * width;
 
-    if (index === options.length - 1) {
-      borderRadius = "10px";
-      left -= 1;
-      borderRadius = "0 16px 16px 0";
-    }
-
-    if (index !== 0 && (index !== options.length - 1)) {
-      width += 1;
-      left -= 1;
-    }
-
-
-    setPosition({ width: `${width}px`, left: `${left}px`, borderRadius });
+    setPosition({ width: `${width}px`, left: `${left}px` });
   }, [activeRef, value, options]);
 
   return (
-    <div className="vm-toggles">
+    <div
+      className={classNames({
+        "vm-toggles": true,
+        [`vm-toggles_${size}`]: size,
+      })}
+    >
       {label && (
         <label className="vm-toggles__label">
           {label}
@@ -66,15 +55,14 @@ const Toggle: FC<ToggleProps> = ({ options, value, label, onChange }) => {
         className="vm-toggles-group"
         style={{ gridTemplateColumns: `repeat(${options.length}, 1fr)` }}
       >
-        {position.borderRadius && <div
+        <div
           className="vm-toggles-group__highlight"
           style={position}
-        />}
-        {options.map((option, i) => (
+        />
+        {options.map((option) => (
           <div
             className={classNames({
               "vm-toggles-group-item": true,
-              "vm-toggles-group-item_first": i === 0,
               "vm-toggles-group-item_active": option.value === value,
               "vm-toggles-group-item_icon": option.icon && option.title
             })}

app/vmui/packages/vmui/src/components/Main/Toggle/style.scss

@@ -20,6 +20,8 @@
     align-items: center;
     justify-content: center;
     overflow: hidden;
+    border-radius: $border-radius-small;
+    background: $color-hover-black;
 
     &-item {
       position: relative;
@@ -27,55 +29,68 @@
       align-items: center;
       justify-content: center;
       padding: $padding-small;
-      border-right: $border-divider;
-      border-top: $border-divider;
-      border-bottom: $border-divider;
       font-size: $font-size-small;
       color: $color-text-secondary;
-      font-weight: bold;
+      font-weight: 500;
       cursor: pointer;
       text-align: center;
-      transition: color 150ms ease-in;
+      transition: opacity 150ms ease-in, color 150ms ease-in;
       z-index: 2;
       user-select: none;
 
-      &_first {
-        border-radius: 16px 0 0 16px;
-        border-left: $border-divider
-      }
-
-      &:last-child {
-        border-radius: 0 16px 16px 0;
-        border-left: none;
-      }
-
       &_icon {
         grid-template-columns: 14px auto;
-        gap: 4px;
+        gap: calc($padding-small / 2);
       }
 
-      &:hover {
-        color: $color-primary;
+      &:hover:not(&_active) {
+        opacity: 0.8;
       }
 
       &_active {
-        color: $color-primary;
-        border-color: transparent;
-
-        &:hover {
-          background-color: transparent;
-        }
+        color: $color-text;
+        font-weight: 600;
       }
     }
 
     &__highlight {
       position: absolute;
       top: 0;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      padding: 3px;
       height: 100%;
       background-color: rgba($color-primary, 0.08);
-      border: 1px solid $color-primary;
       transition: left 200ms cubic-bezier(0.280, 0.840, 0.420, 1), border-radius 200ms linear;
       z-index: 1;
+
+      &:after {
+        content: '';
+        height: 100%;
+        width: 100%;
+        background-color: $color-background-block;
+        border-radius: $border-radius-small;
+        box-shadow: $box-shadow;
+      }
+    }
+  }
+
+  &_large &-group {
+    border-radius: $border-radius-medium;
+
+    &-item {
+      padding: $padding-global $padding-small;
+
+      &_icon {
+        grid-template-columns: 16px auto;
+        gap: $padding-small;
+      }
+    }
+
+    &__highlight {
+      padding: 4px;
+      border-radius: $border-radius-medium;
     }
   }
 }

app/vmui/packages/vmui/src/constants/faviconColors.ts

@@ -0,0 +1,14 @@
+export const faviconColors = [
+  "#A1A1AA",
+  "#71717A",
+  "#020202",
+  "#E94600",
+  "#FF7A00",
+  "#F2B705",
+  "#84CC16",
+  "#16B86A",
+  "#00AFAF",
+  "#2979FF",
+  "#8B5CF6",
+  "#E83E9A",
+] as const;

app/vmui/packages/vmui/src/layouts/MainLayout/MainLayout.tsx

@@ -14,6 +14,9 @@ import useFetchDefaultTimezone from "../../hooks/useFetchDefaultTimezone";
 import useFetchAppConfig from "../../hooks/useFetchAppConfig";
 import WebStorageCheck from "../../components/WebStorageCheck/WebStorageCheck";
 import { migrateStorageToPrefixedKeys } from "../../utils/storage";
+import {
+  useBrowserTabSync
+} from "../../components/Configurators/GlobalSettings/BrowserTabController/hooks/useBrowserTabSync";
 
 const MainLayout: FC = () => {
   const appModeEnable = getAppModeEnable();
@@ -21,6 +24,7 @@ const MainLayout: FC = () => {
   const { pathname } = useLocation();
   const [searchParams, setSearchParams] = useSearchParams();
 
+  useBrowserTabSync();
   useFetchDashboards();
   useFetchDefaultTimezone();
   useFetchAppConfig();

app/vmui/packages/vmui/src/utils/favicon.tsx

@@ -0,0 +1,29 @@
+import faviconRaw from "../../assets/favicon.svg?raw";
+
+export const createFaviconUrl = (color = "#020202"): string => {
+  const svgDocument = new DOMParser().parseFromString(faviconRaw, "image/svg+xml");
+  const svg = svgDocument.documentElement;
+
+  if (svg.localName !== "svg") {
+    throw new Error("Invalid favicon SVG");
+  }
+
+  svg.setAttribute("fill", color);
+
+  const serializedSvg = new XMLSerializer().serializeToString(svg);
+  return `data:image/svg+xml;charset=utf-8,${encodeURIComponent(serializedSvg)}`;
+};
+
+export const updateFaviconColor = (color = "#020202"): void => {
+  const favicon = document.querySelector<HTMLLinkElement>("#favicon");
+  if (favicon) {
+    favicon.href = createFaviconUrl(color);
+  }
+
+  const maskIcon = document.querySelector<HTMLLinkElement>("#mask-icon");
+  if (maskIcon) {
+    maskIcon.setAttribute("color", color);
+  }
+};
+
+export const getFaviconStorageKey = () => window.location.pathname.replace(/\/+$/, "") || "/";

app/vmui/packages/vmui/src/utils/storage.ts

@@ -17,7 +17,11 @@ export const ALL_STORAGE_KEYS = [
   "POINTS_SHOW_ALL",
 ] as const;
 
-export type StorageKeys = (typeof ALL_STORAGE_KEYS)[number];
+export type FaviconStorageKey = `FAVICON_COLOR:${string}`;
+
+export type StorageKeys =
+  | (typeof ALL_STORAGE_KEYS)[number]
+  | FaviconStorageKey;
 
 type PrefixedStorageKeys = `${typeof STORAGE_PREFIX}${StorageKeys}`;
 
@@ -58,7 +62,10 @@ export const getFromStorage = (key: StorageKeys, withPrefix = true): undefined |
 
 export const removeFromStorage = (keys: StorageKeys[], withPrefix = true): void => {
   const storageKeys = withPrefix ? keys.map(toPrefixedKey) : keys;
-  storageKeys.forEach(k => window.localStorage.removeItem(k));
+  storageKeys.forEach(k => {
+    window.localStorage.removeItem(k);
+    window.dispatchEvent(new StorageEvent("storage", { key: k }));
+  });
 };
 
 /**

app/vmui/packages/vmui/src/utils/time.ts

@@ -205,19 +205,21 @@ export const getUTCByTimezone = (timezone: string) => {
 };
 
 export const getTimezoneList = (search = "") => {
-  const regexp = new RegExp(search, "i");
+  const normalizedSearch = search.toLowerCase();
 
-  return supportedTimezones.reduce((acc: {[key: string]: Timezone[]}, region) => {
+  return supportedTimezones.reduce((acc: { [key: string]: Timezone[] }, region) => {
     const zone = (region.match(/^(.*?)\//) || [])[1] || "unknown";
     const utc = getUTCByTimezone(region);
-    const utcForSearch = utc.replace(/UTC|0/, "");
+    const utcForSearch = utc.replace(/^UTC/, "");
     const regionForSearch = region.replace(/[/_]/g, " ");
+
     const item = {
       region,
       utc,
       search: `${region} ${utc} ${regionForSearch} ${utcForSearch}`
     };
-    const includeZone = !search || (search && regexp.test(item.search));
+
+    const includeZone = !normalizedSearch || item.search.toLowerCase().includes(normalizedSearch);
 
     if (includeZone && acc[zone]) {
       acc[zone].push(item);

app/vmui/packages/vmui/vite.config.ts

@@ -50,6 +50,13 @@ export default defineConfig(() => {
               return "vendor";
             }
           },
+          assetFileNames: (assetInfo) => {
+            if (assetInfo.names.includes("favicon.svg")) {
+              return "assets/favicon.svg";
+            }
+
+            return "assets/[name]-[hash][extname]";
+          },
         },
       },
     },

apptest/client.go

@@ -135,7 +135,7 @@ func tenantViaURL(addr, prefix, tenant, suffix string) string {
 }
 
 // tenantViaHeaders returns path in cluster's URL format where tenant is omitted in URL
-// Only supported if -enableMultitenancyViaHeaders is specified
+// Only supported if -enableMultitenancyViaHeaders is enabled
 func tenantViaHeaders(addr, prefix, suffix string) string {
 	return fmt.Sprintf("http://%s/%s/%s", addr, prefix, suffix)
 }

apptest/tests/multitenancy_via_headers_test.go

@@ -25,12 +25,10 @@ func TestClusterMultiTenantSelectViaHeaders(t *testing.T) {
 	})
 	vminsert := tc.MustStartVminsert("vminsert", []string{
 		"-storageNode=" + vmstorage.VminsertAddr(),
-		"-enableMultitenancyViaHeaders",
 	})
 	vmselect := tc.MustStartVmselect("vmselect", []string{
 		"-storageNode=" + vmstorage.VmselectAddr(),
 		"-search.tenantCacheExpireDuration=0",
-		"-enableMultitenancyViaHeaders",
 	})
 
 	multitenant := make(http.Header)

apptest/tests/vmagent_remotewrite_test.go

@@ -594,7 +594,6 @@ func TestSingleVMAgentMultitenancy(t *testing.T) {
 		fmt.Sprintf(`-remoteWrite.url=%s/api/v1/write`, remoteWriteSrv.URL),
 		"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent-multitenancy",
 		"-enableMultitenantHandlers",
-		"-enableMultitenancyViaHeaders",
 	})
 
 	vmagent.APIV1ImportPrometheus(t, []string{

docs/ai-tools/README.md

@@ -1,3 +1,11 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
 VictoriaMetrics Observability Stack integrates with AI assistants through [MCP servers](https://docs.victoriametrics.com/ai-tools/#mcp-servers)
 and [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills).
 The integrations allow AI agents and automation tools to query Metrics, Logs, and Traces, analyze telemetry data, 

docs/guides/connecting-vm-components-to-cloud-storage/README.md

@@ -1,3 +1,11 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
 Several VictoriaMetrics components can connect to cloud storage to read or write object data.
 
 The following table shows the supported types of storage for each component:

docs/guides/grafana-vmauth-openid-configuration/README.md

@@ -1,3 +1,11 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
 Using [Grafana](https://grafana.com/) with [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) is an effective way to provide [multi-tenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) access to your metrics, logs, and traces.
 vmauth provides a way to authenticate users using [JWT tokens](https://en.wikipedia.org/wiki/JSON_Web_Token) {{% available_from "v1.138.0" %}} issued by an external identity provider.
 Those tokens can include information about the user and their tenant, which vmauth can use to restrict access so users only see metrics in their own tenant.

docs/opentelemetry/README.md

@@ -1,3 +1,11 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
 VictoriaMetrics software provides native [OpenTelemetry](https://opentelemetry.io/) ingestion across **metrics**, **logs**, and **traces** via dedicated components.
 This allows running OpenTelemetry-based observability pipeline with VictoriaMetrics software as your backend.
 
@@ -88,4 +96,4 @@ Depending on the Grafana datasource plugin there could be multiple correlations
 1. Trace to metrics, metric to logs, metric to traces - see [correlations via VictoriaMetrics plugin](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/datasource/#correlations).
 1. Metrics to logs or traces correlations are possible via Prometheus datasource as well.
 1. Plugins Tempo, Jaeger, and Zipkin can correlate with logs or metrics using [Trace to logs](https://grafana.com/docs/grafana/latest/explore/trace-integration/#trace-to-logs)
-  and [Trace to metrics](https://grafana.com/docs/grafana/latest/visualizations/explore/trace-integration/#trace-to-metrics) feature.
+  and [Trace to metrics](https://grafana.com/docs/grafana/latest/visualizations/explore/trace-integration/#trace-to-metrics) feature.

docs/playgrounds/README.md

@@ -1,3 +1,11 @@
+---
+build:
+  list: never
+  publishResources: false
+  render: never
+sitemap:
+  disable: true
+---
 VictoriaMetrics offers public playgrounds where you can try the full observability stack online.
 
 Some playgrounds are based on the [OpenTelemetry Astronomy Shop demo](https://github.com/open-telemetry/opentelemetry-demo), a sample microservices application that generates realistic metrics, logs, and traces. Other playgrounds use benchmark workloads such as [prometheus-benchmark](https://github.com/VictoriaMetrics/prometheus-benchmark) to demonstrate ingestion and query performance for Prometheus-compatible systems.
@@ -158,4 +166,4 @@ Iximiuz Labs provides various [learning-by-doing resources for VictoriaMetrics](
   - [VictoriaMetrics cluster](https://labs.iximiuz.com/playgrounds/victoriametrics-cluster)
   - [VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/playgrounds/victoriametrics-kubernetes)
 
-Iximiuz Labs requires a [free account](https://labs.iximiuz.com/signup) to access the materials.
+Iximiuz Labs requires a [free account](https://labs.iximiuz.com/signup) to access the materials.

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -95,16 +95,15 @@ See also multitenancy [via headers](#multitenancy-via-headers) and [via labels](
 
 ### Multitenancy via headers
 
-By default, VictoriaMetrics allows specifying `accountID` and `projectID` only in the request URL.
+With `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag enabled (enabled by default {{% available_from "#" %}})
+tenant ID can be specified via HTTP headers `AccountID` and `ProjectID`. This flag needs to be enabled on vminserts and vmselects.
 
-Set `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag to support 
-specifying `accountID` and `projectID` via HTTP headers `AccountID` and `ProjectID` respectively.
-This flag needs to be specified separately for vminserts and vmselects.
-
-When `--enableMultitenancyViaHeaders` is enabled, [URL format](#url-format) can be simplified to the following:
+With `--enableMultitenancyViaHeaders` enabled [URL format](#url-format) can be simplified to the following:
 - `http://<vminsert>:8480/insert/<suffix>` for writes
 - `http://<vmselect>:8481/select/prometheus/<suffix>` for reads
 
+> Set --enableMultitenancyViaHeaders=false to disable simplified URL format.
+
 For example, the following query will only select metric `up` from `accountID=2` and `projectID=3`:
 ```
 curl 'https://<vmselect>:8481/select/prometheus/api/v1/query' \
@@ -291,7 +290,7 @@ If you need multi-AZ setup, then it is recommended running independent clusters
 into all the cluster - see [these docs](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy) for details.
 Then an additional `vmselect` nodes can be configured for reading the data from multiple clusters according to [these docs](#multi-level-cluster-setup).
 
-See [victoria-metrics-distributed chart](https://docs.victoriametrics.com/helm/victoria-metrics-distributed/) for an example.
+See [VMDistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/) Kubernetes operator resource for an example.
 
 ## Cluster setup
 

docs/victoriametrics/README.md

@@ -1306,7 +1306,7 @@ since it uses lower amounts of RAM, CPU and network bandwidth than Prometheus.
 If you use identically configured [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) instances for collecting the same data
 and sending it to VictoriaMetrics, then do not forget enabling [deduplication](#deduplication) at VictoriaMetrics side.
 
-See [victoria-metrics-distributed chart](https://docs.victoriametrics.com/helm/victoria-metrics-distributed/) for an example.
+See [VMDistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/) Kubernetes operator resource for an example.
 
 ## Deduplication
 

docs/victoriametrics/changelog/CHANGELOG.md

@@ -25,8 +25,22 @@ The sandbox cluster installation runs under the constant load generated by
 See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
 
 ## tip
+
+**Update Note 1:** `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: default value of `-enableMultitenancyViaHeaders` command-line flag has changed from `false` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default. With this change, mentioned components will start supporting URLs with omitted tenant ID in the path: `https://<vmselect>:8481/select/prometheus/api/v1/query` will become a valid URL. To disable multitenancy via headers and simplified URLs set `--enableMultitenancyViaHeaders=false` on vmagent, vminsert and vmselect. 
+
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): extend `-replay.continueWithExecutionErr` to also handle the `400 Bad Request` response code, since it is used for Prometheus querying API requests when request parameters are missing or incorrect. See [#11352](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11352).
+* FEATURE: `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: set default value of `-enableMultitenancyViaHeaders` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default, aligning VictoriaMetrics multitenancy behavior with [multitenancy in VictoriaLogs](https://docs.victoriametrics.com/victorialogs/#multitenancy). See related ticket [#11365](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11365).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add an option to customize the favicon color. This makes it easier to distinguish between different installations opened in multiple browser tabs. See [#11329](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11329).
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): avoid suggesting the unrelated `-enableTCP6` command-line flag when scraping a target over a Unix domain socket fails. See [#11320](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11320). Thanks to @lwmacct for contribution.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): skip labels with empty name at [/api/v1/import](https://docs.victoriametrics.com/victoriametrics/#how-to-import-data-in-json-line-format). Previously such a label replaced the metric name, so a series sent with `"metric":{"__name__":"foo","":"bar"}` was stored under the name `bar`. Other ingestion protocols already skip such labels. See [#4962](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4962). Thanks to @Vandit1604 for contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly parse small fractional Unix timestamps in timestamp args such as `start` and `end` in `/api/v1/query_range` and `--vm-native-filter-time-start` and `--vm-native-filter-time-end` in `vmctl`. Previously, fractional Unix timestamps with the integer part below `9223372` were interpreted with the wrong unit, for example `12.0` was parsed as `12000` seconds instead of `12` seconds. See [#11324](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11324).
-* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmstorage` and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): persist the previous working set cache during graceful shutdown when it is likely to contain the active working set. This prevents saving an empty or cold current cache right after split-mode cache rotation, which could otherwise slow down ingestion or queries after restart until the cache warms up again. See [#11299](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11299).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): change the HTTP response code for [Prometheus querying API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-usage) requests from `422 Unprocessable Entity` to `400 Bad Request` when request parameters are missing or incorrect. See [#11330](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11330).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): respect the custom query step specified via `g0.step_input` when opening a URL. Previously, it could be reset to the automatically calculated step and potentially cause dashboards to freeze. See [#11137](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11137).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly assign scrape target IP address at IPv6-only networks for [docker_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#docker_sd_configs). See [#10965](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10965).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): rename `vmalert_rule_group_results_limit` back to `vmalert_group_rule_results_limit`. The metric was introduced in [v1.147.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11470) but was accidentally given the wrong name. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): properly update group-level `eval_delay` and `eval_alignment` for existing groups during runtime when config reload is triggered periodically or manually via `/-/reload`. Previously, these settings weren't updated after config reload during runtime. See [#11374](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11374).
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): scale the default `-search.maxConcurrentRequests` with the number of available CPU cores instead of capping it at 16. See [#11191](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11191). Thanks to @Dhru1Tanna for contribution.
 
 ## [v1.149.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.149.0)
 
@@ -47,6 +61,7 @@ Released at 2026-08-05
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support [Prometheus native histograms](https://prometheus.io/docs/specs/native_histograms/) migration in [remote read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/). Native histograms are converted into `_count`, `_sum` and `_bucket` series with `vmrange` labels in the same way as VictoriaMetrics [converts native histograms received via Prometheus remote write protocol](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/#native-histograms), except that for native histograms with custom buckets the original bucket bounds are preserved instead of being estimated with the exponential formula. Previously native histograms were silently ignored in `SAMPLES` mode, while in stream mode the migration failed with `EOF` error. See [#11292](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11292). Thanks to @liuxu623 for contribution.
 * FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enable [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Previously, `-disableRerouting` defaulted to `true`, which limited ingestion throughput to the slowest `vmstorage` node. Now `-disableRerouting` defaults to `false`, so `vminsert` automatically routes data away from the slowest `vmstorage` node, improving overall ingestion performance. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the selected auto-refresh interval in the URL. See [VictoriaLogs#1310](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1310).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add client side least-loaded load-balancing with `DNS` discovery. See [#2388](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2388) and these [vmagent DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmagent/#dns-urls), [vmalert DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmalert/#dns-urls).
 
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): ignore HTTP proxy environment variables when scraping targets over Unix domain sockets. See [#11318](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11318). Thanks to @lwmacct for contribution.
@@ -84,7 +99,6 @@ Released at 2026-07-20
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): support scraping metrics over Unix domain sockets. The socket path can be configured via the `__unix_socket__` target label. See [#11156](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11156). Thanks to @vinyas-bharadwaj for contribution.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Improve background discovery performance for [http_sd](https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs) discovery. See [#8838](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8838).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add client side least-loaded load-balancing with `DNS` discovery. See [#2388](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2388) and these [vmagent DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmagent/#dns-urls), [vmalert DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmalert/#dns-urls).
 * FEATURE: [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-maxBackfillAge` command-line flag for limiting ingestion of samples with historical timestamps, for example, when older data has been moved between storage tiers (nvme/hdd, hot/cold). See [#11199](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11199). Thanks to @AshwinRamaniPsg for contribution.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): automatically preload relabeling rules configured via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` in the [metrics relabel debug UI](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabel-debugging). See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918).
 
@@ -106,6 +120,8 @@ Released at 2026-07-06
 
 **Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): contains a bug that causes increased CPU and memory usage when `-remoteWrite.urlRelabelConfig` or `-remoteWrite.streamAggr.config` flags are used. The bug was introduced in [#10854](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10854). Upgrade to v1.148.0 or rollback to v1.146.0. See [#11250](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11250).
 
+**Update Note 2:** [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): vmalert updates VictoriaLogs LogsQL query parser to [v1.51.0](https://docs.victoriametrics.com/victorialogs/changelog/#v1510), which contains a breaking change in LogsQL filter pipes handling. If you used vmalert with `vlogs` query type and query expressions contained deprecated syntax - these rules will fail the validation on vmalert restart. Please review the [VictoriaLogs v1.51.0 changelog](https://docs.victoriametrics.com/victorialogs/changelog/#v1510) and update your alerting rules accordingly before upgrading.
+
 * SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
 
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `default_vm_access_claim` field into `jwt` section of auth config. It could be used at [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), if `JWT` token doesn't have `vm_access` claim. See [#11054](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11054).

docs/victoriametrics/keyConcepts/keyConcepts.md

@@ -6,15 +6,16 @@ build:
 sitemap:
   disable: true
 ---
+
 ## Data model
 
 ### What is a metric
 
 Simply put, `metric` is a numeric measure or observation of something.
 
-The most common use-cases for metrics are:
+The most common use cases for metrics are:
 
-- check how the system behaves at the particular time period;
+- check how the system behaves at a particular time period;
 - correlate behavior changes to other measurements;
 - observe or forecast trends;
 - trigger events (alerts) if the metric exceeds a threshold.
@@ -25,7 +26,7 @@ Let's start with an example. To track how many requests our application serves,
 name `requests_total`.
 
 You can be more specific here by saying `requests_success_total` (for only successful requests)
-or `request_errors_total` (for requests which failed). Choosing a metric name is very important and supposed to clarify
+or `request_errors_total` (for requests which failed). Choosing a metric name is very important and is supposed to clarify
 what is actually measured to every person who reads it, just like **variable names** in programming.
 
 #### Labels
@@ -54,14 +55,14 @@ requests_total{path="/", code="200"}
 Labels can be automatically attached to the [time series](#time-series) 
 written via [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics) 
 or [Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/).
-VictoriaMetrics supports enforcing of label filters for [query API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-enhancements)
+VictoriaMetrics supports enforcing label filters for the [query API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-enhancements)
 to emulate data isolation. However, the real data isolation can be achieved via [multi-tenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).
 
 #### Time series
 
 A combination of a metric name and its labels defines a `time series`. For example,
 `requests_total{path="/", code="200"}` and `requests_total{path="/", code="403"}`
-are two different time series because they have different values for `code` label.
+are two different time series because they have different values for the `code` label.
 
 The number of unique time series has an impact on database resource usage.
 See [what is an active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) and
@@ -69,8 +70,8 @@ See [what is an active time series](https://docs.victoriametrics.com/victoriamet
 
 #### Cardinality
 
-The number of unique [time series](#time-series) is named `cardinality`. Too big number of unique time series is named `high cardinality`.
-High cardinality may result in increased resource usage at VictoriaMetrics.
+The number of unique [time series](#time-series) is named `cardinality`. Having too many unique time series is named `high cardinality`.
+High cardinality may result in increased resource usage in VictoriaMetrics.
 See [these docs](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-cardinality) for more details.
 
 #### Raw samples
@@ -108,13 +109,13 @@ of the [time series](https://docs.victoriametrics.com/victoriametrics/keyconcept
 | requests_total{path="/health", code="200"} |    4    |  1676297730 |
 ....
 ```
-Here we have a time series `requests_total{path="/health", code="200"}` which has a value update each `30s`.
-This means, its resolution is also a `30s`.
+Here we have a time series `requests_total{path="/health", code="200"}` which has a value updated every `30s`.
+This means its resolution is also `30s`.
 
 > In terms of [pull model](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#pull-model), resolution is equal 
 > to `scrape_interval` and is controlled by the monitoring system (server).
 > For [push model](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#push-model), resolution is an interval between
-> samples timestamps and is controlled by a client (metrics collector).
+> sample timestamps and is controlled by a client (metrics collector).
 
 Try to keep time series resolution consistent, since some [MetricsQL](#metricsql) functions may expect it to be so.
 
@@ -126,10 +127,10 @@ type exists specifically to help users to understand how the metric was measured
 
 #### Counter
 
-Counter is a metric, which counts some events. Its value increases or stays the same over time.
-It cannot decrease in general case. The only exception is e.g. `counter reset`,
+A counter is a metric that counts an event. Its value increases or stays the same over time.
+It cannot decrease in the general case. The only exception is, e.g., `counter reset`,
 when the metric resets to zero. The `counter reset` can occur when the service, which exposes the counter, restarts.
-So, the `counter` metric shows the number of observed events since the service start.
+So, the `counter` metric shows the number of observed events since the service started.
 
 In programming, `counter` is a variable that you **increment** each time something happens.
 
@@ -139,7 +140,7 @@ In programming, `counter` is a variable that you **increment** each time somethi
 above is that time series `vm_http_requests_total{instance="localhost:8428", job="victoriametrics", path="api/v1/query_range"}`
 was rapidly changing from 1:38 pm to 1:39 pm, then there were no changes until 1:41 pm.
 
-Counter is used for measuring the number of events, like the number of requests, errors, logs, messages, etc.
+A counter is used for measuring the number of events, like the number of requests, errors, logs, messages, etc.
 The most common [MetricsQL](#metricsql) functions used with counters are:
 
 * [rate](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate) - calculates the average per-second speed of metric change.
@@ -148,7 +149,7 @@ The most common [MetricsQL](#metricsql) functions used with counters are:
   time period specified in square brackets.
   For example, `increase(requests_total[1h])` shows the number of requests served over the last hour.
 
-It is OK to have fractional counters. For example, `request_duration_seconds_sum` counter may sum the durations of all the requests.
+It is OK to have fractional counters. For example, the `request_duration_seconds_sum` counter may sum the durations of all the requests.
 Every duration may have a fractional value in seconds, e.g. `0.5` of a second. So the cumulative sum of all the request durations
 may be fractional too.
 
@@ -162,12 +163,12 @@ Gauge is used for measuring a value that can go up and down:
 ![gauge](gauge.webp)
 
 The metric `process_resident_memory_anon_bytes` on the graph shows the memory usage of the application at every given time.
-It is changing frequently, going up and down showing how the process allocates and frees the memory.
+It is changing frequently, going up and down, showing how the process allocates and frees the memory.
 In programming, `gauge` is a variable to which you **set** a specific value as it changes.
 
 Gauge is used in the following scenarios:
 
-* measuring temperature, memory usage, disk usage etc;
+* measuring temperature, memory usage, disk usage, etc;
 * storing the state of some process. For example, gauge `config_reloaded_successful` can be set to `1` if everything is
   good, and to `0` if configuration failed to reload;
 * storing the timestamp when the event happened. For example, `config_last_reload_success_timestamp_seconds`
@@ -178,11 +179,11 @@ and [rollup functions](https://docs.victoriametrics.com/victoriametrics/metricsq
 
 #### Histogram
 
-Histogram is a set of [counter](#counter) metrics with different `vmrange` or `le` labels.
+A histogram is a set of [counter](#counter) metrics with different `vmrange` or `le` labels.
 The `vmrange` or `le` labels define measurement boundaries of a particular bucket.
 When the observed measurement hits a particular bucket, then the corresponding counter is incremented.
 
-Histogram buckets usually have `_bucket` suffix in their names.
+Histogram buckets usually have a `_bucket` suffix in their names.
 For example, VictoriaMetrics tracks the distribution of rows processed per query with the `vm_rows_read_per_query` histogram.
 The exposition format for this histogram has the following form:
 
@@ -200,10 +201,10 @@ The `vm_rows_read_per_query_bucket{vmrange="4.084e+02...4.642e+02"} 2` line mean
 that there were 2 queries with the number of rows in the range `(408.4 - 464.2]`
 since the last VictoriaMetrics start.
 
-The counters ending with `_bucket` suffix allow estimating arbitrary percentile
+The counters ending with the `_bucket` suffix allow estimating arbitrary percentiles
 for the observed measurement with the help of [histogram_quantile](https://docs.victoriametrics.com/victoriametrics/metricsql/#histogram_quantile)
 function. For example, the following query returns the estimated 99th percentile
-on the number of rows read per each query during the last hour (see `1h` in square brackets):
+on the number of rows read per query during the last hour (see `1h` in square brackets):
 
 ```metricsql
 histogram_quantile(0.99, sum(increase(vm_rows_read_per_query_bucket[1h])) by (vmrange))
@@ -215,15 +216,15 @@ This query works in the following way:
    number of events over the last hour.
 1. The `sum(...) by (vmrange)` calculates per-bucket events by summing per-instance buckets
    with the same `vmrange` values.
-1. The `histogram_quantile(0.99, ...)` calculates 99th percentile over `vmrange` buckets returned at step 2.
+1. The `histogram_quantile(0.99, ...)` calculates the 99th percentile over `vmrange` buckets returned at step 2.
 
 Histogram metric type exposes two additional counters ending with `_sum` and `_count` suffixes:
 
 - the `vm_rows_read_per_query_sum` is a sum of all the observed measurements,
-  e.g. the sum of rows served by all the queries since the last VictoriaMetrics start.
+  e.g., the sum of rows served by all the queries since the last VictoriaMetrics start.
 
 - the `vm_rows_read_per_query_count` is the total number of observed events,
-  e.g. the total number of observed queries since the last VictoriaMetrics start.
+  e.g., the total number of observed queries since the last VictoriaMetrics start.
 
 These counters allow calculating the average measurement value on a particular lookbehind window.
 For example, the following query calculates the average number of rows read per query
@@ -233,7 +234,7 @@ during the last 5 minutes (see `5m` in square brackets):
 increase(vm_rows_read_per_query_sum[5m]) / increase(vm_rows_read_per_query_count[5m])
 ```
 
-The `vm_rows_read_per_query` histogram may be used in Go application in the following way
+The `vm_rows_read_per_query` histogram may be used in a Go application in the following way
 by using the [github.com/VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) package:
 
 ```go
@@ -246,7 +247,7 @@ for _, query := range queries {
 }
 ```
 
-Now let's see what happens each time when `rowsReadPerQuery.Update` is called:
+Now let's see what happens each time `rowsReadPerQuery.Update` is called:
 
 * counter `vm_rows_read_per_query_sum` is incremented by value of `len(query.Rows)` expression;
 * counter `vm_rows_read_per_query_count` increments by 1;
@@ -262,7 +263,7 @@ and calculating [quantiles](https://prometheus.io/docs/practices/histograms/#qua
 Grafana doesn't understand buckets with `vmrange` labels, so the [prometheus_buckets](https://docs.victoriametrics.com/victoriametrics/metricsql/#prometheus_buckets)
 function must be used for converting buckets with `vmrange` labels to buckets with `le` labels before building heatmaps in Grafana.
 
-Histograms are usually used for measuring the distribution of latency, sizes of elements (batch size, for example) etc. There are two
+Histograms are usually used for measuring the distribution of latency, sizes of elements (batch size, for example), etc. There are two
 implementations of a histogram supported by VictoriaMetrics:
 
 1. [Prometheus histogram](https://prometheus.io/docs/practices/histograms/). The canonical histogram implementation is
@@ -271,7 +272,7 @@ implementations of a histogram supported by VictoriaMetrics:
    histogram requires a user to define ranges (`buckets`) statically.
 1. [VictoriaMetrics histogram](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350)
    supported by [VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) instrumentation library.
-   Victoriametrics histogram automatically handles bucket boundaries, so users don't need to think about them.
+   VictoriaMetrics histogram automatically handles bucket boundaries, so users don't need to think about them.
 
 We recommend reading the following articles before you start using histograms:
 
@@ -303,7 +304,7 @@ The visualization of summaries is pretty straightforward:
 
 Such an approach makes summaries easier to use but also puts significant limitations compared to [histograms](#histogram):
 
-- It is impossible to calculate quantile over multiple summary metrics, e.g. `sum(go_gc_duration_seconds{quantile="0.75"})`,
+- It is impossible to calculate a quantile over multiple summary metrics, e.g. `sum(go_gc_duration_seconds{quantile="0.75"})`,
   `avg(go_gc_duration_seconds{quantile="0.75"})` or `max(go_gc_duration_seconds{quantile="0.75"})`
   won't return the expected 75th percentile over `go_gc_duration_seconds` metrics collected from multiple instances
   of the application. See [this article](https://latencytipoftheday.blogspot.de/2014/06/latencytipoftheday-you-cant-average.html) for details.
@@ -313,16 +314,16 @@ Such an approach makes summaries easier to use but also puts significant limitat
 - It is impossible to calculate quantiles for measurements collected over an arbitrary time range. Usually, `summary`
   quantiles are calculated over a fixed time range such as the last 5 minutes.
 
-Summaries are usually used for tracking the pre-defined percentiles for latency, sizes of elements (batch size, for example) etc.
+Summaries are usually used for tracking the pre-defined percentiles for latency, sizes of elements (batch size, for example), etc.
 
 ### Instrumenting application with metrics
 
 As was said at the beginning of the [types of metrics](#types-of-metrics) section, metric type defines how it was
 measured. VictoriaMetrics TSDB doesn't know about metric types. All it sees are metric names, labels, values, and timestamps.
-What are these metrics, what do they measure, and how - all this depends on the application which emits them.
+What these metrics are, what they measure, and how - all these depend on the application which emits them.
 
-To instrument your application with metrics compatible with VictoriaMetrics we recommend
-using [github.com/VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) package.
+To instrument your application with metrics compatible with VictoriaMetrics, we recommend
+using the [github.com/VictoriaMetrics/metrics](https://github.com/VictoriaMetrics/metrics) package.
 See more details on how to use it in [this article](https://victoriametrics.medium.com/how-to-monitor-go-applications-with-victoriametrics-c04703110870).
 
 VictoriaMetrics is also compatible with [Prometheus client libraries for metrics instrumentation](https://prometheus.io/docs/instrumenting/clientlibs/).
@@ -331,20 +332,20 @@ VictoriaMetrics is also compatible with [Prometheus client libraries for metrics
 
 We recommend following [Prometheus naming convention for metrics](https://prometheus.io/docs/practices/naming/). There
 are no strict restrictions, so any metric name and labels are accepted by VictoriaMetrics.
-But the convention helps to keep names meaningful, descriptive, and clear to other people.
-Following convention is a good practice.
+But this convention helps to keep names meaningful, descriptive, and clear to other people.
+Following the convention is a good practice.
 
 #### Labels
 
 Every measurement can contain an arbitrary number of `key="value"` labels. The good practice is to keep this number limited.
-Otherwise, it would be difficult to deal with measurements containing a big number of labels.
+Otherwise, it would be difficult to deal with measurements containing a large number of labels.
 By default, VictoriaMetrics limits the number of labels per measurement to `40` and drops other labels.
-This limit can be changed via `-maxLabelsPerTimeseries` command-line flag if necessary (but this isn't recommended).
+This limit can be changed via the `-maxLabelsPerTimeseries` command-line flag if necessary (but this isn't recommended).
 
 Every label value can contain an arbitrary string value. The good practice is to use short and meaningful label values to
 describe the attribute of the metric, not to tell the story about it. For example, label-value pair
-`environment="prod"` is ok, but `log_message="long log message with a lot of details..."` is not ok. By default,
-VictoriaMetrics limits label's value size with 4KiB. This limit can be changed via `-maxLabelValueLen` command-line flag.
+`environment="prod"` is OK, but `log_message="long log message with a lot of details..."` is not OK. By default,
+VictoriaMetrics limits label values to 4KiB. This limit can be changed via the `-maxLabelValueLen` command-line flag.
 
 It is very important to keep under control the number of unique label values, since every unique label value
 leads to a new [time series](#time-series). Try to avoid using volatile label values such as session ID or query ID in order to
@@ -356,7 +357,7 @@ avoid excessive resource usage and database slowdown.
 supports [multi-tenancy](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
 for data isolation.
 
-Multi-tenancy can be emulated for [single-server](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) 
+Multi-tenancy can be emulated for the [single-server](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) 
 version of VictoriaMetrics by adding [labels](#labels) on [write path](#write-data)
 and enforcing [labels filtering](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#prometheus-querying-api-enhancements) 
 on [read path](#query-data).
@@ -391,10 +392,10 @@ It is allowed to push/write metrics to [single-node VictoriaMetrics](https://doc
 to [cluster component vminsert](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#architecture-overview)
 and to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
 
-The pros of push model:
+The pros of the push model:
 
 * Simpler configuration at VictoriaMetrics side - there is no need to configure VictoriaMetrics with locations of the monitored applications.
-  There is no need in complex [service discovery schemes](https://docs.victoriametrics.com/victoriametrics/sd_configs/).
+  There is no need for complex [service discovery schemes](https://docs.victoriametrics.com/victoriametrics/sd_configs/).
 * Simpler security setup - there is no need to set up access from VictoriaMetrics to each monitored application.
 
 See [Foiled by the Firewall: A Tale of Transition From Prometheus to VictoriaMetrics](https://www.percona.com/blog/2020/12/01/foiled-by-the-firewall-a-tale-of-transition-from-prometheus-to-victoriametrics/)
@@ -406,18 +407,18 @@ The cons of push protocol:
   Every application needs to be individually configured with the address of the monitoring system
   for metrics delivery. It also needs to be configured with the interval between metric pushes
   and the strategy in case of metric delivery failure.
-* Non-trivial setup for metrics' delivery into multiple monitoring systems.
+* Non-trivial setup for metrics delivery into multiple monitoring systems.
 * It may be hard to tell whether the application went down or just stopped sending metrics for a different reason.
 * Applications can overload the monitoring system by pushing metrics at too short intervals.
 
 ### Pull model
 
-Pull model is an approach popularized by [Prometheus](https://prometheus.io/), where the monitoring system decides when
+The pull model is an approach popularized by [Prometheus](https://prometheus.io/), where the monitoring system decides when
 and where to pull metrics from:
 
 ![pull model](pull_model.webp)
 
-In pull model, the monitoring system needs to be aware of all the applications it needs to monitor. The metrics are
+In the pull model, the monitoring system needs to be aware of all the applications it needs to monitor. The metrics are
 scraped (pulled) from the known applications (aka `scrape targets`) via HTTP protocol on a regular basis (aka `scrape_interval`).
 
 VictoriaMetrics supports discovering Prometheus-compatible targets and scraping metrics from them in the same way as Prometheus does -
@@ -431,7 +432,7 @@ The pros of the pull model:
 * Easier to debug - VictoriaMetrics knows about all the monitored applications (aka `scrape targets`).
   The `up == 0` query instantly shows unavailable scrape targets.
   The actual information about scrape targets is available at `http://victoriametrics:8428/targets` and `http://vmagent:8429/targets`.
-* Monitoring system controls the frequency of metrics' scrape, so it is easier to control its load.
+* The monitoring system controls the frequency of metrics scraping, so it is easier to control its load.
 * Applications aren't aware of the monitoring system and don't need to implement the logic for metrics delivery.
 
 The cons of the pull model:
@@ -448,13 +449,13 @@ The most common approach for data collection is using both models:
 
 ![data collection](data_collection.webp)
 
-In this approach the additional component is used - [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/). Vmagent is
-a lightweight agent whose main purpose is to collect, filter, relabel and deliver metrics to VictoriaMetrics.
+In this approach, the additional component is used - [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/). Vmagent is
+a lightweight agent whose main purpose is to collect, filter, relabel, and deliver metrics to VictoriaMetrics.
 It supports all [push](#push-model) and [pull](#pull-model) protocols mentioned above.
 
 The basic monitoring setup of VictoriaMetrics and vmagent is described
 in the [example docker-compose manifest](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#readme).
-In this example vmagent [scrapes a list of targets](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-single.yml)
+In this example, vmagent [scrapes a list of targets](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-single.yml)
 and [forwards collected data to VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/9751ea10983d42068487624849cac7ad6fd7e1d8/deployment/docker/compose-vm-single.yml#L16).
 VictoriaMetrics is then used as a [datasource for Grafana](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/provisioning/datasources/prometheus/single.yml)
 installation for querying collected data.
@@ -480,7 +481,7 @@ The API consists of two main handlers for serving [instant queries](#instant-que
 
 ### Instant query
 
-Instant query executes the `query` expression at the given `time`:
+An instant query executes the `query` expression at the given `time`:
 
 ```
 GET | POST /api/v1/query?query=...&time=...&step=...&timeout=...
@@ -497,13 +498,13 @@ Params:
   For example, the request `/api/v1/query?query=up&step=1m` looks for the last written raw sample for the metric `up`
   in the `(now()-1m, now()]` interval (the first millisecond is not included). If omitted, `step` is set to `5m` (5 minutes)
   by default.
-* `timeout` - optional query timeout. For example, `timeout=5s`. Query is canceled when the timeout is reached.
-  By default the timeout is set to the value of `-search.maxQueryDuration` command-line flag passed to single-node VictoriaMetrics
-  or to `vmselect` component of VictoriaMetrics cluster.
+* `timeout` - optional query timeout. For example, `timeout=5s`. The query is canceled when the timeout is reached.
+  By default, the timeout is set to the value of the `-search.maxQueryDuration` command-line flag passed to the single-node VictoriaMetrics
+  or to the `vmselect` component of the VictoriaMetrics cluster.
 
-The result of Instant query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
-matching the filter in `query` expression. Each returned series contains exactly one `(timestamp, value)` entry,
-where `timestamp` equals to the `time` query arg, while the `value` contains `query` result at the requested `time`.
+The result of an Instant query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
+matching the filter in the `query` expression. Each returned series contains exactly one `(timestamp, value)` entry,
+where `timestamp` equals the `time` query arg, while the `value` contains the `query` result at the requested `time`.
 
 To understand how instant queries work, let's begin with a data sample:
 
@@ -530,7 +531,7 @@ ranging from 1m to 3m. If we plot this data sample on the graph, it will have th
 {width="500"}
     
 To get the value of the `foo_bar` series at some specific moment of time, for example `2022-05-10T08:03:00Z`, in
-VictoriaMetrics we need to issue an **instant query**:
+VictoriaMetrics, we need to issue an **instant query**:
 
 ```sh
 curl "http://<victoria-metrics-addr>/api/v1/query?query=foo_bar&time=2022-05-10T08:03:00.000Z"
@@ -595,13 +596,13 @@ Params:
   The `query` is executed at `start`, `start+step`, `start+2*step`, ..., `start+N*step` timestamps,
   where `N` is the whole number of steps that fit between `start` and `end`.
   `end` is included only when it equals to `start+N*step`.
-  If the `step` isn't set, then it default to `5m` (5 minutes).
-* `timeout` - optional query timeout. For example, `timeout=5s`. Query is canceled when the timeout is reached.
-  By default the timeout is set to the value of `-search.maxQueryDuration` command-line flag passed to single-node VictoriaMetrics
-  or to `vmselect` component in VictoriaMetrics cluster.
+  If the `step` isn't set, then it defaults to `5m` (5 minutes).
+* `timeout` - optional query timeout. For example, `timeout=5s`. The query is canceled when the timeout is reached.
+  By default, the timeout is set to the value of the `-search.maxQueryDuration` command-line flag passed to the single-node VictoriaMetrics
+  or to the `vmselect` component in a VictoriaMetrics cluster.
 
-The result of Range query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
-matching the filter in `query` expression. Each returned series contains `(timestamp, value)` results for the `query` executed
+The result of a Range query is a list of [time series](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#time-series)
+matching the filter in the `query` expression. Each returned series contains `(timestamp, value)` results for the `query` executed
 at `start`, `start+step`, `start+2*step`, ..., `start+N*step` timestamps. In other words, Range query is an [Instant query](#instant-query)
 executed independently at `start`, `start+step`, ..., `start+N*step` timestamps with the only difference that an instant query
 does not return `ephemeral` samples (see below). Instead, if the database does not contain any samples for the requested time and step,
@@ -705,7 +706,7 @@ In response, VictoriaMetrics returns `17` sample-timestamp pairs for the series
 from `2022-05-10T07:59:00Z` to `2022-05-10T08:17:00Z`. But, if we take a look at the original data sample again, we'll
 see that it contains only 13 raw samples. What happens here is that the range query is actually
 an [instant query](#instant-query) executed `1 + (start-end)/step` times on the time range from `start` to `end`. If we plot
-this request in VictoriaMetrics the graph will be shown as the following:
+this request in VictoriaMetrics, the graph will be shown as follows:
 
 ![range query](range_query.webp)
 {width="500"}
@@ -720,13 +721,13 @@ This behavior of adding ephemeral data points comes from the specifics of the [p
 * Scrape may be skipped if the monitoring system is overloaded.
 * Scrape may fail due to network issues.
 
-According to these specifics, the range query assumes that if there is a missing raw sample then it is likely a missed
+According to these specifics, the range query assumes that if there is a missing raw sample, then it is likely a missed
 scrape, so it fills it with the previous raw sample. The same will work for cases when `step` is lower than the actual
 interval between samples. In fact, if we set `step=1s` for the same request, we'll get about 1 thousand data points in
 response, where most of them are `ephemeral`.
 
 Sometimes, the lookbehind window for locating the datapoint isn't big enough and the graph will contain a gap. For range
-queries, lookbehind window isn't equal to the `step` parameter. It is calculated as the median of the intervals between
+queries, the lookbehind window isn't equal to the `step` parameter. It is calculated as the median of the intervals between
 the last 20 raw samples in the requested time range. In this way, VictoriaMetrics automatically adjusts the lookbehind
 window to fill gaps and detect stale series at the same time.
 
@@ -734,7 +735,7 @@ Range queries are mostly used for plotting time series data over specified time
 useful in the following scenarios:
 
 * Track the state of a metric on the given time interval;
-* Correlate changes between multiple metrics on the time interval;
+* Correlate changes between multiple metrics over the time interval;
 * Observe trends and dynamics of the metric change.
 
 If you need to export raw samples from VictoriaMetrics, then take a look at [export APIs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-time-series).
@@ -745,7 +746,7 @@ By default, Victoria Metrics does not immediately return the recently written sa
 written prior to the time specified by the `-search.latencyOffset` command-line flag, which has a default offset of 30 seconds.
 This is true for both `query` and `query_range` and may give the impression that data is written to the VM with a 30-second delay.
 
-This flag prevents from non-consistent results due to the fact that only part of the values are scraped in the last scrape interval.
+This flag prevents inconsistent results due to the fact that only part of the values are scraped in the last scrape interval.
 
 Here is an illustration of a potential problem when `-search.latencyOffset` is set to zero:
 
@@ -758,12 +759,12 @@ duration throughout the `-search.latencyOffset` duration:
 ![with latency offset](with_latencyOffset.webp)
 {width="1000"}
 
-It can be overridden on per-query basis via `latency_offset` query arg.
+It can be overridden on a per-query basis via the `latency_offset` query arg.
 
 VictoriaMetrics buffers recently ingested samples in memory for up to a few seconds and then periodically flushes these samples to disk.
-This buffering improves data ingestion performance. The buffered samples are invisible in query results, even if `-search.latencyOffset` command-line flag is set to 0,
+This buffering improves data ingestion performance. The buffered samples are invisible in query results, even if the `-search.latencyOffset` command-line flag is set to 0,
 or if `latency_offset` query arg is set to 0.
-You can send GET request to `/internal/force_flush` http handler at single-node VictoriaMetrics
+You can send a GET request to the `/internal/force_flush` HTTP handler at a single-node VictoriaMetrics
 or to `vmstorage` at [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/)
 in order to forcibly flush the buffered samples to disk, so they become visible for querying. The `/internal/force_flush` handler
 is provided for debugging and testing purposes only. Do not call it in production, since this may significantly slow down data ingestion
@@ -771,7 +772,7 @@ performance and increase resource usage.
 
 ### MetricsQL
 
-VictoriaMetrics provide a special query language for executing read queries - [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/).
+VictoriaMetrics provides a special query language for executing read queries - [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/).
 It is a [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics)-like query language with a powerful set of
 functions and features for working specifically with time series data. MetricsQL is backward-compatible with PromQL,
 so it shares most of the query concepts. The basic concepts for PromQL and MetricsQL are
@@ -779,7 +780,7 @@ described [here](https://valyala.medium.com/promql-tutorial-for-beginners-9ab455
 
 #### Filtering
 
-In sections [instant query](#instant-query) and [range query](#range-query) we've already used MetricsQL to get data for
+In sections [instant query](#instant-query) and [range query](#range-query), we've already used MetricsQL to get data for
 metric `foo_bar`. It is as simple as just writing a metric name in the query:
 
 ```metricsql
@@ -793,14 +794,14 @@ requests_total{path="/", code="200"}
 requests_total{path="/", code="403"} 
 ```
 
-To select only time series with specific label value specify the matching filter in curly braces:
+To select only time series with a specific label value, specify the matching filter in curly braces:
 
 ```metricsql
 requests_total{code="200"} 
 ```
 
 The query above returns all time series with the name `requests_total` and label `code="200"`. We use the operator `=` to
-match label value. For negative match use `!=` operator. Filters also support positive regex matching via `=~`
+match the label value. For negative matches, use the `!=` operator. Filters also support positive regex matching via `=~`
 and negative regex matching via `!~`:
 
 ```metricsql
@@ -813,7 +814,7 @@ Filters can also be combined:
 requests_total{code=~"200", path="/home"}
 ```
 
-The query above returns all time series with `requests_total` name, which simultaneously have labels `code="200"` and `path="/home"`.
+The query above returns all time series with the `requests_total` name, which simultaneously have labels `code="200"` and `path="/home"`.
 
 #### Filtering by name
 
@@ -829,7 +830,7 @@ The query above returns series for two metrics: `requests_error_total` and `requ
 
 #### Filtering by multiple "or" filters
 
-[MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) supports selecting time series, which match at least one of multiple "or" filters.
+[MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) supports selecting time series that match at least one of multiple "or" filters.
 Such filters must be delimited by `or` inside curly braces. For example, the following query selects time series with
 `{job="app1",env="prod"}` or `{job="app2",env="dev"}` labels:
 
@@ -838,7 +839,7 @@ Such filters must be delimited by `or` inside curly braces. For example, the fol
 ```
 
 The number of `or` groups can be arbitrary. The number of `,`-delimited label filters per each `or` group can be arbitrary.
-Per-group filters are applied with `and` operation, e.g. they select series simultaneously matching all the filters in the group.
+Per-group filters are applied with the `and` operation, e.g., they select series simultaneously matching all the filters in the group.
 
 This functionality allows passing the selected series to [rollup functions](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions)
 such as [rate()](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate)
@@ -849,7 +850,7 @@ rate({job="app1",env="prod" or job="app2",env="dev"}[5m])
 
 ```
 
-If you need to select series matching multiple filters for the same label, then it is better from performance PoV
+If you need to select series matching multiple filters for the same label, then it is better from a performance PoV
 to use regexp filter `{label=~"value1|...|valueN"}` instead of `{label="value1" or ... or label="valueN"}`.
 
 
@@ -878,8 +879,8 @@ query may break or may lead to incorrect results. The basics of the matching rul
 
 * MetricsQL engine strips metric names from all the time series on the left and right side of the arithmetic operation
   without touching labels.
-* For each time series on the left side MetricsQL engine searches for the corresponding time series on the right side
-  with the same set of labels, applies the operation for each data point and returns the resulting time series with the
+* For each time series on the left side, the MetricsQL engine searches for the corresponding time series on the right side
+  with the same set of labels, applies the operation for each data point, and returns the resulting time series with the
   same set of labels. If there are no matches, then the time series is dropped from the result.
 * The matching rules may be augmented with `ignoring`, `on`, `group_left` and `group_right` modifiers.
   See [these docs](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching) for details.
@@ -896,7 +897,7 @@ MetricsQL supports the following comparison operators:
 * less-or-equal - `<=`
 
 These operators may be applied to arbitrary MetricsQL expressions as with arithmetic operators. The result of the
-comparison operation is time series with only matching data points. For instance, the following query would return
+comparison operation is a time series with only matching data points. For instance, the following query would return
 series only for processes where memory usage exceeds `100MB`:
 
 ```metricsql
@@ -906,7 +907,7 @@ process_resident_memory_bytes > 100*1024*1024
 #### Aggregation and grouping functions
 
 MetricsQL allows aggregating and grouping of time series. Time series are grouped by the given set of labels and then the
-given aggregation function is applied individually per each group. For instance, the following query returns
+given aggregation function is applied individually to each group. For instance, the following query returns
 summary memory usage for each `job`:
 
 ```metricsql
@@ -919,14 +920,14 @@ See [docs for aggregate functions in MetricsQL](https://docs.victoriametrics.com
 
 One of the most widely used functions for [counters](#counter)
 is [rate](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate). It calculates the average per-second increase rate individually
-per each matching time series. For example, the following query shows the average per-second data receive speed
-per each monitored `node_exporter` instance, which exposes the `node_network_receive_bytes_total` metric:
+for each matching time series. For example, the following query shows the average per-second data receive speed
+for each monitored `node_exporter` instance, which exposes the `node_network_receive_bytes_total` metric:
 
 ```metricsql
 rate(node_network_receive_bytes_total)
 ```
 
-By default, VictoriaMetrics calculates the `rate` over [raw samples](#raw-samples) on the lookbehind window specified in the `step` param
+By default, VictoriaMetrics calculates the `rate` over [raw samples](#raw-samples) on the lookbehind window specified in the `step` parameter
 passed either to [instant query](#instant-query) or to [range query](#range-query).
 The interval on which `rate` needs to be calculated can be specified explicitly
 as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations) in square brackets:
@@ -935,10 +936,10 @@ as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#floa
  rate(node_network_receive_bytes_total[5m])
 ```
 
-In this case VictoriaMetrics uses the specified lookbehind window - `5m` (5 minutes) - for calculating the average per-second increase rate.
+In this case, VictoriaMetrics uses the specified lookbehind window - `5m` (5 minutes) - for calculating the average per-second increase rate.
 Bigger lookbehind windows usually lead to smoother graphs.
 
-`rate` strips metric name while leaving all the labels for the inner time series. If you need to keep the metric name,
+`rate` strips the metric name while leaving all the labels for the inner time series. If you need to keep the metric name,
 then add [keep_metric_names](https://docs.victoriametrics.com/victoriametrics/metricsql/#keep_metric_names) modifier
 after the `rate(..)`. For example, the following query leaves metric names after calculating the `rate()`:
 
@@ -952,7 +953,7 @@ rate(node_network_receive_bytes_total) keep_metric_names
 
 VictoriaMetrics has a built-in graphical User Interface for querying and visualizing metrics -
 [VMUI](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui).
-Open `http://victoriametrics:8428/vmui` page, type the query and see the results:
+Open the `http://victoriametrics:8428/vmui` page, type the query, and see the results:
 
 ![vmui](vmui.webp)
 
@@ -963,8 +964,8 @@ in the same way as Grafana queries Prometheus.
 ## Modify data
 
 VictoriaMetrics stores time series data in [MergeTree](https://en.wikipedia.org/wiki/Log-structured_merge-tree)-like
-data structures. While this approach is very efficient for write-heavy databases, it applies some limitations on data
-updates. In short, modifying already written [time series](#time-series) requires re-writing the whole data block where
+data structures. While this approach is very efficient for write-heavy databases, it imposes some limitations on data
+updates. In short, modifying already written [time series](#time-series) requires rewriting the whole data block where
 it is stored. Due to this limitation, VictoriaMetrics does not support direct data modification.
 
 ### Deletion

docs/victoriametrics/vmagent.md

@@ -642,7 +642,7 @@ specified via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` c
 
 vmagent can write data to multiple distinct tenants if:
 * its `-remoteWrite.url` points to the [VictoriaMetrics cluster multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
-* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` command-line flags are both set
+* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` (enabled by default {{% available_from "#" %}}) command-line flags are both set
 * clients ingest data into vmagent with the tenants specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}
 
 ```mermaid

docs/victoriametrics/vmalert_common_flags.md

@@ -405,7 +405,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
   -remoteWrite.url string
      Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
   -replay.continueWithExecutionErr
-     Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
+     Whether to continue replaying other rules if a rule execution fails with a 400 or 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int

lib/httpserver/httpserver.go

@@ -488,7 +488,8 @@ func isProtectedByAuthFlag(path string) bool {
 	return strings.HasSuffix(path, "/config") || strings.HasSuffix(path, "/reload") ||
 		strings.HasSuffix(path, "/resetRollupResultCache") || strings.HasSuffix(path, "/delSeries") || strings.HasSuffix(path, "/delete_series") ||
 		strings.HasSuffix(path, "/force_merge") || strings.HasSuffix(path, "/force_flush") || strings.HasSuffix(path, "/snapshot") ||
-		strings.HasPrefix(path, "/snapshot/") || strings.HasSuffix(path, "/admin/status/metric_names_stats/reset")
+		strings.HasPrefix(path, "/snapshot/") || strings.HasSuffix(path, "/admin/status/metric_names_stats/reset") ||
+		strings.HasSuffix(path, "/remotewrite/maintenance")
 }
 
 // CheckAuthFlag checks whether the given authKey is set and valid

lib/httpserver/prometheus.go

@@ -5,9 +5,18 @@ import (
 	"net/http"
 )
 
-// SendPrometheusError sends err to w in Prometheus querying API response format.
-//
-// See https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview for more details
+// InvalidParamError sets HTTP status code to 400 Bad Request for Prometheus querying APIs when parameters are missing or incorrect,
+// see https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview.
+func InvalidParamError(err error) *ErrorWithStatusCode {
+	return &ErrorWithStatusCode{
+		Err:        err,
+		StatusCode: http.StatusBadRequest,
+	}
+}
+
+// SendPrometheusError sends err to w in Prometheus querying API response format,
+// and sets HTTP status code to 422 Unprocessable Entity when code is not set,
+// see https://prometheus.io/docs/prometheus/latest/querying/api/#format-overview for more details.
 func SendPrometheusError(w http.ResponseWriter, r *http.Request, err error) {
 	errStr := err.Error()
 	logHTTPError(r, errStr)

lib/netutil/statdial.go

@@ -2,6 +2,7 @@ package netutil
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"strconv"
@@ -41,7 +42,7 @@ func newStatDialFunc(metricPrefix string, dialFunc func(ctx context.Context, net
 		sc.dialsTotal.Inc()
 		if err != nil {
 			sc.dialErrors.Inc()
-			if !TCP6Enabled() && !isTCPv4Addr(addr) {
+			if !TCP6Enabled() && !isTCPv4Addr(addr) && !isUnixSocketDialError(err) {
 				err = fmt.Errorf("%w; try -enableTCP6 command-line flag for dialing ipv6 addresses", err)
 			}
 			return nil, err
@@ -52,6 +53,14 @@ func newStatDialFunc(metricPrefix string, dialFunc func(ctx context.Context, net
 	}
 }
 
+func isUnixSocketDialError(err error) bool {
+	var opErr *net.OpError
+	if !errors.As(err, &opErr) || opErr.Addr == nil {
+		return false
+	}
+	return opErr.Addr.Network() == "unix"
+}
+
 type statDialConn struct {
 	closed atomic.Int32
 	net.Conn

lib/netutil/statdial_test.go

@@ -1,9 +1,38 @@
 package netutil
 
 import (
+	"errors"
+	"fmt"
+	"net"
 	"testing"
 )
 
+func TestIsUnixSocketDialError(t *testing.T) {
+	f := func(err error, want bool) {
+		t.Helper()
+
+		got := isUnixSocketDialError(err)
+		if got != want {
+			t.Fatalf("unexpected result; got %v; want %v", got, want)
+		}
+	}
+
+	errDial := errors.New("dial error")
+	f(nil, false)
+	f(errDial, false)
+	f(&net.OpError{Op: "dial", Net: "unix", Err: errDial}, false)
+	f(&net.OpError{Op: "dial", Net: "tcp", Addr: &net.TCPAddr{}, Err: errDial}, false)
+
+	errUnix := &net.OpError{
+		Op:   "dial",
+		Net:  "unix",
+		Addr: &net.UnixAddr{Name: "exporter.sock", Net: "unix"},
+		Err:  errDial,
+	}
+	f(errUnix, true)
+	f(fmt.Errorf("wrapped error: %w", errUnix), true)
+}
+
 func TestIsTCPv4Addr(t *testing.T) {
 	f := func(addr string, resultExpected bool) {
 		t.Helper()

lib/promscrape/discovery/docker/container.go

@@ -37,8 +37,9 @@ type containerNetworkSettings struct {
 }
 
 type containerNetwork struct {
-	IPAddress string
-	NetworkID string
+	GlobalIPv6Address string
+	IPAddress         string
+	NetworkID         string
 }
 
 func getContainersLabels(cfg *apiConfig) ([]*promutil.Labels, error) {
@@ -118,14 +119,18 @@ func addContainersLabels(containers []container, networkLabels map[string]*promu
 			networks = map[string]containerNetwork{firstNetworkMode: firstNetwork}
 		}
 		for _, n := range networks {
+			ipAddress := n.IPAddress
+			if len(ipAddress) == 0 {
+				ipAddress = n.GlobalIPv6Address
+			}
 			var added bool
 			for _, p := range c.Ports {
 				if p.Type != "tcp" {
 					continue
 				}
 				m := promutil.NewLabels(16)
-				m.Add("__address__", discoveryutil.JoinHostPort(n.IPAddress, p.PrivatePort))
-				m.Add("__meta_docker_network_ip", n.IPAddress)
+				m.Add("__address__", discoveryutil.JoinHostPort(ipAddress, p.PrivatePort))
+				m.Add("__meta_docker_network_ip", ipAddress)
 				m.Add("__meta_docker_port_private", strconv.Itoa(p.PrivatePort))
 				if p.PublicPort > 0 {
 					m.Add("__meta_docker_port_public", strconv.Itoa(p.PublicPort))
@@ -141,11 +146,11 @@ func addContainersLabels(containers []container, networkLabels map[string]*promu
 				// Use fallback port when no exposed ports are available or if all are non-TCP
 				addr := hostNetworkingHost
 				if c.HostConfig.NetworkMode != "host" {
-					addr = discoveryutil.JoinHostPort(n.IPAddress, defaultPort)
+					addr = discoveryutil.JoinHostPort(ipAddress, defaultPort)
 				}
 				m := promutil.NewLabels(16)
 				m.Add("__address__", addr)
-				m.Add("__meta_docker_network_ip", n.IPAddress)
+				m.Add("__meta_docker_network_ip", ipAddress)
 				addCommonLabels(m, c, networkLabels[n.NetworkID])
 				// Remove possible duplicate labels, which can appear after addCommonLabels() call
 				m.RemoveDuplicates()

lib/promscrape/discovery/docker/container_test.go

@@ -431,6 +431,96 @@ func TestAddContainerLabels(t *testing.T) {
 		}),
 	}
 	f(c, networkLabels, labelssExpected)
+
+	data = []byte(`[
+  {
+    "Name": "bridge",
+    "Id": "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
+    "Created": "2021-03-18T14:36:04.290821903+08:00",
+    "Scope": "local",
+    "Driver": "bridge",
+    "EnableIPv6": true,
+    "IPAM": {
+      "Driver": "default",
+      "Options": null,
+      "Config": [
+        {
+          "Subnet": "fc00::/64",
+          "Gateway": "fc00::1"
+        }
+      ]
+    },
+    "Internal": false,
+    "Attachable": false,
+    "Ingress": false,
+    "ConfigFrom": {
+      "Network": ""
+    },
+    "ConfigOnly": false,
+    "Containers": {},
+    "Options": {
+      "com.docker.network.bridge.default_bridge": "true",
+      "com.docker.network.bridge.enable_icc": "true",
+      "com.docker.network.bridge.enable_ip_masquerade": "true",
+      "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
+      "com.docker.network.bridge.name": "docker0",
+      "com.docker.network.driver.mtu": "1500"
+    },
+    "Labels": {}
+  }
+]`)
+	networks, err = parseNetworks(data)
+	if err != nil {
+		t.Fatalf("fail to parse networks: %v", err)
+	}
+	networkLabels = getNetworkLabelsByNetworkID(networks)
+
+	// NetworkMode != host
+	c = container{
+		ID:    "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
+		Names: []string{"/crow-server"},
+		Labels: map[string]string{
+			"com.docker.compose.config-hash":      "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
+			"com.docker.compose.container-number": "1",
+			"com.docker.compose.oneoff":           "False",
+			"com.docker.compose.project":          "crowserver",
+			"com.docker.compose.service":          "crow-server",
+			"com.docker.compose.version":          "1.11.2",
+		},
+		HostConfig: containerHostConfig{
+			NetworkMode: "bridge",
+		},
+		NetworkSettings: containerNetworkSettings{
+			Networks: map[string]containerNetwork{
+				"bridge": {
+					GlobalIPv6Address: "fc00::2",
+					NetworkID:         "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
+				},
+			},
+		},
+	}
+	labelssExpected = []*promutil.Labels{
+		promutil.NewLabelsFromMap(map[string]string{
+			"__address__":                "[fc00::2]:8012",
+			"__meta_docker_container_id": "90bc3b31aa13da5c0b11af2e228d54b38428a84e25d4e249ae9e9c95e51a0700",
+			"__meta_docker_container_label_com_docker_compose_config_hash":      "c9f0bd5bb31921f94cff367d819a30a0cc08d4399080897a6c5cd74b983156ec",
+			"__meta_docker_container_label_com_docker_compose_container_number": "1",
+			"__meta_docker_container_label_com_docker_compose_oneoff":           "False",
+			"__meta_docker_container_label_com_docker_compose_project":          "crowserver",
+			"__meta_docker_container_label_com_docker_compose_service":          "crow-server",
+			"__meta_docker_container_label_com_docker_compose_version":          "1.11.2",
+			"__meta_docker_container_name":                                      "/crow-server",
+			"__meta_docker_container_network_mode":                              "bridge",
+			"__meta_docker_network_id":                                          "1dd8d1a8bef59943345c7231d7ce8268333ff5a8c5b3c94881e6b4742b447634",
+			"__meta_docker_network_ingress":                                     "false",
+			"__meta_docker_network_internal":                                    "false",
+			"__meta_docker_network_ip":                                          "fc00::2",
+			"__meta_docker_network_name":                                        "bridge",
+			"__meta_docker_network_scope":                                       "local",
+		}),
+	}
+	f(c, networkLabels, labelssExpected)
+
 }
 
 func TestDockerMultiNetworkLabels(t *testing.T) {

lib/protoparser/vmimport/parser.go

@@ -197,6 +197,11 @@ func (tu *tagsUnmarshaler) addBytes(b []byte) []byte {
 func (tu *tagsUnmarshaler) unmarshalTags(o *fastjson.Object) error {
 	tu.err = nil
 	o.Visit(func(key []byte, v *fastjson.Value) {
+		if len(key) == 0 {
+			// Skip tags with empty name, since they override the metric name.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4962
+			return
+		}
 		tag := tu.addTag()
 		tag.Key = tu.addBytes(key)
 		sb, err := v.StringBytes()

lib/protoparser/vmimport/parser_test.go

@@ -140,6 +140,26 @@ func TestRowsUnmarshalSuccess(t *testing.T) {
 		}},
 	})
 
+	// Line with a tag with empty name.
+	// Such a tag must be skipped, since it overrides the metric name.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4962
+	f(`{"metric":{"__name__":"foo","":"bar","baz":"xx"},"values":[1.23],"timestamps":[456]}`, &Rows{
+		Rows: []Row{{
+			Tags: []Tag{
+				{
+					Key:   []byte("__name__"),
+					Value: []byte("foo"),
+				},
+				{
+					Key:   []byte("baz"),
+					Value: []byte("xx"),
+				},
+			},
+			Values:     []float64{1.23},
+			Timestamps: []int64{456},
+		}},
+	})
+
 	// Multiple lines
 	f(`{"metric":{"foo":"bar","baz":"xx"},"values":[1.23, -3.21],"timestamps" : [456,789]}
 {"metric":{"__name__":"xx"},"values":[34],"timestamps" : [11]}

lib/pushmetrics/pushmetrics.go

@@ -24,7 +24,9 @@ var (
 	disableCompression = flag.Bool("pushmetrics.disableCompression", false, "Whether to disable request body compression when pushing metrics to every -pushmetrics.url")
 )
 
-func init() {
+// InitSecretFlags manages the secret flags for this pkg and must be called by app-level initSecretFlags.
+// It should run before logger initialization and package Init() (if exists).
+func InitSecretFlags() {
 	// The -pushmetrics.url flag can contain basic auth creds, so it mustn't be visible when exposing the flags.
 	flagutil.RegisterSecretFlag("pushmetrics.url")
 }

lib/storage/index_db.go

@@ -99,7 +99,7 @@ type indexDB struct {
 	// legacy indexDBs, since these indexDBs are readonly.
 	// This field cannot be used for the partition indexDBs, since they may receive data
 	// with bigger timestamps at any time.
-	legacyMinMissingTimestampByKey map[string]int64
+	legacyMinMissingTimestampByKey map[TenantToken]int64
 	// protects legacyMinMissingTimestampByKey
 	legacyMinMissingTimestampByKeyLock sync.Mutex
 
@@ -174,7 +174,7 @@ func mustOpenIndexDB(id uint64, tr TimeRange, name, path string, s *Storage, isR
 	tfssCache := lrucache.NewCache(getTagFiltersCacheSize)
 	tb := mergeset.MustOpenTable(path, dataFlushInterval, tfssCache.Reset, 0, mergeTagToMetricIDsRows, isReadOnly)
 	db := &indexDB{
-		legacyMinMissingTimestampByKey: make(map[string]int64),
+		legacyMinMissingTimestampByKey: make(map[TenantToken]int64),
 		id:                             id,
 		tr:                             tr,
 		name:                           name,
@@ -508,6 +508,11 @@ func (db *indexDB) SearchLabelNames(qt *querytracer.Tracer, tfss []*TagFilters,
 	qt = qt.NewChild("search label names: filters=%s, timeRange=%s, maxLabelNames=%d, maxMetrics=%d", tfss, &tr, maxLabelNames, maxMetrics)
 	defer qt.Done()
 
+	if !db.legacyContainsTimeRange(tr) {
+		qt.Printf("indexDB doesn't contain data for the given time range: %v", &tr)
+		return nil, nil
+	}
+
 	is := db.getIndexSearch(deadline)
 	lns, err := is.searchLabelNamesWithFiltersOnTimeRange(qt, tfss, tr, maxLabelNames, maxMetrics)
 	db.putIndexSearch(is)
@@ -710,6 +715,11 @@ func (db *indexDB) SearchLabelValues(qt *querytracer.Tracer, labelName string, t
 	qt = qt.NewChild("search label values: labelName=%q, filters=%s, timeRange=%s, maxLabelValues=%d, maxMetrics=%d", labelName, tfss, &tr, maxLabelValues, maxMetrics)
 	defer qt.Done()
 
+	if !db.legacyContainsTimeRange(tr) {
+		qt.Printf("indexDB doesn't contain data for the given time range: %v", &tr)
+		return nil, nil
+	}
+
 	key := labelName
 	if key == "__name__" {
 		key = ""
@@ -958,6 +968,11 @@ func (db *indexDB) SearchTagValueSuffixes(qt *querytracer.Tracer, tr TimeRange,
 		&tr, tagKey, tagValuePrefix, delimiter, maxTagValueSuffixes)
 	defer qt.Done()
 
+	if !db.legacyContainsTimeRange(tr) {
+		qt.Printf("indexDB doesn't contain data for the given time range: %v", &tr)
+		return nil, nil
+	}
+
 	// TODO: cache results?
 
 	is := db.getIndexSearch(deadline)
@@ -1092,6 +1107,11 @@ func (db *indexDB) SearchGraphitePaths(qt *querytracer.Tracer, tr TimeRange, qHe
 	qt = qt.NewChild("search graphite paths: timeRange=%s, qHead=%q, qTail=%q, maxPaths=%d", &tr, bytesutil.ToUnsafeString(qHead), bytesutil.ToUnsafeString(qTail), maxPaths)
 	defer qt.Done()
 
+	if !db.legacyContainsTimeRange(tr) {
+		qt.Printf("indexDB doesn't contain data for the given time range: %v", &tr)
+		return nil, nil
+	}
+
 	n := bytes.IndexAny(qTail, "*[{")
 	if n < 0 {
 		// Verify that qHead matches a metric name.
@@ -1273,6 +1293,11 @@ func (db *indexDB) GetTSDBStatus(qt *querytracer.Tracer, tfss []*TagFilters, dat
 	qt = qt.NewChild("collect TSDB status: filters=%s, date=%s, focusLabel=%q, topN=%d, maxMetrics=%d", tfss, dateToString(date), focusLabel, topN, maxMetrics)
 	defer qt.Done()
 
+	if !db.legacyContainsDate(date) {
+		qt.Printf("indexDB doesn't contain data for the given date: %s", dateToString(date))
+		return &TSDBStatus{}, nil
+	}
+
 	is := db.getIndexSearch(deadline)
 	defer db.putIndexSearch(is)
 	status, err := is.getTSDBStatus(qt, tfss, date, focusLabel, topN, maxMetrics)
@@ -1720,6 +1745,11 @@ func (db *indexDB) SearchTSIDs(qt *querytracer.Tracer, tfss []*TagFilters, tr Ti
 	qt = qt.NewChild("search TSIDs: filters=%s, timeRange=%s, maxMetrics=%d", tfss, &tr, maxMetrics)
 	defer qt.Done()
 
+	if !db.legacyContainsTimeRange(tr) {
+		qt.Printf("indexDB doesn't contain data for the given time range: %v", &tr)
+		return nil, nil
+	}
+
 	metricIDs, err := db.searchMetricIDs(qt, tfss, tr, maxMetrics, deadline)
 	if err != nil {
 		return nil, db.wrapError("search TSIDs", err)
@@ -1803,6 +1833,11 @@ func (db *indexDB) SearchMetricNames(qt *querytracer.Tracer, tfss []*TagFilters,
 	qt = qt.NewChild("search metric names: filters=%s, timeRange=%s, maxMetrics=%d", tfss, &tr, maxMetrics)
 	defer qt.Done()
 
+	if !db.legacyContainsTimeRange(tr) {
+		qt.Printf("indexDB doesn't contain data for the given time range: %v", &tr)
+		return nil, nil
+	}
+
 	metricIDs, err := db.searchMetricIDs(qt, tfss, tr, maxMetrics, deadline)
 	if err != nil {
 		return nil, db.wrapError("search metric names", err)
@@ -2219,18 +2254,12 @@ func (is *indexSearch) searchMetricIDsInternal(qt *querytracer.Tracer, tfss []*T
 	qt = qt.NewChild("search for metric ids: filters=%s, timeRange=%s, maxMetrics=%d", tfss, &tr, maxMetrics)
 	defer qt.Done()
 
-	metricIDs := &uint64set.Set{}
-
-	if !is.legacyContainsTimeRange(tr) {
-		qt.Printf("indexdb doesn't contain data for the given timeRange=%s", &tr)
-		return metricIDs, nil
-	}
-
 	if tr.MinTimestamp >= is.db.s.minTimestampForCompositeIndex {
 		tfss = convertToCompositeTagFilterss(tfss)
 		qt.Printf("composite filters=%s", tfss)
 	}
 
+	metricIDs := &uint64set.Set{}
 	for _, tfs := range tfss {
 		if len(tfs.tfs) == 0 {
 			// An empty filters must be equivalent to `{__name__!=""}`

lib/storage/index_db_legacy.go

@@ -7,7 +7,6 @@ import (
 	"strconv"
 	"sync/atomic"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -87,48 +86,61 @@ func mustOpenLegacyIndexDB(path string, s *Storage) *legacyIndexDB {
 	return legacyIDB
 }
 
-func (is *indexSearch) legacyContainsTimeRange(tr TimeRange) bool {
+func (db *indexDB) legacyContainsDate(date uint64) bool {
+	var tr TimeRange
+	if date == globalIndexDate {
+		tr = globalIndexTimeRange
+	} else {
+		tr.MinTimestamp = int64(date) * msecPerDay
+		tr.MaxTimestamp = int64(date+1)*msecPerDay - 1
+	}
+	return db.legacyContainsTimeRange(tr)
+}
+
+func (db *indexDB) legacyContainsTimeRange(tr TimeRange) bool {
 	if tr == globalIndexTimeRange {
 		return true
 	}
 
-	db := is.db
 	if !db.noRegisterNewSeries.Load() {
 		// indexDB could register new time series - it is not safe to cache minMissingTimestamp
 		return true
 	}
 
-	// use common prefix as a key for minMissingTimestamp
-	// it's needed to properly track timestamps for cluster version
-	// which uses tenant labels for the index search
-	kb := &is.kb
-	kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixDateToMetricID)
-	key := kb.B
-
+	// vmsingle does not have tenants and therefore has just one key.
+	// While vmstorage can potentially have many tenants and the actual
+	// accountID and projectID will be set from the request.
+	key := TenantToken{
+		AccountID: 0,
+		ProjectID: 0,
+	}
 	db.legacyMinMissingTimestampByKeyLock.Lock()
-	minMissingTimestamp, ok := db.legacyMinMissingTimestampByKey[string(key)]
+	minMissingTimestamp, ok := db.legacyMinMissingTimestampByKey[key]
 	db.legacyMinMissingTimestampByKeyLock.Unlock()
 
 	if ok && tr.MinTimestamp >= minMissingTimestamp {
+		// Fast path.
 		return false
 	}
-	if is.legacyContainsTimeRangeSlow(kb, tr) {
+
+	// Slow path.
+	is := db.getIndexSearch(noDeadline)
+	defer db.putIndexSearch(is)
+	if is.legacyContainsTimeRange(tr) {
 		return true
 	}
 
 	db.legacyMinMissingTimestampByKeyLock.Lock()
-	minMissingTimestamp, ok = db.legacyMinMissingTimestampByKey[string(key)]
+	minMissingTimestamp, ok = db.legacyMinMissingTimestampByKey[key]
 	if !ok || tr.MinTimestamp < minMissingTimestamp {
-		db.legacyMinMissingTimestampByKey[string(key)] = tr.MinTimestamp
+		db.legacyMinMissingTimestampByKey[key] = tr.MinTimestamp
 	}
 	db.legacyMinMissingTimestampByKeyLock.Unlock()
 
 	return false
 }
 
-func (is *indexSearch) legacyContainsTimeRangeSlow(prefixBuf *bytesutil.ByteBuffer, tr TimeRange) bool {
-	ts := &is.ts
-
+func (is *indexSearch) legacyContainsTimeRange(tr TimeRange) bool {
 	// Verify whether the tr.MinTimestamp is included into `ts` or is smaller than the minimum date stored in `ts`.
 	// Do not check whether tr.MaxTimestamp is included into `ts` or is bigger than the max date stored in `ts` for performance reasons.
 	// This means that this func can return true if `tr` is located below the min date stored in `ts`.
@@ -136,12 +148,17 @@ func (is *indexSearch) legacyContainsTimeRangeSlow(prefixBuf *bytesutil.ByteBuff
 	// The main practical case allows skipping searching in prev indexdb (`ts`) when `tr`
 	// is located above the max date stored there.
 	minDate := uint64(tr.MinTimestamp) / msecPerDay
-	prefix := prefixBuf.B
-	prefixBuf.B = encoding.MarshalUint64(prefixBuf.B, minDate)
-	ts.Seek(prefixBuf.B)
+
+	kb := &is.kb
+	kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixDateToMetricID)
+	prefix := kb.B
+	kb.B = encoding.MarshalUint64(kb.B, minDate)
+
+	ts := &is.ts
+	ts.Seek(kb.B)
 	if !ts.NextItem() {
 		if err := ts.Error(); err != nil {
-			logger.Panicf("FATAL: error when searching for minDate=%d, prefix %q: %s", minDate, prefixBuf.B, err)
+			logger.Panicf("FATAL: error when searching for minDate=%d, prefix %q: %s", minDate, kb.B, err)
 		}
 		return false
 	}

lib/storage/index_db_legacy_test.go

@@ -29,11 +29,7 @@ func TestLegacyContainsTimeRange(t *testing.T) {
 
 	f := func(idb *indexDB, tr TimeRange, want bool) {
 		t.Helper()
-		is := idb.getIndexSearch(noDeadline)
-		defer idb.putIndexSearch(is)
-
-		got := is.legacyContainsTimeRange(tr)
-
+		got := idb.legacyContainsTimeRange(tr)
 		if got != want {
 			t.Fatalf("legacyContainsTimeRange(%s) for index db %s returns unexpected result: got %t, want %t", tr.String(), idb.name, got, want)
 		}
@@ -97,8 +93,8 @@ func TestLegacyContainsTimeRange(t *testing.T) {
 	f(legacyIDBs.getIDBCurr(), tr, true)
 	f(idb, tr, true)
 
-	// Fully inside trPt, overlaps with trPrev on the right side and trCurr on
-	// the left side.
+	// Fully inside trPt, overlaps with trPrev on the right side and with trCurr
+	// on the left side.
 	tr = TimeRange{
 		MinTimestamp: time.Date(2025, 1, 7, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2025, 1, 21, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),

lib/storage/index_db_test.go

@@ -2241,15 +2241,16 @@ func TestIndexSearchLegacyContainsTimeRange_Concurrent(t *testing.T) {
 	for i := range concurrency {
 		ts := minTimestamp + msecPerDay*i
 		wg.Go(func() {
-			is := idb.getIndexSearch(noDeadline)
-			_ = is.legacyContainsTimeRange(TimeRange{ts, ts})
-			idb.putIndexSearch(is)
+			_ = idb.legacyContainsTimeRange(TimeRange{ts, ts})
 		})
 	}
 	wg.Wait()
 
-	key := marshalCommonPrefix(nil, nsPrefixDateToMetricID)
-	if got, want := idb.legacyMinMissingTimestampByKey[string(key)], minTimestamp; got != want {
+	key := TenantToken{
+		AccountID: 0,
+		ProjectID: 0,
+	}
+	if got, want := idb.legacyMinMissingTimestampByKey[key], minTimestamp; got != want {
 		t.Fatalf("unexpected min timestamp: got %v, want %v", time.UnixMilli(got).UTC(), time.UnixMilli(want).UTC())
 	}
 }

lib/storage/partition.go

@@ -1290,6 +1290,9 @@ func (pt *partition) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isFi
 		putBlockStreamReader(bsr)
 	}
 	if err != nil {
+		if mpNew != nil {
+			putInmemoryPart(mpNew)
+		}
 		return err
 	}
 	if mpNew != nil {
@@ -1444,6 +1447,8 @@ func (pt *partition) openCreatedPart(ph *partHeader, pws []*partWrapper, mpNew *
 		// The created part is empty. Remove it
 		if mpNew == nil {
 			fs.MustRemoveDir(dstPartPath)
+		} else {
+			putInmemoryPart(mpNew)
 		}
 		return nil
 	}

lib/storage/storage_legacy_test.go

@@ -33,16 +33,22 @@ func TestLegacyStorage_SearchMetricNames(t *testing.T) {
 		return mrs, want
 	}
 	const numMetrics = 1000
-	tr := TimeRange{
+	tr1 := TimeRange{
 		MinTimestamp: time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2024, 5, 31, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	legacyData, wantLegacy := genData(numMetrics, "legacy", tr)
-	newData, wantNew := genData(numMetrics, "new", tr)
-	wantNew = append(wantNew, wantLegacy...)
-	slices.Sort(wantNew)
+	tr2 := TimeRange{
+		MinTimestamp: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2024, 6, 30, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	legacyData, wantLegacy := genData(numMetrics, "legacy", tr1)
+	new1Data, wantNew1 := genData(numMetrics, "new1", tr1)
+	new2Data, wantNew2 := genData(numMetrics, "new2", tr2)
+	newData := slices.Concat(new1Data, new2Data)
+	wantLegacyAndNew1 := slices.Concat(wantLegacy, wantNew1)
+	slices.Sort(wantLegacyAndNew1)
 
-	assertSearchResults := func(s *Storage, want []string) {
+	assertSearchResults := func(s *Storage, tr TimeRange, want []string) {
 		t.Helper()
 		tfsAll := NewTagFilters()
 		if err := tfsAll.Add([]byte("__name__"), []byte(".*"), false, true); err != nil {
@@ -67,10 +73,11 @@ func TestLegacyStorage_SearchMetricNames(t *testing.T) {
 	}
 
 	assertLegacyData := func(s *Storage) {
-		assertSearchResults(s, wantLegacy)
+		assertSearchResults(s, tr1, wantLegacy)
 	}
 	assertNewData := func(s *Storage) {
-		assertSearchResults(s, wantNew)
+		assertSearchResults(s, tr1, wantLegacyAndNew1)
+		assertSearchResults(s, tr2, wantNew2)
 	}
 	testSearchOpWithLegacyIndexDBs(t, legacyData, newData, assertLegacyData, assertNewData)
 }
@@ -95,15 +102,22 @@ func TestLegacyStorage_SearchLabelNames(t *testing.T) {
 		return mrs, want
 	}
 	const numMetrics = 1000
-	tr := TimeRange{
+	tr1 := TimeRange{
 		MinTimestamp: time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2024, 5, 31, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	legacyData, wantLegacy := genData(numMetrics, "legacy", tr)
-	newData, wantNew := genData(numMetrics, "new", tr)
-	wantNew = append(wantNew, wantLegacy...)
+	tr2 := TimeRange{
+		MinTimestamp: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2024, 6, 30, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	legacyData, wantLegacy := genData(numMetrics, "legacy", tr1)
+	new1Data, wantNew1 := genData(numMetrics, "new1", tr1)
+	new2Data, wantNew2 := genData(numMetrics, "new2", tr2)
+	newData := slices.Concat(new1Data, new2Data)
+	wantLegacyAndNew1 := slices.Concat(wantLegacy, wantNew1)
+	slices.Sort(wantLegacyAndNew1)
 
-	assertSearchResults := func(s *Storage, want []string) {
+	assertSearchResults := func(s *Storage, tr TimeRange, want []string) {
 		t.Helper()
 		got, err := s.SearchLabelNames(nil, nil, tr, 1e9, 1e9, noDeadline)
 		if err != nil {
@@ -118,12 +132,16 @@ func TestLegacyStorage_SearchLabelNames(t *testing.T) {
 	assertLegacyData := func(s *Storage) {
 		want := append(wantLegacy, "__name__")
 		slices.Sort(want)
-		assertSearchResults(s, want)
+		assertSearchResults(s, tr1, want)
 	}
 	assertNewData := func(s *Storage) {
-		want := append(wantNew, "__name__")
+		want := append(wantLegacyAndNew1, "__name__")
 		slices.Sort(want)
-		assertSearchResults(s, want)
+		assertSearchResults(s, tr1, want)
+
+		want = append(wantNew2, "__name__")
+		slices.Sort(want)
+		assertSearchResults(s, tr2, want)
 	}
 	testSearchOpWithLegacyIndexDBs(t, legacyData, newData, assertLegacyData, assertNewData)
 }
@@ -148,16 +166,22 @@ func TestLegacyStorage_SearchLabelValues(t *testing.T) {
 		return mrs, want
 	}
 	const numMetrics = 1000
-	tr := TimeRange{
+	tr1 := TimeRange{
 		MinTimestamp: time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2024, 5, 31, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	legacyData, wantLegacy := genData(numMetrics, "legacy", tr)
-	newData, wantNew := genData(numMetrics, "new", tr)
-	wantNew = append(wantNew, wantLegacy...)
-	slices.Sort(wantNew)
+	tr2 := TimeRange{
+		MinTimestamp: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2024, 6, 30, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	legacyData, wantLegacy := genData(numMetrics, "legacy", tr1)
+	new1Data, wantNew1 := genData(numMetrics, "new1", tr1)
+	new2Data, wantNew2 := genData(numMetrics, "new2", tr2)
+	newData := slices.Concat(new1Data, new2Data)
+	wantLegacyAndNew1 := slices.Concat(wantLegacy, wantNew1)
+	slices.Sort(wantLegacyAndNew1)
 
-	assertSearchResults := func(s *Storage, want []string) {
+	assertSearchResults := func(s *Storage, tr TimeRange, want []string) {
 		t.Helper()
 		got, err := s.SearchLabelValues(nil, "label", nil, tr, 1e9, 1e9, noDeadline)
 		if err != nil {
@@ -171,11 +195,12 @@ func TestLegacyStorage_SearchLabelValues(t *testing.T) {
 
 	assertLegacyData := func(s *Storage) {
 		t.Helper()
-		assertSearchResults(s, wantLegacy)
+		assertSearchResults(s, tr1, wantLegacy)
 	}
 	assertNewData := func(s *Storage) {
 		t.Helper()
-		assertSearchResults(s, wantNew)
+		assertSearchResults(s, tr1, wantLegacyAndNew1)
+		assertSearchResults(s, tr2, wantNew2)
 	}
 	testSearchOpWithLegacyIndexDBs(t, legacyData, newData, assertLegacyData, assertNewData)
 }
@@ -197,16 +222,22 @@ func TestLegacyStorage_SearchTagValueSuffixes(t *testing.T) {
 		return mrs, want
 	}
 	const numMetrics = 1000
-	tr := TimeRange{
+	tr1 := TimeRange{
 		MinTimestamp: time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2024, 5, 31, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	legacyData, wantLegacy := genData(numMetrics, "legacy", tr)
-	newData, wantNew := genData(numMetrics, "new", tr)
-	wantNew = append(wantNew, wantLegacy...)
-	slices.Sort(wantNew)
+	tr2 := TimeRange{
+		MinTimestamp: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2024, 6, 30, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	legacyData, wantLegacy := genData(numMetrics, "legacy", tr1)
+	new1Data, wantNew1 := genData(numMetrics, "new1", tr1)
+	new2Data, wantNew2 := genData(numMetrics, "new2", tr2)
+	newData := slices.Concat(new1Data, new2Data)
+	wantLegacyAndNew1 := slices.Concat(wantLegacy, wantNew1)
+	slices.Sort(wantLegacyAndNew1)
 
-	assertSearchResults := func(s *Storage, want []string) {
+	assertSearchResults := func(s *Storage, tr TimeRange, want []string) {
 		t.Helper()
 		got, err := s.SearchTagValueSuffixes(nil, tr, "", "prefix.", '.', 1e9, noDeadline)
 		if err != nil {
@@ -221,11 +252,12 @@ func TestLegacyStorage_SearchTagValueSuffixes(t *testing.T) {
 
 	assertLegacyData := func(s *Storage) {
 		t.Helper()
-		assertSearchResults(s, wantLegacy)
+		assertSearchResults(s, tr1, wantLegacy)
 	}
 	assertNewData := func(s *Storage) {
 		t.Helper()
-		assertSearchResults(s, wantNew)
+		assertSearchResults(s, tr1, wantLegacyAndNew1)
+		assertSearchResults(s, tr2, wantNew2)
 	}
 	testSearchOpWithLegacyIndexDBs(t, legacyData, newData, assertLegacyData, assertNewData)
 }
@@ -247,16 +279,22 @@ func TestLegacyStorage_SearchGraphitePaths(t *testing.T) {
 		return mrs, want
 	}
 	const numMetrics = 1000
-	tr := TimeRange{
+	tr1 := TimeRange{
 		MinTimestamp: time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2024, 5, 31, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	legacyData, wantLegacy := genData(numMetrics, "legacy", tr)
-	newData, wantNew := genData(numMetrics, "new", tr)
-	wantNew = append(wantNew, wantLegacy...)
-	slices.Sort(wantNew)
+	tr2 := TimeRange{
+		MinTimestamp: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2024, 6, 30, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	legacyData, wantLegacy := genData(numMetrics, "legacy", tr1)
+	new1Data, wantNew1 := genData(numMetrics, "new1", tr1)
+	new2Data, wantNew2 := genData(numMetrics, "new2", tr2)
+	newData := slices.Concat(new1Data, new2Data)
+	wantLegacyAndNew1 := slices.Concat(wantLegacy, wantNew1)
+	slices.Sort(wantLegacyAndNew1)
 
-	assertSearchResults := func(s *Storage, want []string) {
+	assertSearchResults := func(s *Storage, tr TimeRange, want []string) {
 		t.Helper()
 		got, err := s.SearchGraphitePaths(nil, tr, []byte("*.*"), 1e9, noDeadline)
 		if err != nil {
@@ -271,16 +309,17 @@ func TestLegacyStorage_SearchGraphitePaths(t *testing.T) {
 
 	assertLegacyData := func(s *Storage) {
 		t.Helper()
-		assertSearchResults(s, wantLegacy)
+		assertSearchResults(s, tr1, wantLegacy)
 	}
 	assertNewData := func(s *Storage) {
 		t.Helper()
-		assertSearchResults(s, wantNew)
+		assertSearchResults(s, tr1, wantLegacyAndNew1)
+		assertSearchResults(s, tr2, wantNew2)
 	}
 	testSearchOpWithLegacyIndexDBs(t, legacyData, newData, assertLegacyData, assertNewData)
 }
 
-func TestLegacyStorage_Search(t *testing.T) {
+func TestLegacyStorage_SearchData(t *testing.T) {
 	genData := func(numMetrics int, prefix string, tr TimeRange) []MetricRow {
 		mrs := make([]MetricRow, numMetrics)
 		for i := range numMetrics {
@@ -295,14 +334,20 @@ func TestLegacyStorage_Search(t *testing.T) {
 		return mrs
 	}
 	const numMetrics = 1000
-	tr := TimeRange{
+	tr1 := TimeRange{
 		MinTimestamp: time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
 		MaxTimestamp: time.Date(2024, 5, 31, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
 	}
-	legacyData := genData(numMetrics, "legacy", tr)
-	newData := genData(numMetrics, "new", tr)
+	tr2 := TimeRange{
+		MinTimestamp: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
+		MaxTimestamp: time.Date(2024, 6, 30, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
+	}
+	legacyData := genData(numMetrics, "legacy", tr1)
+	new1Data := genData(numMetrics, "new1", tr1)
+	new2Data := genData(numMetrics, "new2", tr2)
+	newData := slices.Concat(new1Data, new2Data)
 
-	assertSearchResults := func(s *Storage, want []MetricRow) {
+	assertSearchResults := func(s *Storage, tr TimeRange, want []MetricRow) {
 		tfsAll := NewTagFilters()
 		if err := tfsAll.Add([]byte("__name__"), []byte(".*"), false, true); err != nil {
 			t.Fatalf("unexpected error in TagFilters.Add: %v", err)
@@ -314,13 +359,13 @@ func TestLegacyStorage_Search(t *testing.T) {
 
 	assertLegacyData := func(s *Storage) {
 		t.Helper()
-		want := legacyData
-		assertSearchResults(s, want)
+		assertSearchResults(s, tr1, legacyData)
 	}
 	assertNewData := func(s *Storage) {
 		t.Helper()
-		want := slices.Concat(legacyData, newData)
-		assertSearchResults(s, want)
+		want := slices.Concat(legacyData, new1Data)
+		assertSearchResults(s, tr1, want)
+		assertSearchResults(s, tr2, new2Data)
 	}
 	testSearchOpWithLegacyIndexDBs(t, legacyData, newData, assertLegacyData, assertNewData)
 }

lib/workingsetcache/cache.go

@@ -30,12 +30,6 @@ const (
 	modeWhole     = 2
 )
 
-const (
-	// minCurrCacheSaveMissRate is the minimum miss rate of curr cache
-	// for saving prev instead of curr during split mode.
-	minCurrCacheSaveMissRate = 0.8
-)
-
 // Cache is a cache for working set entries.
 //
 // The cache evicts inactive entries after the given expireDuration.
@@ -47,10 +41,6 @@ type Cache struct {
 	// csHistory holds cache stats history
 	csHistory fastcache.Stats
 
-	// prevStatsAtRotation holds prev cache stats at the moment it became prev from curr.
-	// It is used for calculating prev miss rate since the last cache rotation.
-	prevStatsAtRotation fastcache.Stats
-
 	// mode indicates whether to use only curr and skip prev.
 	//
 	// This flag is set to modeSwitching if curr is filled for more than 50% space.
@@ -155,7 +145,6 @@ func newCacheInternal(curr, prev *fastcache.Cache, mode, maxBytes int, expireDur
 	c.maxBytes = maxBytes
 	c.curr.Store(curr)
 	c.prev.Store(prev)
-	prev.UpdateStats(&c.prevStatsAtRotation)
 	c.stopCh = make(chan struct{})
 	c.mode.Store(uint32(mode))
 	c.runWatchers(expireDuration)
@@ -195,7 +184,7 @@ func (c *Cache) expirationWatcher(expireDuration time.Duration) {
 		prev := c.prev.Load()
 		curr := c.curr.Load()
 		c.updateCacheStatsHistoryBeforeRotationLocked(prev, curr)
-		c.storeCurrStatsBeforeRotationLocked(curr)
+
 		c.prev.Store(curr)
 		prev.Reset()
 		c.curr.Store(prev)
@@ -316,7 +305,7 @@ func (c *Cache) transitIntoWholeModeLocked(maxBytesSize uint64, t *time.Ticker)
 	prev := c.prev.Load()
 	curr := c.curr.Load()
 	c.updateCacheStatsHistoryBeforeRotationLocked(prev, curr)
-	c.storeCurrStatsBeforeRotationLocked(curr)
+
 	c.prev.Store(curr)
 	prev.Reset()
 
@@ -367,7 +356,6 @@ func (c *Cache) transitIntoWholeModeLocked(maxBytesSize uint64, t *time.Ticker)
 	c.updateCacheStatsHistoryBeforeRotationLocked(prev, curr)
 
 	c.prev.Store(newWithAutoCleanup(1024))
-	c.prevStatsAtRotation.Reset()
 	prev.Reset()
 }
 
@@ -375,15 +363,14 @@ func (c *Cache) transitIntoWholeModeLocked(maxBytesSize uint64, t *time.Ticker)
 func (c *Cache) MustSave(filePath string) {
 	startTime := time.Now()
 
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	cacheToSave, cs, cacheName := c.selectCacheToSave()
+	var cs fastcache.Stats
+	curr := c.curr.Load()
+	curr.UpdateStats(&cs)
 
 	concurrency := cgroup.AvailableCPUs()
 
-	logger.Infof("saving %s cache to %s by using %d concurrent workers", cacheName, filePath, concurrency)
-	err := cacheToSave.SaveToFileConcurrent(filePath, concurrency)
+	logger.Infof("saving cache to %s by using %d concurrent workers", filePath, concurrency)
+	err := curr.SaveToFileConcurrent(filePath, concurrency)
 	if err != nil {
 		logger.Panicf("FATAL: cannot save cache to %s: %s", filePath, err)
 	}
@@ -391,47 +378,6 @@ func (c *Cache) MustSave(filePath string) {
 	logger.Infof("cache has been successfully saved to %s in %.3f seconds; entriesCount: %d, sizeBytes: %d", filePath, time.Since(startTime).Seconds(), cs.EntriesCount, cs.BytesSize)
 }
 
-func (c *Cache) selectCacheToSave() (*fastcache.Cache, fastcache.Stats, string) {
-	curr := c.curr.Load()
-
-	var csCurr fastcache.Stats
-	curr.UpdateStats(&csCurr)
-
-	if c.mode.Load() != modeSplit {
-		return curr, csCurr, "curr"
-	}
-
-	prev := c.prev.Load()
-
-	var csPrev fastcache.Stats
-	prev.UpdateStats(&csPrev)
-
-	if csPrev.EntriesCount == 0 || csPrev.GetCalls == 0 {
-		return curr, csCurr, "curr"
-	}
-
-	csPrevAtRotation := &c.prevStatsAtRotation
-	prevMissRateAfterRotation := float64(1)
-	if csPrev.GetCalls > csPrevAtRotation.GetCalls {
-		prevGetCallsAfterRotation := csPrev.GetCalls - csPrevAtRotation.GetCalls
-		prevMissesAfterRotation := uint64(0)
-		if csPrev.Misses > csPrevAtRotation.Misses {
-			prevMissesAfterRotation = csPrev.Misses - csPrevAtRotation.Misses
-		}
-		if prevMissesAfterRotation < prevGetCallsAfterRotation {
-			prevMissRateAfterRotation = float64(prevMissesAfterRotation) / float64(prevGetCallsAfterRotation)
-		}
-	}
-
-	// Prefer saving prev cache when:
-	// 1. 80% requests were missed in curr cache and served by prev cache.
-	// 2. less than 80% requests were missed in prev cache since the last rotation.
-	if csCurr.GetCalls < 10 || (float64(csCurr.Misses)/float64(csCurr.GetCalls) > minCurrCacheSaveMissRate && prevMissRateAfterRotation < minCurrCacheSaveMissRate) {
-		return prev, csPrev, "prev"
-	}
-	return curr, csCurr, "curr"
-}
-
 // Stop stops the cache.
 //
 // The cache cannot be used after the Stop call.
@@ -460,18 +406,12 @@ func (c *Cache) Reset() {
 		// so we have to restore it into original size for split mode
 		c.prev.Store(newWithAutoCleanup(c.maxBytes / 2))
 		c.curr.Store(newWithAutoCleanup(c.maxBytes / 2))
-		c.prevStatsAtRotation.Reset()
+
 		c.mode.Store(modeSplit)
 	}
 
 	prev.Reset()
 	curr.Reset()
-	c.prevStatsAtRotation.Reset()
-}
-
-func (c *Cache) storeCurrStatsBeforeRotationLocked(curr *fastcache.Cache) {
-	c.prevStatsAtRotation.Reset()
-	curr.UpdateStats(&c.prevStatsAtRotation)
 }
 
 // UpdateStats updates fcs with cache stats.

lib/workingsetcache/cache_synctest_test.go

@@ -5,7 +5,6 @@ package workingsetcache
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"testing"
 	"testing/synctest"
 	"time"
@@ -166,163 +165,6 @@ func TestSetGetStatsInSplitMode_cacheLoadedFromEmptyFile(t *testing.T) {
 	})
 }
 
-func TestMustSaveSelectsCacheInSplitMode(t *testing.T) {
-	t.Run("prefers prev cache if curr is rarely visited", func(t *testing.T) {
-		cachePath := filepath.Join(t.TempDir(), "cache")
-		synctest.Test(t, func(t *testing.T) {
-			var (
-				k   = []byte("k")
-				v   = []byte("v")
-				dst []byte
-			)
-
-			c := Load(cachePath, 1024)
-			c.Set(k, v)
-			for range 10 {
-				dst = c.Get(dst[:0], k)
-			}
-
-			// prev and curr were rotated, k is now in prev, curr is empty.
-			time.Sleep(*cacheExpireDuration + time.Minute)
-			synctest.Wait()
-			assertMode(t, c, modeSplit)
-
-			c.MustSave(cachePath)
-			c.Stop()
-
-			c = Load(cachePath, 1024)
-			defer c.Stop()
-			if got := c.Get(dst[:0], k); string(got) != string(v) {
-				t.Fatalf("unexpected value loaded from saved cache; got %q; want %q", got, v)
-			}
-		})
-	})
-
-	t.Run("prefers prev cache when prev is still useful", func(t *testing.T) {
-		cachePath := filepath.Join(t.TempDir(), "cache")
-		synctest.Test(t, func(t *testing.T) {
-			const keysCount = 10
-			var (
-				v   = []byte("v")
-				dst []byte
-			)
-
-			c := Load(cachePath, 1024)
-			for i := range keysCount {
-				c.Set([]byte(fmt.Sprintf("prev_%d", i)), v)
-			}
-
-			// prev and curr were rotated, prev_0-prev_9 are now in prev, curr is empty.
-			time.Sleep(*cacheExpireDuration + time.Minute)
-			synctest.Wait()
-			assertMode(t, c, modeSplit)
-
-			// all get calls are missed in curr cache, but can be served by prev cache.
-			for i := range keysCount {
-				dst = c.Get(dst[:0], []byte(fmt.Sprintf("prev_%d", i)))
-				if string(dst) != string(v) {
-					t.Fatalf("unexpected value loaded from prev cache for key %q; got %q; want %q", fmt.Sprintf("prev_%d", i), dst, v)
-				}
-			}
-
-			c.MustSave(cachePath)
-			c.Stop()
-
-			c = Load(cachePath, 1024)
-			defer c.Stop()
-			for i := range keysCount {
-				key := []byte(fmt.Sprintf("prev_%d", i))
-				if got := c.Get(dst[:0], key); string(got) != string(v) {
-					t.Fatalf("unexpected value loaded from saved cache for key %q; got %q; want %q", key, got, v)
-				}
-			}
-		})
-	})
-
-	t.Run("prefers curr cache when prev is cold", func(t *testing.T) {
-		cachePath := filepath.Join(t.TempDir(), "cache")
-		synctest.Test(t, func(t *testing.T) {
-			const keysCount = 10
-			var (
-				v   = []byte("v")
-				dst []byte
-			)
-
-			c := Load(cachePath, 1024)
-			for i := range keysCount {
-				c.Set([]byte(fmt.Sprintf("prev_%d", i)), v)
-			}
-
-			// prev and curr were rotated, prev_0-prev_9 are now in prev, curr is empty.
-			time.Sleep(*cacheExpireDuration + time.Minute)
-			synctest.Wait()
-			assertMode(t, c, modeSplit)
-
-			// all get calls are missed in both curr and prev cache.
-			for i := range keysCount {
-				newKey := []byte(fmt.Sprintf("new_%d", i))
-				dst = c.Get(dst[:0], newKey)
-			}
-
-			c.MustSave(cachePath)
-			c.Stop()
-
-			c = Load(cachePath, 1024)
-			defer c.Stop()
-			for i := range keysCount {
-				prevKey := []byte(fmt.Sprintf("prev_%d", i))
-				if got := c.Get(dst[:0], prevKey); len(got) != 0 {
-					t.Fatalf("unexpected prev value loaded from saved cache for key %q; got %q; want an empty value", prevKey, got)
-				}
-			}
-		})
-	})
-
-	t.Run("prefers curr cache when prev hits rate is low after rotation", func(t *testing.T) {
-		cachePath := filepath.Join(t.TempDir(), "cache")
-		synctest.Test(t, func(t *testing.T) {
-			const keysCount = 10
-			var (
-				prevKey = []byte("prev")
-				currKey = []byte("curr")
-				v       = []byte("v")
-				dst     []byte
-			)
-
-			c := Load(cachePath, 1024)
-			c.Set(prevKey, v)
-			// the curr cache hit all the requests.
-			for range keysCount {
-				dst = c.Get(dst[:0], prevKey)
-			}
-
-			// prev and curr were rotated, prevKey is now in prev, whose cache hit ratio is 100%.
-			time.Sleep(*cacheExpireDuration + time.Minute)
-			synctest.Wait()
-			assertMode(t, c, modeSplit)
-
-			c.Set(currKey, v)
-			// the prev cache miss all the requests after the rotation
-			for i := range keysCount {
-				newKey := []byte(fmt.Sprintf("new_%d", i))
-				dst = c.Get(dst[:0], newKey)
-			}
-
-			c.MustSave(cachePath)
-			c.Stop()
-
-			c = Load(cachePath, 1024)
-			defer c.Stop()
-			if got := c.Get(dst[:0], currKey); string(got) != string(v) {
-				t.Fatalf("unexpected value loaded from saved cache for key %q; got %q; want %q", currKey, got, v)
-			}
-			if got := c.Get(dst[:0], prevKey); len(got) != 0 {
-				t.Fatalf("unexpected prev value loaded from saved cache for key %q; got %q; want an empty value", prevKey, got)
-			}
-		})
-	})
-}
-
 func testSetGetStatsInSplitMode(t *testing.T, c *Cache) {
 	var (
 		k1, v1  = []byte("k1"), []byte("v1")